Archived
Compare commits
207
Commits
34b105ea88
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
56e89d62f1 | ||
|
|
5970367173 | ||
|
|
4964ad1037 | ||
|
|
5c281ad99b | ||
|
|
29dfb28f38 | ||
|
|
88b16e9be2 | ||
|
|
b564c315d0 | ||
|
|
5bd03b3984 | ||
|
|
e02fcad739 | ||
|
|
4a92ec6102 | ||
|
|
909cc25169 | ||
|
|
272d5ebd61 | ||
|
|
a6386ff23a | ||
|
|
b51105908a | ||
|
|
8b4d536a11 | ||
|
|
dc90b3f5bd | ||
|
|
0e9c4c8299 | ||
|
|
14d10051e5 | ||
|
|
c58b311ea7 | ||
|
|
7a169eae76 | ||
|
|
a07621d761 | ||
|
|
d791052f77 | ||
|
|
22935b9202 | ||
|
|
efe745f2be | ||
|
|
ff912d0972 | ||
|
|
4c9be8efc8 | ||
|
|
62c05c681c | ||
|
|
bfb728730f | ||
|
|
159db37618 | ||
|
|
b7c3868980 | ||
|
|
ddae6729c1 | ||
|
|
a4426190b0 | ||
|
|
2154ea8e3c | ||
|
|
812c528479 | ||
|
|
8b32e04e2f | ||
|
|
79f44386d2 | ||
|
|
6fe9efa712 | ||
|
|
d96542cd25 | ||
|
|
80dc6980b7 | ||
|
|
80df1b6361 | ||
|
|
816cf96459 | ||
|
|
3f05df5d1a | ||
|
|
e04846e9fa | ||
|
|
020c6a99b0 | ||
|
|
5a9766d4a4 | ||
|
|
f46a08772f | ||
|
|
1c2adfdca9 | ||
|
|
0fbb5ef359 | ||
|
|
8f84731896 | ||
|
|
fbaa463a38 | ||
|
|
2dafbe60e9 | ||
|
|
8d58afea6c | ||
|
|
b611f7375a | ||
|
|
253bbcea2d | ||
|
|
2c6382346f | ||
|
|
b8cafccaf6 | ||
|
|
82d212a2a1 | ||
|
|
df8ebf6a77 | ||
|
|
dc414fbfef | ||
|
|
fd891688ac | ||
|
|
9d8a5a7ede | ||
|
|
4e9e5c27d4 | ||
|
|
7c8a2e03dd | ||
|
|
89f52a6504 | ||
|
|
fb2f5a2962 | ||
|
|
2ecfbb3a29 | ||
|
|
9ad8715871 | ||
|
|
e1a55f099b | ||
|
|
ccc7ba334c | ||
|
|
65d5ed3367 | ||
|
|
08a261ac1a | ||
|
|
23226384e5 | ||
|
|
350f8ebdfe | ||
|
|
09d2cc5056 | ||
|
|
47e64f0284 | ||
|
|
6927b02d4d | ||
|
|
fda4859a95 | ||
|
|
a81717e53c | ||
|
|
d01839c29c | ||
|
|
f67d15536b | ||
|
|
7cd4d18bd5 | ||
|
|
fbf14b68db | ||
|
|
79ac62ffbc | ||
|
|
40034be479 | ||
|
|
045f25bd13 | ||
|
|
ff3cc47027 | ||
|
|
31a2bcdb3c | ||
|
|
2df47d7b1c | ||
|
|
666d1b682e | ||
|
|
e40b1e70db | ||
|
|
6b116586fc | ||
|
|
bf20d9d8cf | ||
|
|
81e335e42d | ||
|
|
2407bfb9ff | ||
|
|
850f4b4d26 | ||
|
|
e7673e3bd3 | ||
|
|
3e39ccf958 | ||
|
|
94bb74966f | ||
|
|
bfb271a91b | ||
|
|
9ff47f6aaa | ||
|
|
ca5f74eadb | ||
|
|
c3db00ceac | ||
|
|
8e02c4726e | ||
|
|
cc5c9ae888 | ||
|
|
c7e31640cf | ||
|
|
c75d726205 | ||
|
|
a4cd5ba375 | ||
|
|
62802cd966 | ||
|
|
e9a885cda9 | ||
|
|
2814262ac0 | ||
|
|
0a536d7b6d | ||
|
|
2bcd85d2df | ||
|
|
ac34bce49f | ||
|
|
af2b0844a1 | ||
|
|
3b4cb69e84 | ||
|
|
c47a590463 | ||
|
|
f35aa35541 | ||
|
|
8114cee2af | ||
|
|
baa8f58473 | ||
|
|
0470869fbb | ||
|
|
8a3c0cd022 | ||
|
|
0fff3efd4e | ||
|
|
eafd1883f6 | ||
|
|
e67916bbd8 | ||
|
|
7039c1b4b3 | ||
|
|
3dadc46ee6 | ||
|
|
d8aa565fee | ||
|
|
82c89bec65 | ||
|
|
b1f758ed71 | ||
|
|
ba4d156573 | ||
|
|
9c38aed789 | ||
|
|
a3f1d0460d | ||
|
|
0af12b369b | ||
|
|
e338b1f0f4 | ||
|
|
e42818a0ab | ||
|
|
bdb8193456 | ||
|
|
ca25e4ab45 | ||
|
|
7a64075702 | ||
|
|
9abe0b59fa | ||
|
|
183d7a99ff | ||
|
|
fad3d7ee71 | ||
|
|
3c50e80972 | ||
|
|
6e4bca7fce | ||
|
|
f4998a7266 | ||
|
|
53d9c4de19 | ||
|
|
9372733861 | ||
|
|
d640ef15ec | ||
|
|
28e1b29e0c | ||
|
|
f0815c4f18 | ||
|
|
72bbdeed35 | ||
|
|
dfd81d8daf | ||
|
|
8c54139905 | ||
|
|
6ed5de186d | ||
|
|
8fab4d8b22 | ||
|
|
b879128efb | ||
|
|
9641b88725 | ||
|
|
36704abc08 | ||
|
|
7fccba4658 | ||
|
|
98e948e30e | ||
|
|
f18bac4533 | ||
|
|
e05481e992 | ||
|
|
dd58fbcb2c | ||
|
|
1ed5afe679 | ||
|
|
38523c6509 | ||
|
|
a78f244621 | ||
|
|
4ab4529440 | ||
|
|
370673195c | ||
|
|
3ad12f8ec7 | ||
|
|
f6e78c837f | ||
|
|
35e330c7f0 | ||
|
|
aa7e53112b | ||
|
|
af6bcea41e | ||
|
|
49fc8b6d7d | ||
|
|
f20d221a60 | ||
|
|
bd8ea8ad4f | ||
|
|
1350c428ed | ||
|
|
76c48fa074 | ||
|
|
df6aab6f44 | ||
|
|
7f8e5a821c | ||
|
|
3db472ce56 | ||
|
|
3ae2d6e17c | ||
|
|
29ef458d89 | ||
|
|
9d0798a1c0 | ||
|
|
0e85230769 | ||
|
|
ad0aabc2a9 | ||
|
|
fe3faaef83 | ||
|
|
8f90d3ecd6 | ||
|
|
b4a62e9e6e | ||
|
|
03894e5e51 | ||
|
|
565de8c46a | ||
|
|
c19d4516ef | ||
|
|
a8d63653fb | ||
|
|
aaadb5695c | ||
|
|
73a31eb205 | ||
|
|
f56458da32 | ||
|
|
66650e315c | ||
|
|
c40e15268e | ||
|
|
0f8c47842a | ||
|
|
aee0a2718f | ||
|
|
4f95be7e08 | ||
|
|
788fb681ec | ||
|
|
96d7784bce | ||
|
|
dc456d5660 | ||
|
|
ecbb317dc3 | ||
|
|
dd2d8b91be | ||
|
|
8831c74f70 | ||
|
|
a96bd2ff38 |
-43
@@ -1,43 +0,0 @@
|
|||||||
# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq
|
|
||||||
# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how
|
|
||||||
# `make image` broke once before (issue 146).
|
|
||||||
ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
|
||||||
# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go
|
|
||||||
# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it.
|
|
||||||
# Genesis builds this file and raises it as the container the process replaces on the first push
|
|
||||||
# (mesh-host internal/bootstrap, novox/hq issue 223).
|
|
||||||
#
|
|
||||||
# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
|
|
||||||
# machine where no mesh exists yet, and run before there is anything to check it against. So it
|
|
||||||
# holds the program and nothing else — no shell, no package manager, no libc, nothing with a CVE
|
|
||||||
# feed of its own. What a person has to audit before trusting a first node is one binary.
|
|
||||||
#
|
|
||||||
# There are no CA certificates in here on purpose. Nothing it does today makes an outbound TLS
|
|
||||||
# connection to a public name: it reaches PostgreSQL on the machine it was raised on, and the
|
|
||||||
# broker is verified against a fingerprint pinned in a token rather than against a public root
|
|
||||||
# (novox/hq ADR 0004). Adding them "just in case" would put a trust store in the one image whose
|
|
||||||
# whole argument is that it contains nothing to reason about.
|
|
||||||
|
|
||||||
FROM ${GO_BASE} AS build
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# **No dependency is committed** (issue 1305): go.sum names each one's version and checksum, and the build
|
|
||||||
# resolves them with -mod=mod. Dependencies first, so a change to the source does not re-copy them.
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
|
|
||||||
# `COPY . .` brings `.git` along, so Go is told not to stamp the commit: the version is said by -X, and a
|
|
||||||
# rebuild of an unchanged program is the same bytes (novox/hq ADR 0267 rule 6, issue 363).
|
|
||||||
COPY . .
|
|
||||||
ARG VERSION=development
|
|
||||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=mod go build -trimpath -buildvcs=false \
|
|
||||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
|
||||||
-o /mesh-controller ./cmd/mesh-controller
|
|
||||||
|
|
||||||
FROM scratch
|
|
||||||
COPY --from=build /mesh-controller /mesh-controller
|
|
||||||
|
|
||||||
# Numeric because there is no /etc/passwd to look a name up in. Nothing here needs to be root:
|
|
||||||
# it opens outbound connections and writes nothing to its own filesystem.
|
|
||||||
USER 65534:65534
|
|
||||||
|
|
||||||
ENTRYPOINT ["/mesh-controller"]
|
|
||||||
@@ -1,112 +0,0 @@
|
|||||||
# novox/hq ADR 0006 — the control plane, in Go.
|
|
||||||
#
|
|
||||||
# The image the bundle pins holds the program and nothing else, so the build is static and the
|
|
||||||
# container is built FROM scratch. That is not a size optimisation: this image is fetched by
|
|
||||||
# digest and run on a machine where no mesh exists to check anything, and everything in it is
|
|
||||||
# something a person would have to audit.
|
|
||||||
|
|
||||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo development)
|
|
||||||
LDFLAGS := -s -w -X main.version=$(VERSION)
|
|
||||||
|
|
||||||
# Where `make check` raises PostgreSQL. A high port and a throwaway container: nothing here
|
|
||||||
# touches a database anybody else is using. Override PG_PORT if this one is taken -- the first
|
|
||||||
# port chosen was already serving something that had been up for six days.
|
|
||||||
PG_PORT ?= 55532
|
|
||||||
PG_CONTAINER ?= mesh-controller-check
|
|
||||||
PG_IMAGE ?= postgres:17-alpine
|
|
||||||
export MESH_TEST_POSTGRES ?= postgres://postgres:check@127.0.0.1:$(PG_PORT)/postgres?sslmode=disable
|
|
||||||
|
|
||||||
.PHONY: build image check test vet fmt postgres postgres-stop clean
|
|
||||||
|
|
||||||
build:
|
|
||||||
CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o build/mesh-controller ./cmd/mesh-controller
|
|
||||||
|
|
||||||
# Tagged 'development' as well as by version, because the lab places images by name and a
|
|
||||||
# scenario naming a version would have to be edited on every build. The version tag is what a
|
|
||||||
# real bundle pins.
|
|
||||||
IMAGE ?= mesh-controller:$(VERSION)
|
|
||||||
DEV_TAG ?= mesh-controller:development
|
|
||||||
|
|
||||||
# The Go base the image is built on.
|
|
||||||
#
|
|
||||||
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
|
|
||||||
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
|
|
||||||
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
|
|
||||||
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
|
|
||||||
# what it cost).
|
|
||||||
#
|
|
||||||
# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds
|
|
||||||
# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab —
|
|
||||||
# still needs a Go base. The digest is the one the manifest declared until then.
|
|
||||||
GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
|
||||||
|
|
||||||
image:
|
|
||||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
|
||||||
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
|
||||||
@echo
|
|
||||||
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
|
||||||
|
|
||||||
# The provisioner ships as an image too, because it is the thing that makes a sealed credential
|
|
||||||
# true on a machine -- and the mesh cannot, having discarded the plaintext.
|
|
||||||
PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
|
|
||||||
PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development
|
|
||||||
|
|
||||||
provisioner-image:
|
|
||||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/postgres-provisioner/Dockerfile \
|
|
||||||
-t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) .
|
|
||||||
@echo
|
|
||||||
@docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
|
||||||
|
|
||||||
# The object store's provisioner, for the same reason: a bucket and a policy are not files, and
|
|
||||||
# the mesh cannot make them -- it discarded the credential it would have to use.
|
|
||||||
OBJECTSTORE_IMAGE ?= mesh-provision-objectstore:$(VERSION)
|
|
||||||
OBJECTSTORE_DEV_TAG ?= mesh-provision-objectstore:development
|
|
||||||
|
|
||||||
objectstore-image:
|
|
||||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/objectstore-provisioner/Dockerfile \
|
|
||||||
-t $(OBJECTSTORE_IMAGE) -t $(OBJECTSTORE_DEV_TAG) .
|
|
||||||
@echo
|
|
||||||
@docker image inspect $(OBJECTSTORE_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
|
||||||
|
|
||||||
# The cache's provisioner, for the same reason as the database's: an ACL user is not a file,
|
|
||||||
# and the mesh cannot make one -- it discarded the credential it would have to use.
|
|
||||||
REDIS_PROVISIONER_IMAGE ?= mesh-provision-redis:$(VERSION)
|
|
||||||
REDIS_PROVISIONER_DEV_TAG ?= mesh-provision-redis:development
|
|
||||||
|
|
||||||
redis-provisioner-image:
|
|
||||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/redis-provisioner/Dockerfile \
|
|
||||||
-t $(REDIS_PROVISIONER_IMAGE) -t $(REDIS_PROVISIONER_DEV_TAG) .
|
|
||||||
@echo
|
|
||||||
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
|
||||||
|
|
||||||
# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
|
|
||||||
# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
|
|
||||||
# store's tests would skip rather than fail anyway.
|
|
||||||
test:
|
|
||||||
go test -timeout 15m ./...
|
|
||||||
|
|
||||||
vet:
|
|
||||||
go vet ./...
|
|
||||||
|
|
||||||
# A quick step for a developer: the checker runs the build module's own formatting and vet (ADR 0501 §5, D7).
|
|
||||||
fmt:
|
|
||||||
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
|
||||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
|
||||||
|
|
||||||
postgres:
|
|
||||||
@docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true
|
|
||||||
@docker run -d --name $(PG_CONTAINER) -e POSTGRES_PASSWORD=check \
|
|
||||||
-p 127.0.0.1:$(PG_PORT):5432 $(PG_IMAGE) >/dev/null
|
|
||||||
@printf 'waiting for postgres'
|
|
||||||
@for i in $$(seq 1 60) ; do \
|
|
||||||
if docker exec $(PG_CONTAINER) pg_isready -U postgres >/dev/null 2>&1 ; then \
|
|
||||||
echo ' — ready' ; exit 0 ; fi ; \
|
|
||||||
printf '.' ; sleep 1 ; \
|
|
||||||
done ; \
|
|
||||||
echo ' — never came up' ; docker logs $(PG_CONTAINER) | tail -20 ; exit 1
|
|
||||||
|
|
||||||
postgres-stop:
|
|
||||||
@docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true
|
|
||||||
|
|
||||||
clean:
|
|
||||||
rm -rf build/
|
|
||||||
@@ -1,218 +1,7 @@
|
|||||||
# mesh-controller
|
# mesh-controller
|
||||||
|
|
||||||
**Tier 2 of Novox Mesh — the control plane.** Everything that needs to know about more than one
|
The controller ended under issue 1126. Its jobs moved to the parts that ADR 0590 and ADR 0593 name: the indexer, the
|
||||||
node.
|
assigner, the provisioner, the grants part and the node-engine. Its last module, route-proxy, moved to mesh-networking.
|
||||||
|
|
||||||
That is the whole test, and it draws the line the host cannot: the host applies and does not
|
This repository holds no code and is archived once the delivery of the group that emptied it (release sdk1) is done.
|
||||||
decide, *because deciding needs knowledge one machine does not have*. Which nodes should run the
|
Its history stays here for reading.
|
||||||
store, which peers belong in an overlay, whether a node has been unreachable for a week — nobody
|
|
||||||
on a single machine can answer any of them.
|
|
||||||
|
|
||||||
The reasoning lives in [novox/hq](https://git.novox.be/novox/hq); this repository carries no
|
|
||||||
argument that is not settled there.
|
|
||||||
|
|
||||||
## What it is not
|
|
||||||
|
|
||||||
- **Not the thing that changes machines.** It decides; the host applies. It never reaches into a
|
|
||||||
node except through the host, over the link, in a bounded vocabulary.
|
|
||||||
- **Not a database.** There is no mesh database. Each context owns its own store and nothing
|
|
||||||
outside a context touches it — including nodes, which hold no credential to any of them.
|
|
||||||
- **Not privileged.** It has no more access to a machine than a declaration can express.
|
|
||||||
|
|
||||||
## What exists today
|
|
||||||
|
|
||||||
**One context of seven, and one of the things it will do.**
|
|
||||||
|
|
||||||
| | |
|
|
||||||
|---|---|
|
|
||||||
| `inventory` | node records and enrolment tokens — **built, as far as identity** |
|
|
||||||
| `identity` | the control plane's signing key, and the keys nodes are known by — **built** |
|
|
||||||
| `config`, `connectivity`, `provisioning`, `delivery`, `observability` | not built |
|
|
||||||
| the interface every surface speaks to | not built; its shape is not decided |
|
|
||||||
|
|
||||||
```
|
|
||||||
mesh-controller migrate bring each context's schema up to date
|
|
||||||
mesh-controller node add <name> create a node record
|
|
||||||
mesh-controller node list the nodes this mesh knows about
|
|
||||||
mesh-controller token issue --node <name> a one-time right to join, for an existing record
|
|
||||||
mesh-controller token issue --new <name> create the record and issue for it
|
|
||||||
mesh-controller identity show this control plane's signing key
|
|
||||||
mesh-controller broker show where the broker is, and what to expect there
|
|
||||||
mesh-controller version what this binary is
|
|
||||||
```
|
|
||||||
|
|
||||||
`migrate` is **step 3 of the foundation bootstrap** — the step the first node cannot get past, run
|
|
||||||
against a database raised moments earlier from the bundle the host carries.
|
|
||||||
|
|
||||||
### Tokens, and what they are missing
|
|
||||||
|
|
||||||
A token is **a one-time right to join, issued for a node record** — which is where re-enrolment is
|
|
||||||
decided, since what an identity binds to is settled when the token is made rather than when it is
|
|
||||||
presented.
|
|
||||||
|
|
||||||
What is built: the secret is 256 bits from the system's random source, shown once, and **stored
|
|
||||||
only as a hash**, so a copy of this database is not a set of working credentials. It is usable
|
|
||||||
exactly once and only before it expires, and both are read from the row rather than from a status
|
|
||||||
something would have had to write. Issuing again for the same node invalidates the outstanding
|
|
||||||
one — two live tokens are two machines able to join as the same node.
|
|
||||||
|
|
||||||
Redemption is a single statement that both finds a live token and spends it, so eight concurrent
|
|
||||||
attempts on one secret produce exactly one winner. There is a test that runs them.
|
|
||||||
|
|
||||||
**A token now carries three of its four parts**, and is one line of base64 a person can copy. The
|
|
||||||
signing key is real: an Ed25519 key this control plane generates once and keeps, whose public half
|
|
||||||
travels in every token. A node believes a declaration because it carries a signature that key made
|
|
||||||
— and pinning only the broker would not do, because it would make the control plane's authority
|
|
||||||
transitive, so a compromised broker could forge declarations, and since the host applies whatever
|
|
||||||
the link delivers that is the whole machine.
|
|
||||||
|
|
||||||
**All four parts are built.** Given `MESH_BROKER_ADDRESS` and `MESH_BROKER_CERTIFICATE`, a token
|
|
||||||
carries everything ADR 0004 requires. Without them it carries two, and `token issue` prints it
|
|
||||||
**and names what is missing** rather than producing something that looks complete and cannot be
|
|
||||||
used.
|
|
||||||
|
|
||||||
**The fingerprint is derived from the certificate, never configured.** A configured one can drift
|
|
||||||
from the certificate it describes, and a drifted pin is worse than none: every node issued a token
|
|
||||||
during the drift refuses to connect, and the failure looks like an attack. It is computed over the
|
|
||||||
DER bytes — what a client actually sees on the wire — so the same certificate re-wrapped with
|
|
||||||
different line endings still produces the same pin.
|
|
||||||
|
|
||||||
### Two contexts, and the rule between them is real
|
|
||||||
|
|
||||||
`identity` is the second context and it exists partly to test a claim this repository had made and
|
|
||||||
never checked: that a context reaches only its own store. It holds `MESH_STORE_IDENTITY`;
|
|
||||||
`inventory` holds `MESH_STORE_INVENTORY`; there is no setting that reaches both and no way to ask
|
|
||||||
for one. Run `migrate` with only one and it stops, naming the grant it does not have.
|
|
||||||
|
|
||||||
A token needs a node record from one and a signing key from the other. Neither reads the other's
|
|
||||||
store — the process holding both grants asks each for its part.
|
|
||||||
|
|
||||||
### How a node is known
|
|
||||||
|
|
||||||
**The node generates a keypair; the mesh records the public half.** The same principle as SSH, and
|
|
||||||
the same rule `08-connectivity` already applies to the overlay keys — which is where this was
|
|
||||||
settled all along, though it took being asked directly to notice.
|
|
||||||
|
|
||||||
Only the public half is ever stored, and that is the property worth having: **a copy of this
|
|
||||||
database grants nothing.** It is a list of who to believe, not a set of credentials, which is what
|
|
||||||
makes *compromise of a node is compromise of that node* literally true.
|
|
||||||
|
|
||||||
Exactly one key is live per node. Re-enrolment revokes the one it replaced, in the same
|
|
||||||
transaction — two live identities for one node record is the stolen-laptop case, with the replaced
|
|
||||||
machine still believed. The database enforces it as well as the code, and there is a test running
|
|
||||||
six concurrent enrolments that fails when the constraint is removed.
|
|
||||||
|
|
||||||
**Not the machine's SSH host key**, though that was the obvious economy. Host keys are regenerated
|
|
||||||
by reinstalls and image clones, which would silently un-enrol a node; their lifecycle belongs to
|
|
||||||
sshd rather than the mesh; and a partial host has no SSH daemon at all.
|
|
||||||
|
|
||||||
## Reaching a store
|
|
||||||
|
|
||||||
**A context is granted only what it exclusively owns.** No shared writes, no read-only role on
|
|
||||||
another context's store, and no connection string that reaches more than one.
|
|
||||||
|
|
||||||
That is a rule about credentials, so it is built as one. There is no mesh-wide connection setting
|
|
||||||
and no way to ask for one:
|
|
||||||
|
|
||||||
```
|
|
||||||
MESH_STORE_INVENTORY=postgres://…/inventory
|
|
||||||
```
|
|
||||||
|
|
||||||
A process granted `inventory` holds that variable and no other. Reaching another context's store
|
|
||||||
is not a matter of restraint — it has no address for it and no credential to present. And it is
|
|
||||||
how the rule is *checked*: what a context can reach is visible in the declaration that runs it,
|
|
||||||
as the list of variables it was given.
|
|
||||||
|
|
||||||
Each context's database is named after the context. There is deliberately no database named for
|
|
||||||
the mesh as a whole.
|
|
||||||
|
|
||||||
## Migrations
|
|
||||||
|
|
||||||
Numbered, embedded in the binary, applied in order, each in a transaction with the row recording
|
|
||||||
it. The applying is four lines; the rest is refusals, and the refusals are the point:
|
|
||||||
|
|
||||||
| it stops when | because |
|
|
||||||
|---|---|
|
|
||||||
| a migration that ran has since been edited | the database holds the old version, the repository holds the new one, and nothing holds the difference |
|
|
||||||
| a migration is numbered below one that already ran | usually two branches taking the same next number — applying it now runs the schema in an order nobody tested |
|
|
||||||
| two migrations share a number | order is the entire guarantee, and two files with one number have none |
|
|
||||||
| a file in the directory is not a valid migration name | a misnamed migration would otherwise never run and nothing would say so |
|
|
||||||
| a migration is empty | it records that something happened and changes nothing, which cannot be told from a mistake |
|
|
||||||
|
|
||||||
All of them stop rather than warn. At the moment any of them is true, nobody knows what the
|
|
||||||
database contains, and there is no correct guess about a schema.
|
|
||||||
|
|
||||||
Running it again does nothing. Two copies running at once take an advisory lock, so a restart
|
|
||||||
during a slow migration does not become two runners racing.
|
|
||||||
|
|
||||||
## Building
|
|
||||||
|
|
||||||
```
|
|
||||||
make build the binary
|
|
||||||
make image the container image
|
|
||||||
make check gofmt, vet, and every test against a real PostgreSQL
|
|
||||||
```
|
|
||||||
|
|
||||||
`make check` raises a throwaway PostgreSQL in a container and takes it down afterwards, including
|
|
||||||
when the tests fail. Without one the tests that need a database **skip and say so** rather than
|
|
||||||
passing quietly — `make test` is the honest subset, not the gate.
|
|
||||||
|
|
||||||
Tests run against a real database rather than a fake because what is being tested *is* the
|
|
||||||
database's behaviour: that DDL is transactional, that an advisory lock serialises, that a
|
|
||||||
checksum mismatch is caught against a record PostgreSQL actually kept. A fake would assert that
|
|
||||||
the fake behaves as expected.
|
|
||||||
|
|
||||||
Every test here has been confirmed to fail when the behaviour it defends is removed. Two did not,
|
|
||||||
when first written, and both are now commented with what they were missing.
|
|
||||||
|
|
||||||
## Checks that cross into the host's repository
|
|
||||||
|
|
||||||
Two things are agreed between this repository and `novox/mesh-host`, and each is a separate struct
|
|
||||||
on each side. A field renamed on one of them fails **silently** — the crossing succeeds and
|
|
||||||
something is simply absent — so both are checked by handing one side's real output to the other's
|
|
||||||
real parser. Neither runs by default; each skips with a reason, because a repository that fails
|
|
||||||
without its neighbour checked out is a repository nobody can build.
|
|
||||||
|
|
||||||
**What this mesh sends, read by the host that receives it:**
|
|
||||||
|
|
||||||
```
|
|
||||||
mesh-controller: ./build/mesh-controller plan <node> --json > /tmp/d.json
|
|
||||||
mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v
|
|
||||||
```
|
|
||||||
|
|
||||||
**What a node says when it joins, read by this mesh:**
|
|
||||||
|
|
||||||
```
|
|
||||||
mesh-host: MESH_ENROL_OUT=/tmp/enrol.json go test ./internal/link/
|
|
||||||
mesh-controller: MESH_ENROL=/tmp/enrol.json make check
|
|
||||||
```
|
|
||||||
|
|
||||||
The second does more than compare shapes: it seals something to the key that arrived and opens it
|
|
||||||
with the private half the host kept. A key that is present, correctly named and simply *wrong*
|
|
||||||
passes every check that only looks at the message.
|
|
||||||
|
|
||||||
## The image
|
|
||||||
|
|
||||||
**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go
|
|
||||||
bundle, `controller`, which the host on the controller's machine unpacks and runs as the process
|
|
||||||
`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what
|
|
||||||
still runs a container of the controller: genesis, which raises the first controller from it and
|
|
||||||
installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the
|
|
||||||
mesh's own build any more — `make image` builds it.
|
|
||||||
|
|
||||||
`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
|
|
||||||
manager, no libc, no CA certificates.
|
|
||||||
|
|
||||||
Not a size optimisation. The bundle a host carries pins this image by digest, and it is fetched
|
|
||||||
and run on a machine where no mesh exists to check anything and a person is expected to have read
|
|
||||||
the bundle and believed it. Everything in the image is something that person would have to audit.
|
|
||||||
|
|
||||||
## Where the reasoning lives
|
|
||||||
|
|
||||||
| | |
|
|
||||||
|---|---|
|
|
||||||
| what the control plane is | novox/hq ADR 0006 |
|
|
||||||
| what it takes to run one, and why Go | novox/hq ADR 0006 |
|
|
||||||
| a context owns its store, exclusively | novox/hq ADR 0008 |
|
|
||||||
| schema changes are numbered migrations | novox/hq ADR 0013 |
|
|
||||||
| a test defends a decision | novox/hq ADR 0017 |
|
|
||||||
|
|||||||
@@ -1,147 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A provision whose credential only a person outside the mesh can give (novox/hq ADR 0337, issue 164, issue 165):
|
|
||||||
// given once for its provider, at the controller's terminal, and sealed in one act to every consumer that binds the
|
|
||||||
// provision from that provider now. The value is never read back, printed or kept.
|
|
||||||
|
|
||||||
// outsideOffer is the provider module's own name when `secret accept <node> <module> <name>` names a provision that
|
|
||||||
// module offers with a credential issued outside the mesh, and "" when it names anything else (an own secret, or a
|
|
||||||
// pair with --provider).
|
|
||||||
func outsideOffer(ctx context.Context, module, name, provider string) (string, error) {
|
|
||||||
if provider != "" {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
shelf, err := open.inventory.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if m, known := shelf[module]; known && m.IssuedOutsideOf(name) {
|
|
||||||
return module, nil
|
|
||||||
}
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// offeredOutsideBy is a module that offers the provision with a credential issued outside the mesh, or "".
|
|
||||||
func offeredOutsideBy(ctx context.Context, provision string) (string, error) {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
shelf, err := open.inventory.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
module, _ := catalogue.OffersIssuedOutside(shelf, provision)
|
|
||||||
return module, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// consumersFrom is every consumer that binds a provision from one provider now, found by resolving every machine as
|
|
||||||
// its next send would, and the machines that could not be resolved, with why.
|
|
||||||
func consumersFrom(ctx context.Context, open *stores, provider, providerModule, provision string) (
|
|
||||||
[]inventory.PairEnd, []string, error) {
|
|
||||||
// One reading of the mesh for every machine this looks at.
|
|
||||||
ctx = withInputCache(ctx)
|
|
||||||
nodes, err := open.inventory.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
var out []inventory.PairEnd
|
|
||||||
var unresolved []string
|
|
||||||
for _, n := range nodes {
|
|
||||||
plan, _, err := resolveNode(ctx, open, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
unresolved = append(unresolved, fmt.Sprintf("%s: %s", n.Name, oneLine(err.Error())))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, bindsFrom(plan, n.Name, provider, providerModule, provision)...)
|
|
||||||
}
|
|
||||||
return out, unresolved, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// bindsFrom is every consumer on one machine whose resolution answers the provision from that provider. Pure.
|
|
||||||
func bindsFrom(plan catalogue.Resolution, machine, provider, providerModule, provision string) []inventory.PairEnd {
|
|
||||||
var out []inventory.PairEnd
|
|
||||||
for _, need := range plan.Needs {
|
|
||||||
if need.ByRecord || need.Name != provision || need.From != provider {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if need.Module != "" && need.Module != providerModule {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, inventory.PairEnd{Consumer: machine, ConsumerModule: need.For, Local: need.Local})
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool {
|
|
||||||
return out[i].ConsumerModule+"\x00"+out[i].Local < out[j].ConsumerModule+"\x00"+out[j].Local
|
|
||||||
})
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// acceptForProvision gives a provision's credential issued outside the mesh, once for its provider (ADR 0337 §3).
|
|
||||||
func acceptForProvision(ctx context.Context, provider, providerModule, provision, value string) error {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
consumers, unresolved, err := consumersFrom(ctx, open, provider, providerModule, provision)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
accepted, err := open.inventory.AcceptSecretForProvision(ctx, provision, provider, providerModule, value, consumers)
|
|
||||||
value = ""
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Print(acceptedSaid(provider, providerModule, provision, accepted, unresolved))
|
|
||||||
if len(accepted.Sealed) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := sendTo(ctx, open, accepted.Machines); err != nil {
|
|
||||||
return fmt.Errorf("%s is sealed and could not be sent: %w; `push %s` sends it", provision, err,
|
|
||||||
strings.Join(accepted.Machines, "`, `push "))
|
|
||||||
}
|
|
||||||
fmt.Printf(" sent %s; each consumer starts again on it\n", strings.Join(accepted.Machines, ", "))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// acceptedSaid is what one accept for a provider did, in words that never carry the value. Pure.
|
|
||||||
func acceptedSaid(provider, providerModule, provision string, a inventory.ProvisionAccepted, unresolved []string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
if len(a.Sealed) == 0 {
|
|
||||||
fmt.Fprintf(&b, "%s from %s on %s: no consumer binds it now, so nothing was sealed; a consumer that binds "+
|
|
||||||
"later waits for it, and this line given again seals it to every consumer then\n", provision,
|
|
||||||
providerModule, provider)
|
|
||||||
} else {
|
|
||||||
var names []string
|
|
||||||
for _, c := range a.Sealed {
|
|
||||||
names = append(names, fmt.Sprintf("%s on %s%s", c.ConsumerModule, c.Consumer, asLocal(c.Local)))
|
|
||||||
}
|
|
||||||
fmt.Fprintf(&b, "%s from %s on %s is given, sealed to %s and to %s\n", provision, providerModule, provider,
|
|
||||||
strings.Join(names, ", "), provider)
|
|
||||||
fmt.Fprintf(&b, " the mesh cannot read it back, never makes one of its own for it, and will not rotate it\n")
|
|
||||||
}
|
|
||||||
for _, c := range a.Unsealable {
|
|
||||||
fmt.Fprintf(&b, " not sealed to %s on %s: no sealing key on that machine; give it again once it has one\n",
|
|
||||||
c.ConsumerModule, c.Consumer)
|
|
||||||
}
|
|
||||||
for _, u := range unresolved {
|
|
||||||
fmt.Fprintf(&b, " not known whether a consumer on %s binds it, which did not resolve; give it again once it does\n", u)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
@@ -1,378 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/lease"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
|
|
||||||
//
|
|
||||||
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
|
|
||||||
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
|
|
||||||
// not happen. Three ways a process stands to the lease:
|
|
||||||
//
|
|
||||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
|
||||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
|
||||||
// Another holder seen, or the key's age passed with no renewal made, is the lease lost: the gate
|
|
||||||
// closes at once and the process exits, so its service manager restarts it as a candidate (serve,
|
|
||||||
// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the
|
|
||||||
// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal
|
|
||||||
// made opens the gate again.
|
|
||||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
|
||||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
|
||||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
|
||||||
// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
|
|
||||||
// between makes it stale and refused like any other. **When nobody holds the lease, the command
|
|
||||||
// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
|
|
||||||
// it, as it would for another controller.
|
|
||||||
// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
|
|
||||||
// refused nothing: there is nothing to order against, and nothing it does reaches a machine.
|
|
||||||
//
|
|
||||||
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
|
|
||||||
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
|
|
||||||
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
|
|
||||||
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
|
|
||||||
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
|
|
||||||
// instead would be a controller that can never send the user list that lets it act.
|
|
||||||
|
|
||||||
// actor is this process's standing to the lease.
|
|
||||||
type actor struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
// held is the lease this process holds: the serving controller's, or a command's own.
|
|
||||||
held *lease.Lease
|
|
||||||
// unleased is why a serving controller acts without the lease; empty while it holds it or is not
|
|
||||||
// serving.
|
|
||||||
unleased string
|
|
||||||
// serving is a serving controller, which never borrows another's epoch.
|
|
||||||
serving bool
|
|
||||||
// kv is the lease bucket, for a command to read the holder's epoch from.
|
|
||||||
kv jetstream.KeyValue
|
|
||||||
close func()
|
|
||||||
// noBus is a process with no bus configured.
|
|
||||||
noBus bool
|
|
||||||
// reset is when the lease bucket was found raised again from nothing and its revisions moved past
|
|
||||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
|
||||||
reset time.Time
|
|
||||||
resetSaid string
|
|
||||||
// renewEvery and poll, when set, are how often the serving controller renews the lease it holds and
|
|
||||||
// looks again at one another holds, in place of the lease's own: a test's, so two controllers hand the
|
|
||||||
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
|
|
||||||
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
|
|
||||||
renewEvery, poll time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// theLease is this process's standing to the lease.
|
|
||||||
var theLease = &actor{}
|
|
||||||
|
|
||||||
// instance names this process among controller instances: its machine, its process and when it
|
|
||||||
// started. The lease's holder and every call this process keeps carry it.
|
|
||||||
var instance = func() string {
|
|
||||||
host, _ := os.Hostname()
|
|
||||||
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
|
||||||
}()
|
|
||||||
|
|
||||||
// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
|
|
||||||
func (a *actor) epoch(ctx context.Context) (uint64, error) {
|
|
||||||
a.mu.Lock()
|
|
||||||
held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
|
|
||||||
a.mu.Unlock()
|
|
||||||
switch {
|
|
||||||
case held != nil:
|
|
||||||
return held.Epoch()
|
|
||||||
case serving && unleased != "":
|
|
||||||
return 0, nil
|
|
||||||
case serving:
|
|
||||||
return 0, lease.ErrNotHeld
|
|
||||||
case noBus:
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
return a.forACommand(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
|
|
||||||
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
|
|
||||||
a.mu.Lock()
|
|
||||||
defer a.mu.Unlock()
|
|
||||||
if a.held != nil {
|
|
||||||
return a.held.Epoch()
|
|
||||||
}
|
|
||||||
if a.kv == nil {
|
|
||||||
address, err := broker.BusAddress()
|
|
||||||
if err != nil {
|
|
||||||
// No bus: this process reaches no machine, and has nothing to order against.
|
|
||||||
a.noBus = true
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
js, err := broker.Dial(address)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
|
|
||||||
"read and nothing is done: %w", err)
|
|
||||||
}
|
|
||||||
api, err := jetstream.New(js.Conn())
|
|
||||||
if err != nil {
|
|
||||||
js.Close()
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := broker.EnsureLeaseBucket(reading, api); err != nil {
|
|
||||||
js.Close()
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
kv, err := api.KeyValue(reading, broker.LeaseBucket)
|
|
||||||
if err != nil {
|
|
||||||
js.Close()
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
a.kv, a.close = kv, js.Close
|
|
||||||
if _, found, err := lease.Current(reading, kv); err == nil && !found {
|
|
||||||
// Nobody: this command takes it for as long as it runs.
|
|
||||||
l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
|
||||||
Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
epoch, err := l.TryTake(reading)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
|
|
||||||
}
|
|
||||||
keeping, stop := context.WithCancel(context.Background())
|
|
||||||
go l.Keep(keeping)
|
|
||||||
a.held = l
|
|
||||||
closeBus := a.close
|
|
||||||
a.close = func() {
|
|
||||||
stop()
|
|
||||||
l.Release(context.Background())
|
|
||||||
closeBus()
|
|
||||||
}
|
|
||||||
return epoch, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
holder, found, err := lease.Current(reading, a.kv)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
|
|
||||||
"more is done under an epoch nobody holds — run the command again")
|
|
||||||
}
|
|
||||||
return holder.Epoch, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// release gives back what this process holds, at its end.
|
|
||||||
func (a *actor) release() {
|
|
||||||
a.mu.Lock()
|
|
||||||
closing := a.close
|
|
||||||
a.close = nil
|
|
||||||
a.mu.Unlock()
|
|
||||||
if closing != nil {
|
|
||||||
closing()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// holderOf is this process as the lease's holder.
|
|
||||||
func holderOf(instance string) lease.Holder {
|
|
||||||
host, _ := os.Hostname()
|
|
||||||
build := runningBuild()
|
|
||||||
if build == "" {
|
|
||||||
build = version
|
|
||||||
}
|
|
||||||
return lease.Holder{Instance: instance, Host: host, Build: build}
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunningBuildVar is where the declaration tells this process which build it is (module.json, the
|
|
||||||
// controller process's env): the version its bundle is delivered as, `${version}` composed by the
|
|
||||||
// catalogue from the bundle's digest. Empty for a process placed by hand.
|
|
||||||
const RunningBuildVar = "MESH_CONTROLLER_VERSION"
|
|
||||||
|
|
||||||
// runningBuild is the version of the build this process is, or empty when the declaration did not say.
|
|
||||||
func runningBuild() string {
|
|
||||||
return strings.TrimSpace(os.Getenv(RunningBuildVar))
|
|
||||||
}
|
|
||||||
|
|
||||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
|
||||||
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
|
|
||||||
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
|
|
||||||
a.mu.Lock()
|
|
||||||
a.serving = true
|
|
||||||
a.mu.Unlock()
|
|
||||||
// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
|
|
||||||
js, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
|
||||||
"lease: %w", broker.BareAddress(address), err)
|
|
||||||
}
|
|
||||||
api, err := jetstream.New(js.Conn())
|
|
||||||
if err != nil {
|
|
||||||
js.Close()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
||||||
err = broker.EnsureLeaseBucket(asserting, api)
|
|
||||||
cancel()
|
|
||||||
if err != nil {
|
|
||||||
js.Close()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
|
||||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
|
||||||
RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
|
||||||
a.mu.Lock()
|
|
||||||
defer a.mu.Unlock()
|
|
||||||
a.reset = time.Now()
|
|
||||||
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
|
|
||||||
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
|
|
||||||
"no machine refuses the next epoch", was, floor, floor)
|
|
||||||
}})
|
|
||||||
if err != nil {
|
|
||||||
js.Close()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
gone := make(chan struct{})
|
|
||||||
epoch, err := l.Take(ctx)
|
|
||||||
switch {
|
|
||||||
case ctx.Err() != nil:
|
|
||||||
js.Close()
|
|
||||||
return nil, ctx.Err()
|
|
||||||
case err != nil && !errors.Is(err, lease.ErrUnwritable):
|
|
||||||
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
|
|
||||||
js.Close()
|
|
||||||
return nil, err
|
|
||||||
case err != nil:
|
|
||||||
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
|
|
||||||
a.mu.Lock()
|
|
||||||
a.unleased = err.Error()
|
|
||||||
a.mu.Unlock()
|
|
||||||
say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
|
|
||||||
"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
|
|
||||||
go a.takeWhenGranted(ctx, l, inv, gone)
|
|
||||||
default:
|
|
||||||
a.took(ctx, l, inv, epoch, gone)
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
a.close = func() {
|
|
||||||
if held, err := l.Epoch(); err == nil {
|
|
||||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
|
|
||||||
say("how epoch %d ended could not be recorded: %v", held, err)
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
}
|
|
||||||
l.Release(context.Background())
|
|
||||||
js.Close()
|
|
||||||
}
|
|
||||||
a.mu.Unlock()
|
|
||||||
return gone, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
|
|
||||||
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
|
|
||||||
a.mu.Lock()
|
|
||||||
a.held, a.unleased = l, ""
|
|
||||||
a.mu.Unlock()
|
|
||||||
h := holderOf(instance)
|
|
||||||
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
||||||
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
|
|
||||||
Build: h.Build, Taken: time.Now()})
|
|
||||||
cancel()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
|
|
||||||
}
|
|
||||||
for _, e := range expired {
|
|
||||||
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
|
|
||||||
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
|
|
||||||
}
|
|
||||||
go l.Keep(ctx)
|
|
||||||
go func() {
|
|
||||||
<-l.Lost()
|
|
||||||
if ctx.Err() == nil {
|
|
||||||
why := l.LostWhy()
|
|
||||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
|
|
||||||
cancel()
|
|
||||||
fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
|
|
||||||
"be started again as a candidate\n", epoch, why)
|
|
||||||
}
|
|
||||||
close(gone)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
|
|
||||||
// controller if another took it meanwhile: two serving at once is what the lease is for.
|
|
||||||
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
|
|
||||||
tick := time.NewTicker(lease.RenewEvery)
|
|
||||||
defer tick.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-tick.C:
|
|
||||||
}
|
|
||||||
epoch, err := l.TryTake(ctx)
|
|
||||||
if errors.Is(err, lease.ErrTaken) {
|
|
||||||
fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
|
|
||||||
"stops and exits\n", err)
|
|
||||||
close(gone)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// Still not written, or not readable this time: unleased, said by S12, tried again.
|
|
||||||
a.mu.Lock()
|
|
||||||
a.unleased = err.Error()
|
|
||||||
a.mu.Unlock()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
a.took(ctx, l, inv, epoch, gone)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// standing is what `status` and the self-check say of this process and the lease.
|
|
||||||
type standing struct {
|
|
||||||
Epoch uint64
|
|
||||||
Held bool
|
|
||||||
Renewed time.Time
|
|
||||||
Unleased string
|
|
||||||
// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
|
|
||||||
Reset time.Time
|
|
||||||
ResetSaid string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *actor) standing() standing {
|
|
||||||
a.mu.Lock()
|
|
||||||
held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
|
|
||||||
a.mu.Unlock()
|
|
||||||
st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
|
|
||||||
if held == nil {
|
|
||||||
return st
|
|
||||||
}
|
|
||||||
epoch, err := held.Epoch()
|
|
||||||
st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
|
|
||||||
return st
|
|
||||||
}
|
|
||||||
|
|
||||||
// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
|
|
||||||
func init() {
|
|
||||||
link.ActingGate = func(ctx context.Context) error {
|
|
||||||
_, err := theLease.epoch(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("this controller may not send: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,460 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"io"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The things the mesh can be asked to do, separated from how it was asked.
|
|
||||||
//
|
|
||||||
// **A surface is an adapter with no decisions in it** (novox/hq ADR 0035). The command line and
|
|
||||||
// the command API call the same functions here, so an assignment refused at one is refused at the
|
|
||||||
// other for the same reason and in the same words. The moment a surface can accept something
|
|
||||||
// another would reject, the mesh has two answers to one question and people learn which to trust.
|
|
||||||
//
|
|
||||||
// Each returns what happened as text a person can read and a caller can pass on. Neither surface
|
|
||||||
// composes its own explanation, because two explanations of one refusal drift.
|
|
||||||
|
|
||||||
// assign puts a module on a node, and says at once what in the mesh no longer works out.
|
|
||||||
//
|
|
||||||
// The assignment is kept even when the node does not resolve: it is what a person meant, and
|
|
||||||
// assignment is not an ordering. A consumer assigned before its provider does not resolve for as
|
|
||||||
// long as it takes to assign the provider, and refusing the first half of a pair would make the
|
|
||||||
// order somebody types two commands in part of the mesh's rules.
|
|
||||||
//
|
|
||||||
// **What is checked is the mesh, not the one machine** — because that is what the assignment
|
|
||||||
// changes. novox/hq 04-ISSUES/017 names the shape: an action can succeed into a state its own
|
|
||||||
// verify rejects, and it happens when the action's test is not the test the verify uses. Here the
|
|
||||||
// action tested one node and the verify is resolution over all of them, so an assignment could be
|
|
||||||
// reported as fine while it took a provision away from every other machine — a module offering a
|
|
||||||
// mesh-scoped provision stops offering it the moment its own node stops resolving, and every
|
|
||||||
// consumer elsewhere is then told *nothing in this mesh provides it*, with a remedy that names a
|
|
||||||
// module already assigned. That was found on a four-node raise and read as a version bump breaking
|
|
||||||
// provider recognition; it was neither the version nor the provider.
|
|
||||||
//
|
|
||||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
|
||||||
// machines this just blocked is worth more than the milliseconds.
|
|
||||||
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
|
||||||
return assignWith(ctx, open, node, assignOptions{}, modules...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assignWith is assign asked with its options: build, for a module known and not built (novox/hq issue 325).
|
|
||||||
func assignWith(ctx context.Context, open *stores, node string, opts assignOptions, modules ...string) (string, error) {
|
|
||||||
if len(modules) == 0 {
|
|
||||||
return "", fmt.Errorf("assign %s names no module", node)
|
|
||||||
}
|
|
||||||
if err := refusedMovingTheController(modules); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
|
||||||
// what the reads before it would say about a machine left out is not the answer's.
|
|
||||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
|
||||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
|
||||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
|
||||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
// **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue
|
|
||||||
// 325): a build in flight keeps the assignment pending, a module known and not built is said with how to
|
|
||||||
// build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act
|
|
||||||
// are judged together (ADR 0207), so an act naming one not registered is not made in part.
|
|
||||||
if missing, err := notInCatalogue(ctx, open, modules); err != nil {
|
|
||||||
return "", err
|
|
||||||
} else if len(missing) > 0 {
|
|
||||||
if len(modules) == 1 {
|
|
||||||
return notRegistered(ctx, open, node, modules[0], opts)
|
|
||||||
}
|
|
||||||
var lines []string
|
|
||||||
for _, m := range missing {
|
|
||||||
where, err := whereIs(ctx, open.inventory, m, time.Now())
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
lines = append(lines, where.said)
|
|
||||||
}
|
|
||||||
return "", fmt.Errorf("%w: nothing was assigned, since modules assigned together are judged together "+
|
|
||||||
"(ADR 0207) and %s not registered:\n %s\n assign them together once each is registered, or each "+
|
|
||||||
"alone to keep it pending on its build", inventory.ErrNoSuchModule, strings.Join(missing, ", "),
|
|
||||||
strings.Join(lines, "\n "))
|
|
||||||
}
|
|
||||||
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
|
||||||
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
|
||||||
// resources are applied through a seat nothing on the node holds is refused, because that order
|
|
||||||
// — the service manager, the package manager and the runtime before anything that installs,
|
|
||||||
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
|
|
||||||
// holders that depend on each other go on in one command.
|
|
||||||
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
|
||||||
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
|
||||||
// assignment resolves against a record rather than against a coincidence.
|
|
||||||
settled, err := recordDerivedHolders(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
var lines []string
|
|
||||||
var added []string
|
|
||||||
for _, module := range modules {
|
|
||||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
|
||||||
if err != nil {
|
|
||||||
return strings.Join(lines, "\n"), err
|
|
||||||
}
|
|
||||||
if !fresh {
|
|
||||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
|
||||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
|
||||||
lines = append(lines, fmt.Sprintf(
|
|
||||||
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
added = append(added, module)
|
|
||||||
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
|
|
||||||
}
|
|
||||||
if len(added) == 0 {
|
|
||||||
return strings.Join(lines, "\n"), nil
|
|
||||||
}
|
|
||||||
answer := strings.Join(lines, "\n")
|
|
||||||
for _, line := range settled {
|
|
||||||
answer += "\n " + line
|
|
||||||
}
|
|
||||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
|
||||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
|
||||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
|
||||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
|
||||||
var issued []string
|
|
||||||
for _, module := range added {
|
|
||||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
|
||||||
issued = append(issued, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **The one resolution the rest of the answer is said from** (novox/hq issue 237), taken after
|
|
||||||
// the write: what this act changed about this node's unmet seat dependencies, and nothing else
|
|
||||||
// (novox/hq ADR 0207) — a dependency of a module just assigned, or one this assignment met; the
|
|
||||||
// rest of the node's list, and every other node's, is `status`'s — and whether the node resolves.
|
|
||||||
after := resolvedAfter(ctx, open, shelf, node)
|
|
||||||
for _, line := range unheldChangeTo(shelf, node, before, after.modules, after.unheld) {
|
|
||||||
answer += "\n " + line
|
|
||||||
}
|
|
||||||
for _, line := range issued {
|
|
||||||
answer += "\n " + line
|
|
||||||
}
|
|
||||||
plan, err := after.plan, after.err
|
|
||||||
if err != nil {
|
|
||||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
|
||||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
|
||||||
return answer + blockedElsewhere(ctx, open, node), err
|
|
||||||
}
|
|
||||||
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
|
||||||
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
|
||||||
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
|
||||||
isAdded := map[string]bool{}
|
|
||||||
for _, m := range added {
|
|
||||||
isAdded[m] = true
|
|
||||||
}
|
|
||||||
for _, u := range plan.Unhostable {
|
|
||||||
if !isAdded[u.Module] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, c := range u.Missing {
|
|
||||||
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
|
||||||
blockedElsewhere(ctx, open, node), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
|
||||||
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
|
|
||||||
// and are not judged again.
|
|
||||||
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
|
|
||||||
map[string]catalogue.Manifest, []string, error) {
|
|
||||||
shelf, err := open.inventory.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
assigned, err := open.inventory.Assigned(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
already := map[string]bool{}
|
|
||||||
for _, a := range assigned {
|
|
||||||
already[a] = true
|
|
||||||
}
|
|
||||||
var adding []string
|
|
||||||
for _, m := range modules {
|
|
||||||
if !already[m] {
|
|
||||||
adding = append(adding, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
|
|
||||||
// with everything else this act changed, so they are not said twice.
|
|
||||||
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
// Two modules declaring one path or unit, or one package present and absent, is refused before anything
|
|
||||||
// is recorded (novox/hq ADR 0210, ADR 0303, 04-ISSUES/235): kept, the node would not resolve until one came
|
|
||||||
// off again. Two modules declaring one package present is not a collision (ADR 0303, issue 628).
|
|
||||||
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
return shelf, assigned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// unassign takes modules off a node. What they leave behind is the host's business: a directory
|
|
||||||
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
|
||||||
//
|
|
||||||
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
|
||||||
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
|
||||||
// about the command's own output would ever have said so.
|
|
||||||
//
|
|
||||||
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
|
|
||||||
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
|
||||||
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
|
||||||
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
|
||||||
if err := refusedMovingTheController(modules); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if len(modules) == 0 {
|
|
||||||
return "", fmt.Errorf("unassign %s names no module", node)
|
|
||||||
}
|
|
||||||
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
|
||||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
|
||||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
shelf, err := open.inventory.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
assigned, err := open.inventory.Assigned(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
// Every one checked before any is taken off, so a refusal leaves the node as it was.
|
|
||||||
runs := map[string]bool{}
|
|
||||||
for _, a := range assigned {
|
|
||||||
runs[a] = true
|
|
||||||
}
|
|
||||||
// A module only pending on the node (novox/hq issue 325) is withdrawn, when it is the act's one module:
|
|
||||||
// that is the undo of an assignment kept while its build runs.
|
|
||||||
if len(modules) == 1 && !runs[modules[0]] {
|
|
||||||
said, withdrawn, err := withdrawPending(ctx, open.inventory, node, modules[0])
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if withdrawn {
|
|
||||||
return said, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, module := range modules {
|
|
||||||
if !runs[module] {
|
|
||||||
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
for _, module := range modules {
|
|
||||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
|
||||||
node, strings.Join(modules, ", "), node)
|
|
||||||
after := resolvedAfter(ctx, open, shelf, node)
|
|
||||||
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
|
||||||
answer += "\n " + line
|
|
||||||
}
|
|
||||||
// Whether the node itself still resolves, from the same resolution: before, only the rest of the
|
|
||||||
// mesh's reads said it, on standard error and once per other machine.
|
|
||||||
if after.err != nil {
|
|
||||||
if unresolvable(after.err) {
|
|
||||||
answer += fmt.Sprintf("\n but %s does not resolve now, so nothing will be sent to it:", node)
|
|
||||||
} else {
|
|
||||||
answer += fmt.Sprintf("\n but whether %s still resolves could not be read:", node)
|
|
||||||
}
|
|
||||||
for _, line := range strings.Split(strings.TrimRight(after.err.Error(), "\n"), "\n") {
|
|
||||||
answer += "\n " + strings.TrimSpace(line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
|
||||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
|
||||||
answer += "\n " + line
|
|
||||||
}
|
|
||||||
return answer + blockedElsewhere(ctx, open, node), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
|
||||||
// (novox/hq issue 237): the plan or why there is none, the modules it has on the node, and its unmet
|
|
||||||
// seat dependencies.
|
|
||||||
type afterAct struct {
|
|
||||||
plan catalogue.Resolution
|
|
||||||
err error
|
|
||||||
modules []string
|
|
||||||
unheld []catalogue.Unheld
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolvedAfter resolves the node as `plan` does, once, after the act's write. When the node resolves,
|
|
||||||
// its modules and unmet seat dependencies are the resolution's — the closure, so a holder pulled in by
|
|
||||||
// a requirement counts as `plan` counts it. When it does not, they are judged over the assignments read
|
|
||||||
// after the write, which can only find more unmet than the closure would: a seat said held there is
|
|
||||||
// held in the closure too.
|
|
||||||
func resolvedAfter(ctx context.Context, open *stores, shelf map[string]catalogue.Manifest, node string) afterAct {
|
|
||||||
plan, _, err := resolveNode(ctx, open, node)
|
|
||||||
if err == nil {
|
|
||||||
out := afterAct{plan: plan, unheld: plan.Unheld}
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
out.modules = append(out.modules, m.Module)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
out := afterAct{err: err}
|
|
||||||
assigned, readErr := open.inventory.Assigned(ctx, node)
|
|
||||||
if readErr != nil {
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
out.modules, out.unheld = assigned, unheldOver(shelf, node, assigned)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
|
||||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
|
||||||
// command API and the controller seat's verbs as they do from the command line.
|
|
||||||
func splitModules(field string) []string {
|
|
||||||
var out []string
|
|
||||||
for _, m := range strings.Split(field, ",") {
|
|
||||||
if m = strings.TrimSpace(m); m != "" {
|
|
||||||
out = append(out, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
|
||||||
//
|
|
||||||
// **The state, not the cause.** Saying "this assignment broke laptop" would mean resolving the
|
|
||||||
// whole mesh twice and would still be a guess about which of several changes did it; saying
|
|
||||||
// "laptop cannot be worked out, and here is what it says" is true, is what somebody has to fix,
|
|
||||||
// and cannot mislead. The machine that was just changed is left out because its own refusal is
|
|
||||||
// already the answer beside this one.
|
|
||||||
//
|
|
||||||
// Nothing here can fail the act it reports on. A mesh that cannot be read is worth saying and is
|
|
||||||
// not a reason to claim the assignment did not happen — it did.
|
|
||||||
func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
|
||||||
// One reading of the mesh for every machine this looks at.
|
|
||||||
ctx = withInputCache(ctx)
|
|
||||||
nodes, err := open.inventory.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return "\n\nThe rest of the mesh could not be checked: " + err.Error()
|
|
||||||
}
|
|
||||||
blocked := map[string]string{}
|
|
||||||
for _, n := range nodes {
|
|
||||||
if n.Name == except {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, _, err := resolveNode(ctx, open, n.Name); err != nil {
|
|
||||||
blocked[n.Name] = err.Error()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(blocked) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(blocked))
|
|
||||||
for name := range blocked {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
|
|
||||||
var out strings.Builder
|
|
||||||
fmt.Fprintf(&out, "\n\nAND %d other machine(s) cannot be worked out as things stand, so "+
|
|
||||||
"nothing will be sent to them:\n", len(names))
|
|
||||||
for _, name := range names {
|
|
||||||
fmt.Fprintf(&out, " %s\n", name)
|
|
||||||
for _, line := range strings.Split(strings.TrimRight(blocked[name], "\n"), "\n") {
|
|
||||||
fmt.Fprintf(&out, " %s\n", strings.TrimSpace(line))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
|
|
||||||
return out.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
|
||||||
// says what it did in one line. Nothing for a module that declares no broker secret and runs no
|
|
||||||
// provisioner client; nothing for one whose user is already minted (a credential is rotated on
|
|
||||||
// purpose, never by re-assigning); and when the bus cannot be reached from here, the line names the
|
|
||||||
// verb and the push that would refuse the module until it is run — never a silent placeholder (novox/hq issue 203).
|
|
||||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
|
||||||
inv := open.inventory
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
m, known := shelf[module]
|
|
||||||
if !known || mayIssue(m) != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
|
||||||
_, reads := m.OwnSecrets["broker"]
|
|
||||||
if !reads && !runsTheProvisionerClient(m) {
|
|
||||||
// A module whose bus accounts are its processes' alone has none of its own: only the processes that
|
|
||||||
// are not issued yet are issued (issue 1226).
|
|
||||||
return processAccountsOnAssign(ctx, inv, m, node)
|
|
||||||
}
|
|
||||||
if !reads {
|
|
||||||
// A module running the provisioner client alone is minted the client's account (issue 917).
|
|
||||||
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
|
||||||
}
|
|
||||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
|
||||||
return ""
|
|
||||||
} else if minted {
|
|
||||||
// Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
|
|
||||||
return processAccountsOnAssign(ctx, inv, m, node)
|
|
||||||
}
|
|
||||||
busAddress, err := broker.BusAddress()
|
|
||||||
if err == nil {
|
|
||||||
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
|
|
||||||
"`push %s` refuses to send %s until it is", err, module, node, node, module)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// processAccountsOnAssign issues the process accounts of an assigned module not issued yet, and says what it did.
|
|
||||||
func processAccountsOnAssign(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) string {
|
|
||||||
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Sprintf("its process accounts are not issued (%v): `module issue %s --node %s` first — "+
|
|
||||||
"`push %s` refuses to send %s until it is", err, m.Module, node, node, m.Module)
|
|
||||||
}
|
|
||||||
return line
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
|
|
||||||
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
|
|
||||||
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
|
|
||||||
// change that says who may change what.
|
|
||||||
func refusedMovingTheController(modules []string) error {
|
|
||||||
verb, through := throughAVerb()
|
|
||||||
if !through || !slices.Contains(modules, controllerModule) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s is assigned and unassigned at the controller's terminal only (mesh-cli on the control-node), "+
|
|
||||||
"never through a verb (this came through %q): the node it runs on is the control-node, whose operator is the "+
|
|
||||||
"terminal (novox/hq ADR 0272). Nothing was assigned", controllerModule, verb)
|
|
||||||
}
|
|
||||||
@@ -1,135 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What an assignment says about the machines it was not about.
|
|
||||||
|
|
||||||
// **An assignment that blocks another machine says so.**
|
|
||||||
//
|
|
||||||
// The shape found on a four-node raise: a module offering a mesh-scoped provision stops offering it
|
|
||||||
// the moment its own node stops resolving, so an assignment to the provider's machine silently took
|
|
||||||
// a provision away from every consumer elsewhere. Those consumers were then told *nothing in this
|
|
||||||
// mesh provides it*, naming as the remedy a module that was already assigned — which read, on the
|
|
||||||
// way back, as a version bump breaking provider recognition. It was neither the version nor the
|
|
||||||
// provider: it was one machine's set of assignments, and nothing said so.
|
|
||||||
//
|
|
||||||
// novox/hq 04-ISSUES/017 names the general shape — an action succeeding into a state its own verify
|
|
||||||
// rejects, because the action's test is not the test the verify uses. `assign` tested one node; the
|
|
||||||
// verify is resolution over all of them.
|
|
||||||
func TestAnAssignmentThatBlocksAnotherMachineSaysWhichAndWhy(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
|
|
||||||
// A provider on the hub and a consumer on the other machine, both resolving.
|
|
||||||
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
|
|
||||||
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
|
|
||||||
Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
|
||||||
Requires: []string{"acme-ca"}})
|
|
||||||
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
said, err := assign(ctx, open, "laptop", "route-proxy")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("a mesh that should resolve did not: %v\n%s", err, said)
|
|
||||||
}
|
|
||||||
if strings.Contains(said, "cannot be worked out") {
|
|
||||||
t.Fatalf("a well mesh was reported as blocked:\n%s", said)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now break the hub's own set, with nothing but the command a person has.
|
|
||||||
one, two := rivals()
|
|
||||||
register(t, open, one)
|
|
||||||
register(t, open, two)
|
|
||||||
if _, err := assign(ctx, open, "anchor", "rival-one"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
said, err = assign(ctx, open, "anchor", "rival-two")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("an assignment that makes its own node incoherent was not reported at all")
|
|
||||||
}
|
|
||||||
|
|
||||||
// The node's own refusal is the error, as it always was. What is new is that the machines this
|
|
||||||
// just took a provision away from are named in the same breath.
|
|
||||||
if !strings.Contains(said, "laptop") {
|
|
||||||
t.Fatalf("the machine this blocked is not named:\n%s\n\n%v", said, err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(said, "acme-ca") {
|
|
||||||
t.Fatalf("what laptop is now missing is not said:\n%s", said)
|
|
||||||
}
|
|
||||||
if !strings.Contains(said, "cannot be worked out as things stand") {
|
|
||||||
t.Fatalf("the report does not say what state the mesh is in:\n%s", said)
|
|
||||||
}
|
|
||||||
// And the assignment is kept: it is what a person meant, and assignment is not an ordering.
|
|
||||||
assigned, err := open.inventory.Assigned(ctx, "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !contains(assigned, "rival-two") {
|
|
||||||
t.Fatalf("the assignment was not kept: %v", assigned)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A consumer assigned before its provider is refused for itself and kept, because assignment is not
|
|
||||||
// an ordering — refusing the first half of a pair would make the order somebody types two commands
|
|
||||||
// in part of the mesh's rules.
|
|
||||||
func TestAConsumerAssignedBeforeItsProviderIsStillAssigned(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
|
||||||
Requires: []string{"acme-ca"}})
|
|
||||||
|
|
||||||
said, err := assign(ctx, open, "laptop", "route-proxy")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("a consumer with nothing to consume resolved:\n%s", said)
|
|
||||||
}
|
|
||||||
assigned, err := open.inventory.Assigned(ctx, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !contains(assigned, "route-proxy") {
|
|
||||||
t.Fatalf("assignment became an ordering: %v", assigned)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unassigning is the ordinary way to stop providing something to another machine, and it reports
|
|
||||||
// the same way for the same reason.
|
|
||||||
func TestUnassigningAProviderNamesWhoIsNowBlocked(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
|
|
||||||
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
|
|
||||||
Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
|
||||||
Requires: []string{"acme-ca"}})
|
|
||||||
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "laptop", "route-proxy"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
said, err := unassign(ctx, open, "anchor", "step-ca")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(said, "laptop") || !strings.Contains(said, "acme-ca") {
|
|
||||||
t.Fatalf("taking the provider away said nothing about who was consuming it:\n%s", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func contains(all []string, one string) bool {
|
|
||||||
for _, s := range all {
|
|
||||||
if s == one {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,224 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/licences"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aStore())
|
|
||||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
|
||||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
|
||||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
|
||||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
|
||||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
|
||||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
|
||||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
|
||||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
|
||||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
|
||||||
// asks where this machine reaches the store, as the docker module does.
|
|
||||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
|
||||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
|
||||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
|
||||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
on := map[string]bool{"anchor": true, "laptop": true}
|
|
||||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
|
||||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
|
||||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var trust string
|
|
||||||
for _, r := range composed(t, open, "laptop").Resources {
|
|
||||||
if r["path"] == "/etc/docker/daemon.json" {
|
|
||||||
trust, _ = r["content"].(string)
|
|
||||||
}
|
|
||||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
|
||||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
|
||||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
|
||||||
}
|
|
||||||
|
|
||||||
// And a replay to the catalogue says where the store is now.
|
|
||||||
announced, err := following{open: open}.Announceable(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
|
||||||
t.Fatalf("the replay announces %+v", announced)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
raw, err := os.ReadFile("../../module.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
m, err := catalogue.ParseManifest(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
|
||||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
register(t, open, control)
|
|
||||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
|
||||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
|
||||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
|
||||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
|
||||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
|
||||||
{Port: 5672, From: catalogue.FromMesh}},
|
|
||||||
Guards: []int{15672},
|
|
||||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
|
||||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
|
||||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
|
||||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
|
||||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
|
||||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
|
||||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
|
||||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
|
||||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
|
||||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var env map[string]any
|
|
||||||
for _, r := range composed(t, open, "anchor").Resources {
|
|
||||||
if r["id"] == "mesh-controller.controller" {
|
|
||||||
env, _ = r["env"].(map[string]any)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if env == nil {
|
|
||||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
|
||||||
}
|
|
||||||
for key, want := range map[string]string{
|
|
||||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
|
||||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
|
||||||
"MESH_STORE_LICENCES_PORT": "6852",
|
|
||||||
"MESH_BROKER_AMQP_PORT": "5679",
|
|
||||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
|
||||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
|
||||||
"MESH_BROKER_ADDRESS_PORT": "",
|
|
||||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
|
||||||
} {
|
|
||||||
if env[key] != want {
|
|
||||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func aLoneNode(t *testing.T) *stores {
|
|
||||||
t.Helper()
|
|
||||||
inventory.ForTest(t)
|
|
||||||
licences.ForTest(t)
|
|
||||||
open, err := openStores(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(open.Close)
|
|
||||||
for _, m := range provided {
|
|
||||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
|
||||||
{"name": "container-runtime", "present": true}}})
|
|
||||||
var profile map[string]any
|
|
||||||
_ = json.Unmarshal(reported, &profile)
|
|
||||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return open
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
|
||||||
open := aLoneNode(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aStore())
|
|
||||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
|
||||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
|
||||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
|
||||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
|
||||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
|
||||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
|
||||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
|
||||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
|
||||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
|
||||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
|
||||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
|
||||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
|
||||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var image any
|
|
||||||
for _, r := range composed(t, open, "anchor").Resources {
|
|
||||||
if r["id"] == "postgres.runtime" {
|
|
||||||
image = r["image"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
|
||||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// An address is read from the node's settings where it is used, never recorded with a port
|
|
||||||
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.
|
|
||||||
|
|
||||||
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
|
||||||
|
|
||||||
// aStore is a module offering the artifact store on 5000, published the long way as the
|
|
||||||
// distribution module does, so a node may be given another number for it.
|
|
||||||
func aStore() catalogue.Manifest {
|
|
||||||
return catalogue.Manifest{Module: "distribution", Version: "1",
|
|
||||||
Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
|
|
||||||
Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
|
|
||||||
Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
|
|
||||||
Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
|
|
||||||
"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The trust a machine writes for the store, and the address every built image is fetched
|
|
||||||
// through, say the port the node gave the store** — not the catalogue's number.
|
|
||||||
|
|
||||||
// **The control plane's own connections say the port the node gave the store and the broker.**
|
|
||||||
//
|
|
||||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
|
||||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
|
||||||
// process is told so beside the sealed connection genesis wrote.
|
|
||||||
|
|
||||||
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
|
||||||
// added here until module.json carries them (the manifest lands one commit after the code that
|
|
||||||
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
|
|
||||||
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
|
||||||
seatPorts := map[string]string{
|
|
||||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
|
||||||
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
|
||||||
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
|
||||||
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
|
||||||
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
|
||||||
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
|
||||||
}
|
|
||||||
out := m
|
|
||||||
out.Resources = nil
|
|
||||||
for _, r := range m.Resources {
|
|
||||||
if r["type"] != "container" && r["type"] != "process" {
|
|
||||||
out.Resources = append(out.Resources, r)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
copied := map[string]any{}
|
|
||||||
for k, v := range r {
|
|
||||||
copied[k] = v
|
|
||||||
}
|
|
||||||
env := map[string]any{}
|
|
||||||
if had, ok := r["env"].(map[string]any); ok {
|
|
||||||
for k, v := range had {
|
|
||||||
env[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for k, v := range seatPorts {
|
|
||||||
if _, said := env[k]; !said {
|
|
||||||
env[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
copied["env"] = env
|
|
||||||
out.Resources = append(out.Resources, copied)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
|
||||||
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
|
||||||
// catalogue have each been built and pushed (finding F2).
|
|
||||||
|
|
||||||
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
|
||||||
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
|
||||||
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
|
||||||
@@ -1,643 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq ADR 0100: taking a module is its cutover; converging a node is one act, previewed, and
|
|
||||||
// refused while a found container is held; returning to adopted keeps what was taken.
|
|
||||||
|
|
||||||
// anAdoptedAnchor is aMesh with the anchor adopted, running a served module the predecessor also
|
|
||||||
// runs and a module with only a file, and a filter module in the catalogue.
|
|
||||||
func anAdoptedAnchor(t *testing.T) (*stores, *[]string) {
|
|
||||||
t.Helper()
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, catalogue.Manifest{Module: "hello-web", Version: "1",
|
|
||||||
Listens: []catalogue.Listening{{Port: 8080, From: catalogue.FromEverywhere}},
|
|
||||||
Resources: []map[string]any{
|
|
||||||
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hi"},
|
|
||||||
{"id": "server", "type": "container", "name": "hello-web", "ports": []any{"8080:80"},
|
|
||||||
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
|
||||||
}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
|
||||||
Resources: []map[string]any{
|
|
||||||
{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"},
|
|
||||||
}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "nftables", Version: "1",
|
|
||||||
Filtering: &catalogue.Filtering{Into: "/etc/nftables.conf"},
|
|
||||||
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
|
|
||||||
"state": "running", "restart-on": []any{"filtering"}}}})
|
|
||||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, m := range []string{"hello-web", "notes"} {
|
|
||||||
if _, err := assign(ctx, open, "anchor", m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sent := &[]string{}
|
|
||||||
saved := sendNodes
|
|
||||||
sendNodes = func(_ context.Context, _ *stores, names []string) error {
|
|
||||||
*sent = append(*sent, names...)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { sendNodes = saved })
|
|
||||||
return open, sent
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportsHolding has the anchor report, on what it was last sent, holding what is given.
|
|
||||||
func reportsHolding(t *testing.T, open *stores, held ...link.Held) {
|
|
||||||
t.Helper()
|
|
||||||
reportsReaching(t, open, []link.Reach{
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
|
||||||
ContainerPort: 80},
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry",
|
|
||||||
Published: true, ContainerPort: 5000},
|
|
||||||
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker",
|
|
||||||
Published: true, ContainerPort: 15672},
|
|
||||||
}, held...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportsReaching has the anchor report, on what it was last sent, what is reachable on it and
|
|
||||||
// holding what is given.
|
|
||||||
func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ...link.Held) {
|
|
||||||
t.Helper()
|
|
||||||
ctx := t.Context()
|
|
||||||
body, err := composed(t, open, "anchor").Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, err := open.inventory.NodeByName(ctx, "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
|
||||||
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
|
||||||
Firewall: "ufw", Held: held, Reachable: reachable,
|
|
||||||
// A machine says which of its links face outside on every apply (novox/hq ADR 0140), and a
|
|
||||||
// filter is not sent to one that has not. The anchor reports one, as a real host does; this
|
|
||||||
// fixture lacked it from 2026-09-28 and nothing ran the test (issue 177).
|
|
||||||
Outward: []string{"eth0"},
|
|
||||||
// And what filters it (ADR 0168): its front end, the runtime's own, and a chain a
|
|
||||||
// predecessor left in the runtime's user chain.
|
|
||||||
Filters: anchorFilters,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// anchorFilters is what the adopted anchor says filters it: ufw's chains, the runtime's, and a
|
|
||||||
// predecessor's chain the mesh did not write.
|
|
||||||
var anchorFilters = []link.Filter{
|
|
||||||
{Where: "table ip filter, chain ufw-reject-input", Owner: "found-firewall", Refuses: "reject"},
|
|
||||||
{Where: "table ip filter, chain DOCKER", Owner: "runtime", Refuses: `iifname != "docker0" oifname "docker0" drop`},
|
|
||||||
{Where: "table ip filter, chain DOCKER-USER", Owner: "other", Refuses: `iifname "eth0" tcp dport 6000 drop`},
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container",
|
|
||||||
Target: "hello-web", Since: time.Now()}
|
|
||||||
heldFile = link.Held{ID: "notes.conf", Module: "notes", Kind: "file",
|
|
||||||
Target: "/etc/notes.conf", Since: time.Now(), Kept: "/var/lib/mesh-host/kept/abc-notes.conf"}
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
if _, err := take(t.Context(), open, "anchor", "nftables", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAssigned) {
|
|
||||||
t.Fatalf("taking an unassigned module gave %v", err)
|
|
||||||
}
|
|
||||||
if _, err := take(t.Context(), open, "laptop", "network", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAdopted) {
|
|
||||||
t.Fatalf("taking on a converged node gave %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) {
|
|
||||||
open, sent := anAdoptedAnchor(t)
|
|
||||||
_, err := converge(t.Context(), open, "anchor", false, "", "")
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "has not reported") {
|
|
||||||
t.Fatalf("a node that never reported was previewed: %v", err)
|
|
||||||
}
|
|
||||||
if len(*sent) != 0 {
|
|
||||||
t.Fatal("a refused converge sent something")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) {
|
|
||||||
open, sent := anAdoptedAnchor(t)
|
|
||||||
reportsHolding(t, open, heldContainer, heldFile)
|
|
||||||
_, err := converge(t.Context(), open, "anchor", true, "", "")
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "take anchor hello-web once its data has moved") {
|
|
||||||
t.Fatalf("the flip was not refused while hello-web holds its found container: %v", err)
|
|
||||||
}
|
|
||||||
if n, _ := open.inventory.NodeByName(t.Context(), "anchor"); !n.Adopted || len(*sent) != 0 {
|
|
||||||
t.Fatal("a refused flip changed something")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTakingNamesWhatItReplaces(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
reportsHolding(t, open, heldContainer, heldFile)
|
|
||||||
ctx := t.Context()
|
|
||||||
// The machine holds something for the module, so the take acts on the preview the operator
|
|
||||||
// saw and names its digest (novox/hq ADR 0163).
|
|
||||||
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
saw := takeDigestIn(t, preview)
|
|
||||||
if !strings.Contains(preview, "nothing taken; `take anchor hello-web --yes "+saw+"`") {
|
|
||||||
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
|
||||||
}
|
|
||||||
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
|
|
||||||
t.Fatal("the preview took something")
|
|
||||||
}
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil || !strings.Contains(err.Error(), "name its digest") {
|
|
||||||
t.Fatalf("--yes without the digest was not refused: %v", err)
|
|
||||||
}
|
|
||||||
said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(said, "container hello-web (hello-web.server)") ||
|
|
||||||
!strings.Contains(said, "push anchor") {
|
|
||||||
t.Fatalf("taking did not say what it replaces and what to run:\n%s", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeDigestIn is the digest a take's preview printed.
|
|
||||||
func takeDigestIn(t *testing.T, preview string) string {
|
|
||||||
t.Helper()
|
|
||||||
for _, line := range strings.Split(preview, "\n") {
|
|
||||||
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
|
||||||
return fields[1]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("the preview printed no digest:\n%s", preview)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// A take acts on the preview the operator saw, and on an account of the machine that is still the
|
|
||||||
// machine: a changed preview and a stale account refuse (novox/hq ADR 0163, rule 1).
|
|
||||||
func TestATakeIsRefusedOnAChangedPreviewOrAStaleAccount(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
reportsHolding(t, open, heldContainer, heldFile)
|
|
||||||
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
saw := takeDigestIn(t, preview)
|
|
||||||
|
|
||||||
// The machine reports again, and what it holds has changed: the found container now carries
|
|
||||||
// facts the preview never showed.
|
|
||||||
changed := heldContainer
|
|
||||||
changed.Facts = map[string]any{"image": "hello:2", "declared_image": "registry.example/hello"}
|
|
||||||
reportsHolding(t, open, changed, heldFile)
|
|
||||||
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
|
||||||
t.Fatalf("a changed preview was acted on: %v", err)
|
|
||||||
}
|
|
||||||
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
|
|
||||||
t.Fatal("a refused take took something")
|
|
||||||
}
|
|
||||||
|
|
||||||
// And an account older than the flip allows.
|
|
||||||
preview, err = take(ctx, open, "anchor", "hello-web", takeOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
saw = takeDigestIn(t, preview)
|
|
||||||
saved := reportFreshFor
|
|
||||||
reportFreshFor = -time.Second
|
|
||||||
defer func() { reportFreshFor = saved }()
|
|
||||||
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "a take acts only on an account newer than") {
|
|
||||||
t.Fatalf("a stale account was acted on: %v", err)
|
|
||||||
}
|
|
||||||
reportFreshFor = saved
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// A module the machine holds nothing for has nothing to compare: --yes alone suffices.
|
|
||||||
if _, err := take(ctx, open, "anchor", "notes", takeOptions{Yes: true}); err == nil {
|
|
||||||
// notes holds a file, so this one needs the digest too.
|
|
||||||
t.Fatal("notes holds a found file and was taken without a digest")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) {
|
|
||||||
open, sent := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reportsHolding(t, open, heldFile)
|
|
||||||
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{
|
|
||||||
"tcp/8080 hello-web (published, container port 80)",
|
|
||||||
"declared by hello-web (from anywhere)",
|
|
||||||
"WILL CLOSE — no module assigned here declares it",
|
|
||||||
// The anchor faces inward and is on the private network: the derived filter admits ssh
|
|
||||||
// from the mesh only.
|
|
||||||
"WILL CLOSE to everything outside the private network — ssh stays open from the mesh",
|
|
||||||
"notes\n replacing the found file /etc/notes.conf (notes.conf), original kept at",
|
|
||||||
"assigns nftables",
|
|
||||||
"the found firewall (ufw) is disabled, never flushed",
|
|
||||||
// What it routes is not a listener: said not to be previewed, and to be dropped.
|
|
||||||
"not previewed: traffic the machine routes that is not a published port",
|
|
||||||
"the derived filter drops it unless a module declares it",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(preview, want) {
|
|
||||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(preview, "15672") {
|
|
||||||
t.Errorf("a loopback listener is in the preview:\n%s", preview)
|
|
||||||
}
|
|
||||||
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the predecessor's
|
|
||||||
// chain is named as not the mesh's and left, so the reader knows before the flip.
|
|
||||||
for _, want := range []string{
|
|
||||||
"table ip filter, chain DOCKER-USER",
|
|
||||||
"NOT THE MESH'S; left in force",
|
|
||||||
`iifname "eth0" tcp dport 6000 drop`,
|
|
||||||
"table ip filter, chain ufw-reject-input",
|
|
||||||
"the found firewall's; retired with it",
|
|
||||||
"the container runtime's own; left",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(preview, want) {
|
|
||||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, line := range strings.Split(preview, "\n") {
|
|
||||||
if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") {
|
|
||||||
t.Errorf("an undeclared published port is not said to close: %s", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
|
||||||
t.Fatal("the preview changed something")
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
n, _ := open.inventory.NodeByName(ctx, "anchor")
|
|
||||||
if n.Adopted {
|
|
||||||
t.Fatal("converge --yes left the node adopted")
|
|
||||||
}
|
|
||||||
taken, _ := open.inventory.Taken(ctx, "anchor")
|
|
||||||
if !reflect.DeepEqual(taken, []string{"hello-web", overlay.Name, "nftables", "notes"}) {
|
|
||||||
t.Fatalf("the flip took %v", taken)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(*sent, []string{"anchor"}) {
|
|
||||||
t.Fatalf("the flip sent %v", *sent)
|
|
||||||
}
|
|
||||||
declared := composed(t, open, "anchor")
|
|
||||||
if declared.Adoption != nil {
|
|
||||||
t.Fatal("a converged node is still sent an adoption envelope")
|
|
||||||
}
|
|
||||||
if !hasID(declared.Resources, "nftables.filtering") {
|
|
||||||
t.Fatal("the converged node is not declared the mesh's filter")
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := adopt(ctx, open, "anchor"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := adopt(ctx, open, "anchor"); err == nil {
|
|
||||||
t.Fatal("adopting an adopted node was not refused")
|
|
||||||
}
|
|
||||||
again, _ := open.inventory.Taken(ctx, "anchor")
|
|
||||||
if !reflect.DeepEqual(again, taken) {
|
|
||||||
t.Fatalf("returning to adopted lost what was taken: %v", again)
|
|
||||||
}
|
|
||||||
declared = composed(t, open, "anchor")
|
|
||||||
if declared.Adoption == nil || len(declared.Adoption.Untaken) != 0 {
|
|
||||||
t.Fatalf("returned to adopted, the envelope is %+v", declared.Adoption)
|
|
||||||
}
|
|
||||||
if hasID(declared.Resources, "nftables.filtering") || hasID(declared.Resources, "nftables.load") {
|
|
||||||
t.Fatal("returned to adopted, the mesh's filter is still declared")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func hasID(resources []map[string]any, id string) bool {
|
|
||||||
for _, r := range resources {
|
|
||||||
if r["id"] == id {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// The command API refuses a flip exactly as the command line does, in the same words.
|
|
||||||
func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
reportsHolding(t, open, heldContainer)
|
|
||||||
_, direct := converge(t.Context(), open, "anchor", true, "", "")
|
|
||||||
if direct == nil {
|
|
||||||
t.Fatal("the flip was not refused")
|
|
||||||
}
|
|
||||||
got := asking(t, letIn{}, "POST", "/converge", `{"node":"anchor","yes":true}`)
|
|
||||||
if got.Code != http.StatusConflict {
|
|
||||||
t.Fatalf("got %d: %s", got.Code, got.Body.String())
|
|
||||||
}
|
|
||||||
var said map[string]any
|
|
||||||
if err := json.Unmarshal(got.Body.Bytes(), &said); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if said["refused"] != direct.Error() {
|
|
||||||
t.Fatalf("the API said %q and the command line %q", said["refused"], direct.Error())
|
|
||||||
}
|
|
||||||
if got := asking(t, letIn{}, "POST", "/take", `{"node":"anchor"}`); got.Code != http.StatusBadRequest {
|
|
||||||
t.Fatalf("a take naming no module got %d", got.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq ADR 0100: the preview says what the derived filter does, rendered as it is rendered. On
|
|
||||||
// a machine that faces inward, ssh is admitted from the private network only, and a port a module
|
|
||||||
// admits from the mesh only closes to everything outside it: both are said to close.
|
|
||||||
func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
|
||||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}})
|
|
||||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reportsReaching(t, open, []link.Reach{
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 5432, By: "postgres"},
|
|
||||||
{Protocol: "tcp", Address: "10.77.0.1", Port: 5432, By: "postgres"},
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
|
||||||
ContainerPort: 80},
|
|
||||||
})
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
lines := map[string]string{}
|
|
||||||
for _, line := range strings.Split(preview, "\n") {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) > 1 && strings.HasPrefix(fields[0], "tcp/") {
|
|
||||||
lines[fields[0]+" "+fields[1]] += line + "\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got := lines["tcp/22 sshd"]; !strings.Contains(got, "WILL CLOSE to everything outside "+
|
|
||||||
"the private network") {
|
|
||||||
t.Errorf("ssh on an inward machine is not said to close outside the mesh:\n%s", preview)
|
|
||||||
}
|
|
||||||
store := lines["tcp/5432 postgres"]
|
|
||||||
if strings.Count(store, "WILL CLOSE to everything outside the private network") != 1 ||
|
|
||||||
!strings.Contains(store, "declared by store (from mesh)") {
|
|
||||||
t.Errorf("the store's narrowing is not said to close, or its mesh address is:\n%s", preview)
|
|
||||||
}
|
|
||||||
if got := lines["tcp/8080 hello-web"]; !strings.Contains(got, "declared by hello-web (from anywhere)") {
|
|
||||||
t.Errorf("a port open to everywhere is not said to stay:\n%s", preview)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// digestIn is the digest a converge preview printed.
|
|
||||||
func digestIn(t *testing.T, preview string) string {
|
|
||||||
t.Helper()
|
|
||||||
for _, line := range strings.Split(preview, "\n") {
|
|
||||||
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
|
||||||
return fields[1]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("the preview printed no digest:\n%s", preview)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// The flip acts on the preview the operator saw: it names that preview's digest, and it is refused
|
|
||||||
// when the digest is missing, when anything the preview says has changed since, or when the node's
|
|
||||||
// account of itself is too old to be the machine as it is.
|
|
||||||
func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
|
||||||
open, sent := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reportsHolding(t, open, heldFile)
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
saw := digestIn(t, preview)
|
|
||||||
if !strings.Contains(preview, "converge anchor --yes "+saw) {
|
|
||||||
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
|
||||||
}
|
|
||||||
unchanged := func() {
|
|
||||||
t.Helper()
|
|
||||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
|
||||||
t.Fatal("a refused flip changed something")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := converge(ctx, open, "anchor", true, "", ""); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), "--yes "+saw) {
|
|
||||||
t.Fatalf("a flip naming no preview was not refused: %v", err)
|
|
||||||
}
|
|
||||||
unchanged()
|
|
||||||
|
|
||||||
// Something new is reachable: the preview the operator saw is not what would happen.
|
|
||||||
reportsReaching(t, open, []link.Reach{
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
|
||||||
ContainerPort: 80},
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 6000, By: "something-new"},
|
|
||||||
}, heldFile)
|
|
||||||
_, err = converge(ctx, open, "anchor", true, saw, "")
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
|
||||||
t.Fatalf("a flip on a changed preview was not refused: %v", err)
|
|
||||||
}
|
|
||||||
unchanged()
|
|
||||||
|
|
||||||
// An account older than the flip trusts is refused, whatever digest is named.
|
|
||||||
again, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
saved := reportFreshFor
|
|
||||||
reportFreshFor = time.Nanosecond
|
|
||||||
_, err = converge(ctx, open, "anchor", true, digestIn(t, again), "")
|
|
||||||
reportFreshFor = saved
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "wait for its next report") {
|
|
||||||
t.Fatalf("a flip on an old account was not refused: %v", err)
|
|
||||||
}
|
|
||||||
unchanged()
|
|
||||||
|
|
||||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, again), ""); err != nil {
|
|
||||||
t.Fatalf("the flip on the preview just seen was refused: %v", err)
|
|
||||||
}
|
|
||||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); n.Adopted {
|
|
||||||
t.Fatal("the flip did not converge the node")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
|
||||||
// composed after it — never sent after it with the node still adopted. And the send inside the
|
|
||||||
// flip is not made to wait on the flip's own hold.
|
|
||||||
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reportsHolding(t, open, heldFile)
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var heldElsewhere, heldHere error
|
|
||||||
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
|
||||||
// Another caller cannot hold the node while the flip sends it.
|
|
||||||
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
|
||||||
release()
|
|
||||||
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
|
||||||
}
|
|
||||||
// The flip's own send holds it without waiting on itself.
|
|
||||||
_, release, err := holdNodes(inner, open, names)
|
|
||||||
if err != nil {
|
|
||||||
heldHere = err
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
release()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if heldElsewhere != nil || heldHere != nil {
|
|
||||||
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
|
||||||
}
|
|
||||||
// And given back once it is done.
|
|
||||||
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
release()
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
|
||||||
// not only its files, and the digest changes when any of them does.
|
|
||||||
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
since := time.Now()
|
|
||||||
held := []link.Held{heldFile,
|
|
||||||
{ID: "notes.data", Module: "notes", Kind: "directory", Target: "/var/lib/notes", Since: since},
|
|
||||||
{ID: "notes.daemon", Module: "notes", Kind: "service", Target: "notes.service", Since: since},
|
|
||||||
{ID: "notes.seed", Module: "notes", Kind: "archive", Target: "/srv/notes", Since: since},
|
|
||||||
{ID: "notes.worker", Module: "notes", Kind: "process", Target: "notes-worker", Since: since},
|
|
||||||
{ID: "notes.account", Module: "notes", Kind: "user", Target: "notes", Since: since},
|
|
||||||
}
|
|
||||||
reportsHolding(t, open, held...)
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{
|
|
||||||
"replacing the found directory /var/lib/notes (notes.data)",
|
|
||||||
"replacing the found service notes.service (notes.daemon)",
|
|
||||||
"replacing the found archive /srv/notes (notes.seed)",
|
|
||||||
"replacing the found process notes-worker (notes.worker)",
|
|
||||||
"replacing the found user notes (notes.account)",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(preview, want) {
|
|
||||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reportsHolding(t, open, held[:len(held)-1]...)
|
|
||||||
fewer, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if digestIn(t, fewer) == digestIn(t, preview) {
|
|
||||||
t.Fatal("the digest does not change with what is held")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq ADR 0100: the flip acts on what the node said is reachable, so an account naming nothing
|
|
||||||
// is refused. Every machine that is up answers on ssh; nothing reported means the host's collectors
|
|
||||||
// did not, and flipping would close ports the preview never named.
|
|
||||||
func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
|
||||||
open, sent := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Only a loopback listener: nothing off the machine, which is the same silence.
|
|
||||||
reportsReaching(t, open, []link.Reach{
|
|
||||||
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
|
|
||||||
}, heldFile)
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(preview, "this account looks partial") {
|
|
||||||
t.Errorf("the preview does not mark a partial account:\n%s", preview)
|
|
||||||
}
|
|
||||||
_, err = converge(ctx, open, "anchor", true, digestIn(t, preview), "")
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "says nothing is reachable on it") {
|
|
||||||
t.Fatalf("the flip was not refused on an account naming nothing: %v", err)
|
|
||||||
}
|
|
||||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
|
||||||
t.Fatal("a refused flip changed something")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
|
||||||
// holds the node, so it waits for whatever is converging it.
|
|
||||||
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reportsHolding(t, open, heldFile)
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
|
||||||
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
|
||||||
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
|
||||||
|
|
||||||
var whileFlipping error
|
|
||||||
sendNodes = func(context.Context, *stores, []string) error {
|
|
||||||
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
|
||||||
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,180 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
|
|
||||||
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
|
|
||||||
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
|
|
||||||
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
|
|
||||||
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
|
|
||||||
// never how often the controller looked.
|
|
||||||
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
|
|
||||||
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
|
|
||||||
now := refused.Add(10 * time.Second)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
|
|
||||||
// The words the live condition bus.controller.refused carried on 2026-10-10.
|
|
||||||
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
|
|
||||||
`Publish to "mesh.seat.mesh-delivery.tool.times"`
|
|
||||||
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
|
|
||||||
First: refused, Last: refused, Count: 1}
|
|
||||||
look := func() conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
|
|
||||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 15 {
|
|
||||||
c = look()
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 1 || len(c.Evidence) != 1 {
|
|
||||||
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
|
|
||||||
c.Observations, len(c.Evidence), c.Evidence)
|
|
||||||
}
|
|
||||||
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
|
|
||||||
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bus refuses it three times more between two looks: the condition counts four refusals, says
|
|
||||||
// the newest, and adds one line of evidence for the look that saw them.
|
|
||||||
again := now.Add(-5 * time.Second)
|
|
||||||
advisory.Last, advisory.Count = again, 4
|
|
||||||
c = look()
|
|
||||||
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
|
|
||||||
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
|
|
||||||
}
|
|
||||||
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
|
|
||||||
t.Fatalf("summary %q", c.Summary)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
|
|
||||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
|
|
||||||
// what reads a record): a machine silent for three looks was observed three times, and says so.
|
|
||||||
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 3 {
|
|
||||||
var err error
|
|
||||||
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
|
|
||||||
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
|
|
||||||
Source: "S1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
|
|
||||||
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
|
|
||||||
// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
|
|
||||||
// without a token names the same key; read beside it, each look added an observation and a line of
|
|
||||||
// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
|
|
||||||
// dead-letter row alone says that key, and counts the messages given up on, never the looks.
|
|
||||||
func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
|
|
||||||
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
now := gaveUp.Add(20 * time.Second)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
|
|
||||||
const key = "bus.EVENTS.media_sonarr.max-deliveries"
|
|
||||||
held := map[string]int{"EVENTS.media_sonarr": 1}
|
|
||||||
newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
|
|
||||||
advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
|
|
||||||
Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
|
|
||||||
First: gaveUp, Last: gaveUp, Count: 1}
|
|
||||||
look := func() conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
|
|
||||||
deadLetters: held, deadLettersNewest: newest}
|
|
||||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("%s: found %v, %v", key, found, err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(c.Summary, "dead-letters verb") {
|
|
||||||
t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
var c conditions.Condition
|
|
||||||
for range 5 {
|
|
||||||
c = look()
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
}
|
|
||||||
if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
|
|
||||||
t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
|
|
||||||
"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The consumer gives up on a second message between two looks: two given up on, one more line.
|
|
||||||
again := now.Add(-10 * time.Second)
|
|
||||||
held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
|
|
||||||
c = look()
|
|
||||||
if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
|
|
||||||
t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
|
|
||||||
c.LastObserved, c.Evidence)
|
|
||||||
}
|
|
||||||
now = now.Add(30 * time.Second)
|
|
||||||
if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
|
|
||||||
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
|
|
||||||
// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
|
|
||||||
func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
|
||||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
|
|
||||||
Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
|
|
||||||
said := watchAdvisories(f)
|
|
||||||
if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
|
|
||||||
t.Fatalf("%+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
|
|
||||||
// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
|
|
||||||
func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
||||||
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
|
||||||
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
|
|
||||||
Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
|
|
||||||
if said := watchAdvisories(f); len(said) != 0 {
|
|
||||||
t.Fatalf("said %+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,253 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
// The account agents run as (novox/hq ADR 0266).
|
|
||||||
//
|
|
||||||
// On the control node every agent session ran as the operator's account, which may become root without a
|
|
||||||
// password — so any agent there could become root without a person, and ADR 0259 §8 (an answer from the
|
|
||||||
// operator's phone authorises an act) rests on that being false where the router and its channels run. The
|
|
||||||
// decision: a node may name an account its agents run as, of their own and without sudo; the operator's
|
|
||||||
// account keeps its sudo.
|
|
||||||
//
|
|
||||||
// - **Named at the controller's terminal only** (`node agent-account`): not a verb, not a setting, so no
|
|
||||||
// agent can name itself another account. Empty is a real state: agents run as the operator there.
|
|
||||||
// - **Composed** as `${machine:agent-account}`, `${machine:agent-home}` and `${machine:agent-root}` for the
|
|
||||||
// agent's module, which declares the account with `root: never`, and as MESH_AGENT_ACCOUNT and
|
|
||||||
// MESH_AGENT_HOME for its tools (catalogue/machine_into_files.go, runtime.go).
|
|
||||||
// - **Judged by the machine itself.** The node-engine reads, on every look, whether an account declared
|
|
||||||
// `root: never` can become root without a person — uid 0, a group that grants root, a sudo rule, a
|
|
||||||
// secret of the mesh it may read — and says it as the declaring module's account verdict, marked
|
|
||||||
// Root "never". agentConfined reads that verdict; the self-check (probe DA) raises
|
|
||||||
// `agent-can-become-root` while it does not hold, and `node show` says it.
|
|
||||||
//
|
|
||||||
// A verdict not given is never a pass: an engine older than the judging, an account not yet declared, a
|
|
||||||
// statement that says nothing of it — each is "not judged", and fails.
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// kindAgentCanBecomeRoot is the condition raised while a machine's agent account can become root without
|
|
||||||
// a person, or is not judged (ADR 0266).
|
|
||||||
const kindAgentCanBecomeRoot = "agent-can-become-root"
|
|
||||||
|
|
||||||
// agentAccountProbe is the self-check's probe of it.
|
|
||||||
const agentAccountProbe = "DA"
|
|
||||||
|
|
||||||
// agentConfined says whether the agents of a machine that names an agent account are confined: the
|
|
||||||
// machine's newest statement holds a healthy account verdict, judged for root, on that account. named is
|
|
||||||
// false for a machine that names none — agents run as the operator account there, which this does not
|
|
||||||
// judge. why is said either way, in the mesh's words; err is a store that could not be read.
|
|
||||||
//
|
|
||||||
// The one judgement: `node show`, the self-check, and ADR 0259's router honouring a verified sender read
|
|
||||||
// it here.
|
|
||||||
// now is the judging clock, threaded so a caller judging several things at one instant judges them all at it.
|
|
||||||
func agentConfined(ctx context.Context, inv *inventory.Inventory, node string, now time.Time) (named, confined bool, why string, err error) {
|
|
||||||
n, err := inv.NodeByName(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
return false, false, "", err
|
|
||||||
}
|
|
||||||
if n.AgentAccount == "" {
|
|
||||||
return false, false, fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)",
|
|
||||||
node, orNoneKnown(n.Account)), nil
|
|
||||||
}
|
|
||||||
h, had, err := inv.HealthOf(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
return true, false, "", err
|
|
||||||
}
|
|
||||||
confined, why = judgedConfined(n.AgentAccount, h, had, now)
|
|
||||||
return true, confined, why, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// verdictFreshFor is how old the statement holding the verdict may be, by this controller's clock. A
|
|
||||||
// node-engine states its health on every change and at least every five minutes (mesh-host's sayAnyway), so
|
|
||||||
// three statements missed is a node-engine stopped, or a machine away. **A stale verdict is not a pass**: an
|
|
||||||
// agent that stopped the node-engine must not leave "cannot become root" standing from before.
|
|
||||||
const verdictFreshFor = 15 * time.Minute
|
|
||||||
|
|
||||||
// judgedConfined is the judgement over one statement, without the store, at now.
|
|
||||||
func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Time) (bool, string) {
|
|
||||||
if !had {
|
|
||||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+
|
|
||||||
"nothing of what it runs", agent)
|
|
||||||
}
|
|
||||||
if age := now.Sub(h.HeardAt); age > verdictFreshFor {
|
|
||||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement was heard at "+
|
|
||||||
"%s, more than %d minutes ago, and a verdict that old is not a verdict on now", agent,
|
|
||||||
h.HeardAt.Local().Format("2006-01-02 15:04"), int(verdictFreshFor.Minutes()))
|
|
||||||
}
|
|
||||||
if h.Contract < link.RootContract {
|
|
||||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+
|
|
||||||
"the judging of an account's root (its statement's contract is %d, the judging is %d)",
|
|
||||||
agent, h.Contract, link.RootContract)
|
|
||||||
}
|
|
||||||
var verdicts []inventory.ResourceHealth
|
|
||||||
for _, r := range h.Resources {
|
|
||||||
if r.Kind == link.KindAccount && r.Target == agent && r.Root == link.RootNever {
|
|
||||||
verdicts = append(verdicts, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(verdicts) == 0 {
|
|
||||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+
|
|
||||||
"verdict on it — no module there declares it never to become root, or the declaration naming it "+
|
|
||||||
"has not been applied", agent)
|
|
||||||
}
|
|
||||||
sort.Slice(verdicts, func(i, j int) bool {
|
|
||||||
return verdicts[i].Module+verdicts[i].Resource < verdicts[j].Module+verdicts[j].Resource
|
|
||||||
})
|
|
||||||
for _, v := range verdicts {
|
|
||||||
switch v.State {
|
|
||||||
case link.StateHealthy:
|
|
||||||
case link.StateUnhealthy:
|
|
||||||
// The engine's own words, which start with link.ReasonRoot when it found a way to root.
|
|
||||||
return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent,
|
|
||||||
orNoneKnown(v.Reason), v.Module, v.Resource)
|
|
||||||
default:
|
|
||||||
return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent,
|
|
||||||
orNoneKnown(v.Reason), v.Module, v.Resource, v.State)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent,
|
|
||||||
h.SaidAt.Local().Format("2006-01-02 15:04"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
|
|
||||||
// search before that is itself the urgent condition: the longest a search is expected to take (link.RootSearchQuiet,
|
|
||||||
// the engine's own value; novox/hq issue 361 — the search runs to completion, with no bound of its own), counted
|
|
||||||
// from when this controller first saw it waiting, never from the engine's start. Quiet raises nothing; it never
|
|
||||||
// makes the agent account confined, which only a healthy verdict from a complete, fresh search does.
|
|
||||||
const searchQuietFor = link.RootSearchQuiet
|
|
||||||
|
|
||||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
|
||||||
const (
|
|
||||||
verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown
|
|
||||||
verdictPending // waiting for the search, and otherwise healthy
|
|
||||||
verdictComplete // judged: healthy, or a way to root found
|
|
||||||
)
|
|
||||||
|
|
||||||
// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it
|
|
||||||
// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when
|
|
||||||
// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at
|
|
||||||
// every restart of the engine.
|
|
||||||
func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int {
|
|
||||||
if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract {
|
|
||||||
return verdictOther
|
|
||||||
}
|
|
||||||
pending, any := false, false
|
|
||||||
for _, r := range h.Resources {
|
|
||||||
if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
any = true
|
|
||||||
switch {
|
|
||||||
case r.State == link.StateHealthy:
|
|
||||||
case r.State == link.StateUnhealthy:
|
|
||||||
return verdictComplete
|
|
||||||
case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending):
|
|
||||||
pending = true
|
|
||||||
default:
|
|
||||||
return verdictOther
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case !any:
|
|
||||||
return verdictOther
|
|
||||||
case pending:
|
|
||||||
return verdictPending
|
|
||||||
}
|
|
||||||
return verdictComplete
|
|
||||||
}
|
|
||||||
|
|
||||||
// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid
|
|
||||||
// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a
|
|
||||||
// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began.
|
|
||||||
func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth,
|
|
||||||
had bool, now time.Time) (bool, error) {
|
|
||||||
switch rootVerdictKind(agent, h, had, now) {
|
|
||||||
case verdictComplete:
|
|
||||||
return false, inv.RootSearchJudged(ctx, node)
|
|
||||||
case verdictPending:
|
|
||||||
since, err := inv.RootSearchPending(ctx, node, now)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return now.Sub(since) <= searchQuietFor, nil
|
|
||||||
}
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's
|
|
||||||
// newest statement, unable to become root without a person (ADR 0266).
|
|
||||||
func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
||||||
inv := d.open.inventory
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, n := range nodes {
|
|
||||||
if n.AgentAccount == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
h, had, err := inv.HealthOf(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
confined, why := judgedConfined(n.AgentAccount, h, had, now)
|
|
||||||
if confined {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Not judged yet only because the node-engine's search runs and no complete, fresh one judges: not the
|
|
||||||
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
|
|
||||||
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
|
|
||||||
// waits past searchQuietFor, or the statement goes stale.
|
|
||||||
if quiet {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root",
|
|
||||||
Machine: n.Name, Severity: conditions.Urgent,
|
|
||||||
Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and "+
|
|
||||||
"no answer from a channel authorises an act there (ADR 0259 §8)", n.Name, why),
|
|
||||||
Said: why})
|
|
||||||
}
|
|
||||||
return sortedFound(out), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// agentAccountLines is what `node show` says of the account agents run as.
|
|
||||||
func agentAccountLines(ctx context.Context, inv *inventory.Inventory, n inventory.Node) []string {
|
|
||||||
if n.AgentAccount == "" {
|
|
||||||
return []string{fmt.Sprintf(" agents run as the operator account (%s); no agent account is named",
|
|
||||||
orNoneKnown(n.Account))}
|
|
||||||
}
|
|
||||||
_, confined, why, err := agentConfined(ctx, inv, n.Name, time.Now())
|
|
||||||
if err != nil {
|
|
||||||
return []string{fmt.Sprintf(" agents run as %s (home %s); whether it can become root could NOT be read: %v",
|
|
||||||
n.AgentAccount, n.AgentHome(), err)}
|
|
||||||
}
|
|
||||||
verdict := "CAN become root, or is not judged: " + why
|
|
||||||
if confined {
|
|
||||||
verdict = why
|
|
||||||
}
|
|
||||||
return []string{fmt.Sprintf(" agents run as %s (home %s)", n.AgentAccount, n.AgentHome()),
|
|
||||||
" " + verdict}
|
|
||||||
}
|
|
||||||
|
|
||||||
// orNoneKnown is a value, or that none is known.
|
|
||||||
func orNoneKnown(s string) string {
|
|
||||||
if strings.TrimSpace(s) == "" {
|
|
||||||
return "none known"
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
@@ -1,161 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
snapshot "git.novox.be/novox/mesh-sdk/go/facts"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-host/rootsearch"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
for _, n := range []string{"anchor", "laptop"} {
|
|
||||||
if _, err := inv.NodeByName(ctx, n); err != nil {
|
|
||||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := inv.SetAccount(ctx, n, "ops", ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
d := &doctor{open: open}
|
|
||||||
found, err := probeAgentAccounts(ctx, d)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found = onlyMachine(found, "anchor")
|
|
||||||
if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent ||
|
|
||||||
!strings.Contains(found[0].Said, "not judged") {
|
|
||||||
t.Fatalf("a named agent account with no verdict: %+v", found)
|
|
||||||
}
|
|
||||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
|
||||||
if w.Headline == "" || w.Needs == "" || w.Resolved == "" {
|
|
||||||
t.Errorf("the condition has no plain words: %+v", w)
|
|
||||||
}
|
|
||||||
|
|
||||||
healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
|
||||||
Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"}
|
|
||||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
|
||||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 {
|
|
||||||
t.Fatalf("a judged agent account still fails: %+v %v", found, err)
|
|
||||||
}
|
|
||||||
if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined {
|
|
||||||
t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err)
|
|
||||||
}
|
|
||||||
if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named ||
|
|
||||||
!strings.Contains(why, "operator account") {
|
|
||||||
t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, _ := f.Encode()
|
|
||||||
if strings.Contains(string(body), "warden") {
|
|
||||||
t.Error("the agent account's name is in the snapshot")
|
|
||||||
}
|
|
||||||
m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor"))
|
|
||||||
if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount {
|
|
||||||
t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
|
||||||
if searchQuietFor != rootsearch.Quiet {
|
|
||||||
t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet)
|
|
||||||
}
|
|
||||||
// A daily search that finishes within the quiet never leaves the account unjudged between two of them.
|
|
||||||
if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet {
|
|
||||||
t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)",
|
|
||||||
rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet)
|
|
||||||
}
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
if _, err := inv.NodeByName(ctx, "anchor"); err != nil {
|
|
||||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
d := &doctor{open: open}
|
|
||||||
say := func(state, reason string) []conditions.Observation {
|
|
||||||
t.Helper()
|
|
||||||
// The engine's since is always now: it was just restarted.
|
|
||||||
v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
|
||||||
Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
|
|
||||||
Account: "agent", Since: time.Now()}
|
|
||||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
|
||||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found, err := probeAgentAccounts(ctx, d)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return onlyMachine(found, "anchor")
|
|
||||||
}
|
|
||||||
running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
|
|
||||||
healthy := func() {
|
|
||||||
t.Helper()
|
|
||||||
if found := say(link.StateHealthy, ""); len(found) != 0 {
|
|
||||||
t.Fatalf("a healthy verdict raised: %+v", found)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if found := say(link.StateUnknown, running); len(found) != 0 {
|
|
||||||
t.Fatalf("a search first seen now was raised: %+v", found)
|
|
||||||
}
|
|
||||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") {
|
|
||||||
t.Fatalf("an account whose search runs was read as confined: %q", why)
|
|
||||||
}
|
|
||||||
// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
|
|
||||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
healthy() // a complete verdict forgets when the waiting began …
|
|
||||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
|
||||||
t.Fatal(err) // … and this restart loop began past the bound
|
|
||||||
}
|
|
||||||
if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
|
||||||
t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
|
|
||||||
}
|
|
||||||
healthy()
|
|
||||||
incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
|
|
||||||
"19:23: timeout); it is tried again later"
|
|
||||||
if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
|
||||||
t.Fatalf("a search that did not finish was not urgent: %+v", found)
|
|
||||||
}
|
|
||||||
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
|
|
||||||
found[0].Severity != conditions.Urgent {
|
|
||||||
t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The agent account's judgement (novox/hq ADR 0266): confined only on a healthy account verdict judged for
|
|
||||||
// root, on the very account; every verdict not given — no statement, an older engine, no verdict on it, a
|
|
||||||
// verdict of unknown — is "not judged" and fails, never a pass.
|
|
||||||
func TestAnAgentAccountIsConfinedOnlyOnAHealthyVerdictJudgedForRoot(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 8, 19, 21, 0, 0, time.UTC)
|
|
||||||
verdict := func(target, root, state, reason string) inventory.ResourceHealth {
|
|
||||||
return inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
|
||||||
Kind: link.KindAccount, Target: target, State: state, Reason: reason, Root: root, Account: target}
|
|
||||||
}
|
|
||||||
statement := func(contract int, rs ...inventory.ResourceHealth) inventory.NodeHealth {
|
|
||||||
return inventory.NodeHealth{Node: "anchor", Contract: contract, SaidAt: at, HeardAt: at, Resources: rs}
|
|
||||||
}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
h inventory.NodeHealth
|
|
||||||
had bool
|
|
||||||
confined bool
|
|
||||||
says string
|
|
||||||
}{
|
|
||||||
{"judged and unable", statement(link.RootContract, verdict("agent", link.RootNever, link.StateHealthy, "")),
|
|
||||||
true, true, "cannot become root without a person"},
|
|
||||||
{"judged and able", statement(link.RootContract, verdict("agent", link.RootNever, link.StateUnhealthy,
|
|
||||||
link.ReasonRoot+": in the group docker, which grants root")), true, false, "in the group docker"},
|
|
||||||
{"a verdict of unknown", statement(link.RootContract, verdict("agent", link.RootNever, link.StateUnknown,
|
|
||||||
"sudo could not be read")), true, false, "not judged"},
|
|
||||||
{"no statement", inventory.NodeHealth{}, false, false, "not judged"},
|
|
||||||
{"an older engine", statement(link.ReadinessContract, verdict("agent", "", link.StateHealthy, "")),
|
|
||||||
true, false, "older than the judging"},
|
|
||||||
{"a verdict on groups only", statement(link.RootContract, verdict("agent", "", link.StateHealthy, "")),
|
|
||||||
true, false, "no verdict on it"},
|
|
||||||
{"a verdict on another account", statement(link.RootContract, verdict("ops", link.RootNever, link.StateHealthy, "")),
|
|
||||||
true, false, "no verdict on it"},
|
|
||||||
} {
|
|
||||||
confined, why := judgedConfined("agent", c.h, c.had, at.Add(time.Minute))
|
|
||||||
if confined != c.confined || !strings.Contains(why, c.says) {
|
|
||||||
t.Errorf("%s: confined %v, %q; want %v saying %q", c.name, confined, why, c.confined, c.says)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A verdict heard longer ago than the bound is no verdict: an agent that stopped the node-engine must not
|
|
||||||
// leave "healthy" standing.
|
|
||||||
fresh := statement(link.RootContract, verdict("agent", link.RootNever, link.StateHealthy, ""))
|
|
||||||
if ok, _ := judgedConfined("agent", fresh, true, at.Add(verdictFreshFor)); !ok {
|
|
||||||
t.Error("a verdict exactly at the bound is still one")
|
|
||||||
}
|
|
||||||
if ok, why := judgedConfined("agent", fresh, true, at.Add(verdictFreshFor+time.Second)); ok ||
|
|
||||||
!strings.Contains(why, "not judged") {
|
|
||||||
t.Errorf("a stale healthy verdict passed: %q", why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// DA raises an urgent condition, with plain words, on a machine whose agent account is not judged unable to
|
|
||||||
// become root; a machine that names none is not its to judge.
|
|
||||||
|
|
||||||
func TestTheAgentRootWordsArePlain(t *testing.T) {
|
|
||||||
w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"})
|
|
||||||
if why, ok := conditions.PlainWords(w, "anchor"); !ok {
|
|
||||||
t.Fatalf("not plain: %s: %+v", why, w)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func onlyMachine(obs []conditions.Observation, machine string) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, o := range obs {
|
|
||||||
if o.Machine == machine {
|
|
||||||
out = append(out, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// The snapshot a merge check composes from carries the agent account as a pseudonym (novox/hq ADR 0266),
|
|
||||||
// so a change is judged against machines that name one, and the name never leaves.
|
|
||||||
|
|
||||||
// No verb runs a `node` command that sets something: through the generic `command` verb, `node account`,
|
|
||||||
// `node agent-account` and every other `node` subcommand but list and show are refused, naming the terminal.
|
|
||||||
func TestNoVerbSetsANodesAccounts(t *testing.T) {
|
|
||||||
for _, line := range []string{
|
|
||||||
"node agent-account novox --clear",
|
|
||||||
"node agent-account novox ops",
|
|
||||||
"node account novox agent",
|
|
||||||
"node account novox",
|
|
||||||
"node add intruder",
|
|
||||||
"node public-domain novox --clear",
|
|
||||||
"node",
|
|
||||||
"node frobnicate",
|
|
||||||
} {
|
|
||||||
argv, err := argvFor("command", map[string]any{"command": line})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "controller's terminal") ||
|
|
||||||
!strings.Contains(err.Error(), "ADR 0266") {
|
|
||||||
t.Errorf("%q ran as %v (%v); want a refusal naming the terminal", line, argv, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, line := range []string{"node show novox", "node list --json", "status --json"} {
|
|
||||||
if _, err := argvFor("command", map[string]any{"command": line}); err != nil {
|
|
||||||
t.Errorf("%q, a read, was refused: %v", line, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := terminalOnly([]string{"node", "account", "a", "b"}); err == nil {
|
|
||||||
t.Error("the refusal is not only the command verb's")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Naming the agent account is the controller's terminal's alone: the `node` verb only shows.
|
|
||||||
func TestTheNodeVerbOnlyShows(t *testing.T) {
|
|
||||||
argv, err := argvFor("node", map[string]any{"node": "anchor"})
|
|
||||||
if err != nil || strings.Join(argv, " ") != "node show anchor" {
|
|
||||||
t.Fatalf("the node verb runs %v (%v)", argv, err)
|
|
||||||
}
|
|
||||||
for _, v := range catalogue.ControllerVerbs {
|
|
||||||
if strings.Contains(v.Name, "agent") {
|
|
||||||
t.Errorf("a verb %q may name the agent account", v.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Until a complete search for setuid programs judges — none since the node-engine's state was kept, or the last
|
|
||||||
// older than the engine lets one judge — the agent account is not judged, and the search runs to its end with no
|
|
||||||
// bound (novox/hq issue 361). DA does not raise that as urgent within searchQuietFor, counted from when this
|
|
||||||
// controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine
|
|
||||||
// restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not
|
|
||||||
// judged; a search that failed, or a way to root found, is urgent at once.
|
|
||||||
@@ -1,186 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The command API: what the mesh can be asked to do, over a network.
|
|
||||||
//
|
|
||||||
// **An adapter and nothing else** (novox/hq ADR 0035). Every route here calls the same function
|
|
||||||
// the command line calls, so a refusal is the same refusal in the same words. Nothing is decided
|
|
||||||
// in this file — the moment it validates something the command line does not, the mesh has two
|
|
||||||
// answers to one question.
|
|
||||||
//
|
|
||||||
// **It refuses everything unless it was told how to know who is asking.** The board is published
|
|
||||||
// on a public name, and this is what stands behind it: an unauthenticated command surface reachable
|
|
||||||
// from the internet is authority over the mesh handed to whoever finds it. So there is no
|
|
||||||
// permissive default and no flag that removes the check — a mesh that has not been told how to
|
|
||||||
// authenticate serves nothing, loudly.
|
|
||||||
//
|
|
||||||
// The intended authenticator is an OAuth2 provider (ADR 0035), which is an ordinary module. Until
|
|
||||||
// one is configured this refuses, which is the correct behaviour rather than a placeholder: a
|
|
||||||
// surface that worked without authentication would be one somebody left running.
|
|
||||||
func apiCommand(ctx context.Context, args []string) error {
|
|
||||||
set := flag.NewFlagSet("api", flag.ContinueOnError)
|
|
||||||
listen := set.String("listen", "127.0.0.1:8081", "where to serve it")
|
|
||||||
issuer := set.String("issuer", "",
|
|
||||||
"the OAuth2 issuer whose tokens this accepts; without it, nothing is served")
|
|
||||||
if _, err := parseAround(set, args); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(*issuer) == "" {
|
|
||||||
// Refused at start rather than per request, so it is discovered by whoever ran it rather
|
|
||||||
// than by whoever finds it.
|
|
||||||
return errors.New(
|
|
||||||
"--issuer is not set, and this serves commands rather than pages: it will not run " +
|
|
||||||
"without being told whose tokens to believe. An OAuth2 provider is an ordinary " +
|
|
||||||
"module (novox/hq ADR 0035)")
|
|
||||||
}
|
|
||||||
|
|
||||||
server := &http.Server{
|
|
||||||
Addr: *listen,
|
|
||||||
ReadHeaderTimeout: 10 * time.Second,
|
|
||||||
Handler: commands(mustAuthenticate(*issuer)),
|
|
||||||
}
|
|
||||||
fmt.Printf("the command API is on http://%s\n", *listen)
|
|
||||||
fmt.Printf(" it accepts tokens from %s and refuses everything else\n", *issuer)
|
|
||||||
fmt.Printf(" every route calls what the command line calls\n")
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
<-ctx.Done()
|
|
||||||
closing, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
_ = server.Shutdown(closing)
|
|
||||||
}()
|
|
||||||
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Authenticator says whether a request may act, and as whom.
|
|
||||||
//
|
|
||||||
// An interface so the check can be driven by a test without an identity provider, and so the one
|
|
||||||
// real implementation is the only thing that has to be right.
|
|
||||||
type Authenticator interface {
|
|
||||||
// Who returns the subject a request is acting as, or an error naming why it may not.
|
|
||||||
Who(r *http.Request) (string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// mustAuthenticate is the real one: a bearer token from the configured issuer.
|
|
||||||
//
|
|
||||||
// **Not yet verifying the signature**, and it says so rather than pretending. Verification needs
|
|
||||||
// the issuer's keys, which needs an identity provider to exist — so this refuses every request
|
|
||||||
// until that is built, which is the same answer as having no API at all and is honest about why.
|
|
||||||
func mustAuthenticate(issuer string) Authenticator { return notYet{issuer: issuer} }
|
|
||||||
|
|
||||||
type notYet struct{ issuer string }
|
|
||||||
|
|
||||||
func (n notYet) Who(*http.Request) (string, error) {
|
|
||||||
return "", fmt.Errorf(
|
|
||||||
"this mesh has no way to verify a token from %s yet: the identity provider is a module "+
|
|
||||||
"and none is running. Use the command line, which authenticates through nothing "+
|
|
||||||
"because it is already behind the machine's own login", n.issuer)
|
|
||||||
}
|
|
||||||
|
|
||||||
// commands is the routing, separate so a test can drive it without a listener.
|
|
||||||
func commands(who Authenticator) http.Handler {
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
|
|
||||||
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
|
||||||
return assign(ctx, open, in.Node, splitModules(in.Module)...)
|
|
||||||
}))
|
|
||||||
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
|
||||||
return unassign(ctx, open, in.Node, splitModules(in.Module)...)
|
|
||||||
}))
|
|
||||||
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
|
||||||
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
|
||||||
return take(ctx, open, in.Node, in.Module, takeOptions{Yes: in.Yes, Digest: in.Digest})
|
|
||||||
}))
|
|
||||||
mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
|
||||||
return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter)
|
|
||||||
}))
|
|
||||||
mux.HandleFunc("POST /adopt", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
|
||||||
return adopt(ctx, open, in.Node)
|
|
||||||
}))
|
|
||||||
|
|
||||||
// Anything else is said plainly, because a command surface answering 404 to a verb somebody
|
|
||||||
// expected is indistinguishable from one that is down.
|
|
||||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
refuse(w, http.StatusNotFound, fmt.Errorf(
|
|
||||||
"%s %s is not something this mesh can be asked; it accepts POST /assign, "+
|
|
||||||
"POST /unassign, POST /take, POST /converge and POST /adopt", r.Method, r.URL.Path))
|
|
||||||
})
|
|
||||||
return mux
|
|
||||||
}
|
|
||||||
|
|
||||||
type request struct {
|
|
||||||
Node string `json:"node"`
|
|
||||||
Module string `json:"module"`
|
|
||||||
// Yes, Digest and Filter are converge's and take's: do it rather than preview it, the digest
|
|
||||||
// of the preview it acts on, and (converge) which module loads the mesh's filter.
|
|
||||||
Yes bool `json:"yes,omitempty"`
|
|
||||||
Digest string `json:"digest,omitempty"`
|
|
||||||
Filter string `json:"filter,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// acting is the shape every route shares: authenticate, read, act, answer.
|
|
||||||
func acting(
|
|
||||||
who Authenticator,
|
|
||||||
needsModule bool,
|
|
||||||
do func(context.Context, *stores, request) (string, error),
|
|
||||||
) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if _, err := who.Who(r); err != nil {
|
|
||||||
refuse(w, http.StatusUnauthorized, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var in request
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
|
||||||
refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if in.Node == "" || (needsModule && in.Module == "") {
|
|
||||||
if needsModule {
|
|
||||||
refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`))
|
|
||||||
} else {
|
|
||||||
refuse(w, http.StatusBadRequest, errors.New(`"node" is needed`))
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
open, err := openStores(r.Context())
|
|
||||||
if err != nil {
|
|
||||||
refuse(w, http.StatusServiceUnavailable, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
|
|
||||||
said, err := do(r.Context(), open, in)
|
|
||||||
if err != nil {
|
|
||||||
// **The refusal the command line would have given, unchanged.** Carrying `said` with
|
|
||||||
// it matters: an assignment that was kept and still does not resolve is two facts,
|
|
||||||
// and dropping either makes the answer wrong.
|
|
||||||
answer(w, http.StatusConflict, map[string]any{"said": said, "refused": err.Error()})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
answer(w, http.StatusOK, map[string]any{"said": said})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func answer(w http.ResponseWriter, status int, body map[string]any) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(status)
|
|
||||||
_ = json.NewEncoder(w).Encode(body)
|
|
||||||
}
|
|
||||||
|
|
||||||
func refuse(w http.ResponseWriter, status int, err error) {
|
|
||||||
answer(w, status, map[string]any{"refused": err.Error()})
|
|
||||||
}
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
type letIn struct{}
|
|
||||||
|
|
||||||
func (letIn) Who(*http.Request) (string, error) { return "somebody", nil }
|
|
||||||
|
|
||||||
func asking(t *testing.T, who Authenticator, method, path, body string) *httptest.ResponseRecorder {
|
|
||||||
t.Helper()
|
|
||||||
recorded := httptest.NewRecorder()
|
|
||||||
commands(who).ServeHTTP(recorded, httptest.NewRequest(method, path, strings.NewReader(body)))
|
|
||||||
return recorded
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Nothing is served to somebody the mesh cannot identify**, and that is the default rather than
|
|
||||||
// a setting. The board this stands behind is published on a public name (novox/hq 11-a-board), so
|
|
||||||
// an unauthenticated command surface is authority over the mesh handed to whoever finds it.
|
|
||||||
func TestAnUnauthenticatedRequestIsRefused(t *testing.T) {
|
|
||||||
got := asking(t, mustAuthenticate("https://identity.example/realms/mesh"),
|
|
||||||
"POST", "/assign", `{"node":"anchor","module":"umami"}`)
|
|
||||||
if got.Code != http.StatusUnauthorized {
|
|
||||||
t.Fatalf("an unidentified caller got %d", got.Code)
|
|
||||||
}
|
|
||||||
// And it says what to do instead, because the answer is not "give up".
|
|
||||||
if !strings.Contains(got.Body.String(), "command line") {
|
|
||||||
t.Errorf("the refusal does not say what still works: %s", got.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The API refuses to start at all without being told whose tokens to believe.
|
|
||||||
//
|
|
||||||
// At start rather than per request, so it is found by whoever ran it rather than by whoever
|
|
||||||
// finds it.
|
|
||||||
func TestTheApiWillNotRunWithoutAnIssuer(t *testing.T) {
|
|
||||||
err := apiCommand(context.Background(), []string{})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("it served commands without being told who may give them")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "issuer") {
|
|
||||||
t.Errorf("the refusal does not name what is missing: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A request missing what it acts on is refused before anything is opened.
|
|
||||||
func TestARequestThatNamesNothingIsRefused(t *testing.T) {
|
|
||||||
// **The exact refusal, not merely "not accepted".** Asserting non-200 passes even when the
|
|
||||||
// request got as far as opening a store and failing there, which proves nothing about whether
|
|
||||||
// anything was checked — that is what the first version of this test did.
|
|
||||||
for _, body := range []string{`{}`, `{"node":"anchor"}`, `{"module":"umami"}`, `not json`} {
|
|
||||||
got := asking(t, letIn{}, "POST", "/assign", body)
|
|
||||||
if got.Code != http.StatusBadRequest {
|
|
||||||
t.Errorf("%s got %d, and a request naming nothing is a bad request", body, got.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A verb nobody implemented is said plainly. A command surface answering 404 to something
|
|
||||||
// somebody expected is indistinguishable from one that is down.
|
|
||||||
func TestAnUnknownRouteSaysWhatIsAccepted(t *testing.T) {
|
|
||||||
got := asking(t, letIn{}, "POST", "/rotate", `{}`)
|
|
||||||
if got.Code != http.StatusNotFound {
|
|
||||||
t.Fatalf("got %d", got.Code)
|
|
||||||
}
|
|
||||||
var said map[string]any
|
|
||||||
_ = json.Unmarshal(got.Body.Bytes(), &said)
|
|
||||||
if !strings.Contains(said["refused"].(string), "/assign") {
|
|
||||||
t.Errorf("it does not say what it does accept: %v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,164 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
|
|
||||||
// for longer than one apply interval (novox/hq issue 233).
|
|
||||||
const kindApplyFailing = "apply-failing"
|
|
||||||
|
|
||||||
// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
|
|
||||||
// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
|
|
||||||
const applyEvery = 5 * time.Minute
|
|
||||||
|
|
||||||
// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
|
|
||||||
// (mesh-host's probe.go): "the package manager (pacman)".
|
|
||||||
const packageManagerKey = "the package manager ("
|
|
||||||
|
|
||||||
// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
|
|
||||||
// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
|
|
||||||
// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
|
|
||||||
// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
|
|
||||||
// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
|
|
||||||
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
|
|
||||||
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
|
|
||||||
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
|
|
||||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
|
||||||
heard := map[string]bool{}
|
|
||||||
for _, m := range f.machines {
|
|
||||||
if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) {
|
|
||||||
heard[m.name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var out []conditions.Observation
|
|
||||||
said := map[string]bool{}
|
|
||||||
for _, d := range f.doing {
|
|
||||||
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, applyFailing(d, f.now))
|
|
||||||
said[d.Node] = true
|
|
||||||
}
|
|
||||||
// **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's
|
|
||||||
// newest word, judged by the same bound — the client says a failure once, and fills again on every change —
|
|
||||||
// on a machine whose node-engine applied what it was sent.
|
|
||||||
failed := map[string][]inventory.ProvReport{}
|
|
||||||
for _, r := range f.provFailed {
|
|
||||||
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
failed[r.Node] = append(failed[r.Node], r)
|
|
||||||
}
|
|
||||||
for _, node := range sortedNames(failed) {
|
|
||||||
out = append(out, provisioningFailing(node, failed[node], f.now))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
|
|
||||||
// place.
|
|
||||||
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
|
|
||||||
sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
|
|
||||||
var names, words []string
|
|
||||||
since := failed[0].At
|
|
||||||
for _, r := range failed {
|
|
||||||
names = append(names, r.Module)
|
|
||||||
words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
|
|
||||||
if r.At.Before(since) {
|
|
||||||
since = r.At
|
|
||||||
}
|
|
||||||
}
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
|
||||||
Severity: conditions.Warning, Said: strings.Join(words, "; "),
|
|
||||||
Resolved: m + " applies what it is sent again"}
|
|
||||||
o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m,
|
|
||||||
strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery))
|
|
||||||
o.Headline = m + " is not applying its update"
|
|
||||||
o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+
|
|
||||||
"details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since)))
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
func orNoReason(s string) string {
|
|
||||||
if s == "" {
|
|
||||||
return "no reason given"
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyFailing is one machine's S21 condition.
|
|
||||||
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
|
|
||||||
m := d.Node
|
|
||||||
since := d.Since.UTC().Format("2006-01-02 15:04 MST")
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
|
||||||
Severity: conditions.Warning, Said: doingWords(d),
|
|
||||||
Resolved: m + " applies what it is sent again"}
|
|
||||||
how := "refused its declaration whole"
|
|
||||||
if d.Outcome != inventory.OutcomeRefused {
|
|
||||||
how = "applied its declaration only in part"
|
|
||||||
}
|
|
||||||
o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
|
|
||||||
o.Headline = m + " is not applying its update"
|
|
||||||
o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
|
|
||||||
m, plainHow(d), humanDuration(now.Sub(*d.Since)))
|
|
||||||
if tool := unansweringTool(d); tool != "" {
|
|
||||||
o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
|
|
||||||
"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
|
|
||||||
o.Headline = fmt.Sprintf("%s cannot install packages", m)
|
|
||||||
o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
|
|
||||||
"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
|
|
||||||
"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
|
|
||||||
}
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// plainHow is how a machine failed, as the operator reads it.
|
|
||||||
func plainHow(d inventory.Doing) string {
|
|
||||||
if d.Outcome == inventory.OutcomeRefused {
|
|
||||||
return "refused every update"
|
|
||||||
}
|
|
||||||
return "applied its updates only in part"
|
|
||||||
}
|
|
||||||
|
|
||||||
// unansweringTool is the package manager a machine said does not answer, or "".
|
|
||||||
func unansweringTool(d inventory.Doing) string {
|
|
||||||
for _, r := range d.Failed {
|
|
||||||
if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
|
|
||||||
return strings.TrimSuffix(rest, ")")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// failedNames is what failed, by name: the refusal, or the failed resources' ids.
|
|
||||||
func failedNames(d inventory.Doing) string {
|
|
||||||
if d.Outcome == inventory.OutcomeRefused {
|
|
||||||
return "refused"
|
|
||||||
}
|
|
||||||
ids := make([]string, 0, len(d.Failed))
|
|
||||||
for _, r := range d.Failed {
|
|
||||||
ids = append(ids, r.ID)
|
|
||||||
}
|
|
||||||
sort.Strings(ids)
|
|
||||||
return "failed " + strings.Join(ids, ", ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
|
|
||||||
func doingWords(d inventory.Doing) string {
|
|
||||||
if d.Outcome == inventory.OutcomeRefused {
|
|
||||||
return "refused: " + oneLine(d.Refused)
|
|
||||||
}
|
|
||||||
var parts []string
|
|
||||||
for _, r := range d.Failed {
|
|
||||||
parts = append(parts, r.ID+": "+oneLine(r.Error))
|
|
||||||
}
|
|
||||||
sort.Strings(parts)
|
|
||||||
return strings.Join(parts, "; ")
|
|
||||||
}
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
|
|
||||||
// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
|
|
||||||
// the same failure on times reports since for before now.
|
|
||||||
func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
|
|
||||||
since := now.Add(-forHowLong)
|
|
||||||
return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
|
|
||||||
Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
|
|
||||||
{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
|
|
||||||
"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
|
|
||||||
"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
|
|
||||||
{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
|
|
||||||
"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
|
|
||||||
"file /etc/pacman.conf could not be read: No such file or directory"}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
|
|
||||||
// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
|
|
||||||
func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
|
|
||||||
got := watchApplyFailing(f)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
|
|
||||||
}
|
|
||||||
o := got[0]
|
|
||||||
if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
|
|
||||||
t.Errorf("the condition is not the machine's: %s", o.Key())
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
|
|
||||||
t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
|
|
||||||
}
|
|
||||||
if strings.Contains(o.Summary, "/etc/") {
|
|
||||||
t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Said, "could not be read") {
|
|
||||||
t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
|
|
||||||
t.Errorf("the plain words do not say it: %+v", o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
|
|
||||||
// node-engine applied the rest.
|
|
||||||
func TestARefusalThatGoesOnIsRaised(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
|
||||||
since := now.Add(-time.Hour)
|
|
||||||
f := calm(now)
|
|
||||||
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
|
|
||||||
Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
|
|
||||||
got := watchApplyFailing(f)
|
|
||||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
|
|
||||||
!strings.Contains(got[0].Said, "pacman does not answer") {
|
|
||||||
t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
|
|
||||||
// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
|
|
||||||
// younger is not yet; and a silent or sleeping machine is S1's.
|
|
||||||
func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
|
||||||
f := calm(now)
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
|
|
||||||
if got := watchApplyFailing(f); len(got) != 1 {
|
|
||||||
t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
|
|
||||||
}
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
|
|
||||||
if got := watchApplyFailing(f); len(got) != 0 {
|
|
||||||
t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
|
|
||||||
}
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
|
||||||
f.machines[0].lastHeard = now.Add(-time.Hour)
|
|
||||||
if got := watchApplyFailing(f); len(got) != 0 {
|
|
||||||
t.Errorf("a silent machine was raised for its last report: %+v", got)
|
|
||||||
}
|
|
||||||
f = calm(now)
|
|
||||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
|
||||||
f.doing[0].Node = "a-machine-the-mesh-does-not-know"
|
|
||||||
if got := watchApplyFailing(f); len(got) != 0 {
|
|
||||||
t.Errorf("a machine not heard of was raised: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
|
|
||||||
// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
|
|
||||||
// is cleared through the keeper on the next tick.
|
|
||||||
func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
||||||
|
|
||||||
failing := calm(now)
|
|
||||||
failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
|
|
||||||
w.see(t.Context(), failing)
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
|
|
||||||
t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
clean := calm(now.Add(5 * time.Minute))
|
|
||||||
clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
|
|
||||||
w.see(t.Context(), clean)
|
|
||||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
||||||
t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ask calls one of a module's tools, through the control plane's own broker connection.
|
|
||||||
//
|
|
||||||
// A module serves tools under an account scoped to exactly that (novox/hq ADR 0047), and nothing
|
|
||||||
// else in the mesh held an account that could ask one — not an operator at a terminal, not an agent
|
|
||||||
// acting for one (novox/hq 04-ISSUES/049). The control plane does, so it is the way in: one
|
|
||||||
// process, one connection, one place a question can be seen to have been asked (ADR 0095).
|
|
||||||
func askCommand(ctx context.Context, args []string) error {
|
|
||||||
positionals, flags := split(args)
|
|
||||||
set := flag.NewFlagSet("ask", flag.ContinueOnError)
|
|
||||||
wait := set.Duration("wait", 60*time.Second, "how long to wait for the module's answer")
|
|
||||||
if err := set.Parse(flags); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(positionals) < 2 || len(positionals) > 3 {
|
|
||||||
return errors.New("ask <module> <tool> [json arguments] [--wait 60s]")
|
|
||||||
}
|
|
||||||
module, tool := positionals[0], positionals[1]
|
|
||||||
var arguments json.RawMessage
|
|
||||||
if len(positionals) == 3 {
|
|
||||||
if !json.Valid([]byte(positionals[2])) {
|
|
||||||
return fmt.Errorf("the arguments are not JSON: %s", positionals[2])
|
|
||||||
}
|
|
||||||
arguments = json.RawMessage(positionals[2])
|
|
||||||
}
|
|
||||||
|
|
||||||
server, err := connectLink(ctx, nil, nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
answer, err := link.Ask(ctx, server.Bus(), module, tool, arguments, *wait)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// The answer as the module gave it, to standard output, for a person or a program. A tool
|
|
||||||
// that answered with an error has still answered: printed the same way, and the exit status
|
|
||||||
// says which.
|
|
||||||
body, err := json.Marshal(answer)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Fprintln(os.Stdout, string(body))
|
|
||||||
if answer.Error != "" {
|
|
||||||
return fmt.Errorf("%s.%s answered with an error: %s", module, tool, answer.Error)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// busWaitBound is how long a module whose ask the bus did not answer is asked again at every look before the plan fails
|
|
||||||
// (issue 1334).
|
|
||||||
const busWaitBound = 10 * time.Minute
|
|
||||||
|
|
||||||
// busWaiting says an open plan has a module of its current tier that the bus did not answer an ask of: the look must come back
|
|
||||||
// for it, as nothing else will when no build of the tier is in flight (issue 1334).
|
|
||||||
func busWaiting(p inventory.Plan) bool {
|
|
||||||
return len(busWaitingIn(p)) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaitingIn are the modules of the plan's current tier left unasked because the bus did not answer, sorted.
|
|
||||||
func busWaitingIn(p inventory.Plan) []string {
|
|
||||||
if !p.Open() || p.Tier >= len(p.Tiers) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var out []string
|
|
||||||
for _, m := range p.Tiers[p.Tier] {
|
|
||||||
if s := p.Modules[m]; s != nil && s.State == "" && s.BusWaitFrom != nil {
|
|
||||||
out = append(out, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaitWords is how a verb says modules are waiting for the bus: not asked yet, and asked again by the plan at its next look.
|
|
||||||
func busWaitWords(p inventory.Plan, modules []string) string {
|
|
||||||
return fmt.Sprintf("%s not asked yet: the bus did not answer; %s asks again at its next look", strings.Join(modules, ", "), p.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// slicesWithout is names without those in drop, in order.
|
|
||||||
func slicesWithout(names, drop []string) []string {
|
|
||||||
var out []string
|
|
||||||
for _, n := range names {
|
|
||||||
if !slices.Contains(drop, n) {
|
|
||||||
out = append(out, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A retry of a plan failed past the bus's bound, whose module the bus still does not answer, names that module as waiting:
|
|
||||||
// read while the plan still says failed, it was named nowhere, and the retry said "asked" with nothing after it (issue
|
|
||||||
// 1403, met again once the hand rebuild no longer resumed the plan first).
|
|
||||||
func TestARetryNamesTheModulesStillWaitingForTheBus(t *testing.T) {
|
|
||||||
since := time.Now().UTC()
|
|
||||||
p := inventory.Plan{ID: "plan-bus", State: inventory.PlanFailed, Tiers: [][]string{{"a"}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &since}}}
|
|
||||||
waiting := busWaitingOnRetry(p)
|
|
||||||
_, said := retryWords(p, []string{"a"}, nil, waiting)
|
|
||||||
if !strings.Contains(said, "a not asked yet: the bus did not answer") || strings.Contains(said, "asked ;") {
|
|
||||||
t.Fatalf("a retry left for the bus is said as %q", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,153 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// busAsker is an asker on a real bus's `asked` bucket, counting what it performs: two of them are two
|
|
||||||
// controllers sharing one record.
|
|
||||||
type busAskerRig struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
called int
|
|
||||||
acts int
|
|
||||||
open []conditions.Condition
|
|
||||||
sent [][]byte
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rig *busAskerRig) asker(t *testing.T, conn *nats.Conn, now time.Time) *asker {
|
|
||||||
return &asker{
|
|
||||||
open: func(context.Context) ([]conditions.Condition, error) {
|
|
||||||
rig.mu.Lock()
|
|
||||||
defer rig.mu.Unlock()
|
|
||||||
return rig.open, nil
|
|
||||||
},
|
|
||||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
|
||||||
store: busAsked{conn: conn},
|
|
||||||
publish: func(_ context.Context, subject string, body []byte, _ string) error {
|
|
||||||
rig.mu.Lock()
|
|
||||||
defer rig.mu.Unlock()
|
|
||||||
if subject == asks.AskSubject(askerName) {
|
|
||||||
rig.sent = append(rig.sent, body)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
call: func(context.Context, conditions.Action, map[string]string) error {
|
|
||||||
time.Sleep(20 * time.Millisecond) // long enough for the other delivery to arrive meanwhile
|
|
||||||
rig.mu.Lock()
|
|
||||||
defer rig.mu.Unlock()
|
|
||||||
rig.called++
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
record: func(context.Context, link.HandAct) error {
|
|
||||||
rig.mu.Lock()
|
|
||||||
defer rig.mu.Unlock()
|
|
||||||
rig.acts++
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
now: func() time.Time { return now },
|
|
||||||
logf: t.Logf,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func askedBus(t *testing.T) *nats.Conn {
|
|
||||||
t.Helper()
|
|
||||||
conn, err := nats.Connect(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(conn.Close)
|
|
||||||
js, err := jetstream.New(conn)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: broker.AskedBucket}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return conn
|
|
||||||
}
|
|
||||||
|
|
||||||
// The review of 2026-10-09 (L7): two deliveries of one warrant, to two controllers at once, perform its act
|
|
||||||
// exactly once and record it once — the record's compare-and-set decides, never the warrant's message id.
|
|
||||||
func TestTwoAnswersAtOnceActOnce(t *testing.T) {
|
|
||||||
conn := askedBus(t)
|
|
||||||
now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)
|
|
||||||
rig := &busAskerRig{open: []conditions.Condition{heldCondition()}}
|
|
||||||
first, second := rig.asker(t, conn, now), rig.asker(t, conn, now)
|
|
||||||
if err := first.reconcile(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(rig.sent) != 1 {
|
|
||||||
t.Fatalf("asked %d times", len(rig.sent))
|
|
||||||
}
|
|
||||||
var q asks.Ask
|
|
||||||
_ = json.Unmarshal(rig.sent[0], &q)
|
|
||||||
release, _ := q.Option("release")
|
|
||||||
w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID,
|
|
||||||
Label: release.Label, Level: release.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
|
||||||
AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
||||||
body, _ := json.Marshal(w)
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, a := range []*asker{first, second, first, second} {
|
|
||||||
wg.Add(1)
|
|
||||||
go func(a *asker) {
|
|
||||||
defer wg.Done()
|
|
||||||
if err := a.Decided(context.Background(), body); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}(a)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
if rig.called != 1 || rig.acts != 1 {
|
|
||||||
t.Fatalf("performed %d time(s), recorded %d time(s)", rig.called, rig.acts)
|
|
||||||
}
|
|
||||||
got, err := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
|
||||||
if err != nil || got == nil || got.Acted != "done" {
|
|
||||||
t.Fatalf("kept as %+v (%v)", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The review of 2026-10-09 (L2): a write decided on a record read earlier never lands over one made since. A
|
|
||||||
// cancel read before the answer was acted on leaves the act's record as it is.
|
|
||||||
func TestAStaleCancelDoesNotWriteOverAnAct(t *testing.T) {
|
|
||||||
conn := askedBus(t)
|
|
||||||
now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC)
|
|
||||||
rig := &busAskerRig{open: []conditions.Condition{heldCondition()}}
|
|
||||||
a := rig.asker(t, conn, now)
|
|
||||||
if err := a.reconcile(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var q asks.Ask
|
|
||||||
_ = json.Unmarshal(rig.sent[0], &q)
|
|
||||||
stale, _ := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
|
||||||
release, _ := q.Option("release")
|
|
||||||
w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID,
|
|
||||||
Label: release.Label, Level: release.Level, Channel: "telegram", At: now, AskDigest: q.Digest(),
|
|
||||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
||||||
body, _ := json.Marshal(w)
|
|
||||||
if err := a.Decided(context.Background(), body); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.cancel(context.Background(), *stale, "the condition ended"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, _ := busAsked{conn: conn}.Get(context.Background(), q.ID)
|
|
||||||
if got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
|
||||||
t.Errorf("a stale cancel wrote over the act: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own
|
|
||||||
// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1).
|
|
||||||
func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
asksRecorded(t)
|
|
||||||
planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
|
|
||||||
twoTiers(t, open)
|
|
||||||
before := time.Now().UTC().Add(-time.Hour)
|
|
||||||
ended := before.Add(time.Minute)
|
|
||||||
failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
|
||||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
|
||||||
Note: "a failed to build in tier 0",
|
|
||||||
Delivery: &inventory.PlanDelivery{},
|
|
||||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
|
||||||
Why: "killed by hand"}},
|
|
||||||
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
|
|
||||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
conn, err := nats.Connect(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
// The controller's seat answers `plans` as it does: by running the verb.
|
|
||||||
sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) {
|
|
||||||
var args map[string]string
|
|
||||||
_ = json.Unmarshal(m.Data, &args)
|
|
||||||
answer := link.Answer{Result: json.RawMessage(`"ok"`)}
|
|
||||||
if len(args) != 1 || args["retry"] == "" {
|
|
||||||
answer = link.Answer{Error: "the retry was asked with more than the plan"}
|
|
||||||
} else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil {
|
|
||||||
answer = link.Answer{Error: err.Error()}
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(answer)
|
|
||||||
_ = m.Respond(body)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = sub.Unsubscribe() }()
|
|
||||||
|
|
||||||
// What S23 raises for it, read from the store as the watchdog reads it.
|
|
||||||
raised := func(context.Context) ([]conditions.Condition, error) {
|
|
||||||
facts, err := gatherFailedPlans(ctx, open.inventory, time.Now())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []conditions.Condition
|
|
||||||
for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) {
|
|
||||||
out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
|
||||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions})
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
store := memAskedStore{}
|
|
||||||
var sent []published
|
|
||||||
var acts []link.HandAct
|
|
||||||
a := &asker{open: raised, store: store,
|
|
||||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
|
||||||
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
|
||||||
sent = append(sent, published{subject, id, body})
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
call: callAction(conn),
|
|
||||||
record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil },
|
|
||||||
now: func() time.Time { return now },
|
|
||||||
logf: t.Logf}
|
|
||||||
cs, _ := raised(ctx)
|
|
||||||
if len(cs) != 1 || len(cs[0].Actions) != 1 {
|
|
||||||
t.Fatalf("the failed plan raised %+v", cs)
|
|
||||||
}
|
|
||||||
if err := a.reconcile(ctx); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var w asks.Warrant
|
|
||||||
for _, r := range store {
|
|
||||||
for _, o := range r.Ask.Options {
|
|
||||||
if o.Label == "Retry" {
|
|
||||||
w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen,
|
|
||||||
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
|
||||||
AskDigest: r.Ask.Digest(),
|
|
||||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w.Ask == "" {
|
|
||||||
t.Fatalf("no ask offers Retry: %+v", store)
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(w)
|
|
||||||
if err := a.Decided(ctx, body); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.Decided(ctx, body); err != nil { // heard again: nothing more
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
m := p.Modules["a"]
|
|
||||||
if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || !m.Key || m.Why != "waits for the builder's key" || m.Build == "build-1" {
|
|
||||||
t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
|
|
||||||
}
|
|
||||||
if got := store[w.Ask]; got.Acted != "done" {
|
|
||||||
t.Fatalf("the ask ended %q", got.Acted)
|
|
||||||
}
|
|
||||||
if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) {
|
|
||||||
t.Fatalf("the warrant was recorded as %+v", acts)
|
|
||||||
}
|
|
||||||
if cs, _ = raised(ctx); len(cs) != 0 {
|
|
||||||
t.Fatalf("the plan-failed condition did not end: %+v", cs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,612 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
// The asker on the bus: its asks in the controller's bucket `asked`, its asks and cancels published on the
|
|
||||||
// seat under the controller's name, the verbs a warrant chooses called with the controller's grant, and the
|
|
||||||
// router's record of its asks read under its name (novox/hq ADR 0259).
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// askerFrom is the serving controller's asker; nil in any other process.
|
|
||||||
var askerFrom *asker
|
|
||||||
|
|
||||||
// askWithin is how long a verb a warrant chose is given to answer.
|
|
||||||
const askWithin = time.Minute
|
|
||||||
|
|
||||||
type busAsked struct{ conn *nats.Conn }
|
|
||||||
|
|
||||||
func (b busAsked) kv(ctx context.Context) (jetstream.KeyValue, error) {
|
|
||||||
js, err := jetstream.New(b.conn)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return js.KeyValue(ctx, broker.AskedBucket)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b busAsked) Get(ctx context.Context, id string) (*asked, error) {
|
|
||||||
kv, err := b.kv(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
e, err := kv.Get(ctx, id)
|
|
||||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var r asked
|
|
||||||
return &r, json.Unmarshal(e.Value(), &r)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create keeps a new ask under its id, and only where none is kept: never over another.
|
|
||||||
func (b busAsked) Create(ctx context.Context, r asked) error {
|
|
||||||
kv, err := b.kv(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
body, err := json.Marshal(r)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = kv.Create(ctx, r.ID, body)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Change applies change to the ask kept under id by compare-and-set on its key's revision (the review of
|
|
||||||
// 2026-10-09, L2): read, changed, and written only over the revision read; when another write came between,
|
|
||||||
// read again and asked again, at most askChangeTries times. change says whether to write at all.
|
|
||||||
func (b busAsked) Change(ctx context.Context, id string, change func(*asked) bool) (bool, error) {
|
|
||||||
kv, err := b.kv(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
for try := 0; try < askChangeTries; try++ {
|
|
||||||
e, err := kv.Get(ctx, id)
|
|
||||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
var r asked
|
|
||||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if !change(&r) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
body, err := json.Marshal(r)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if _, err := kv.Update(ctx, id, body, e.Revision()); err != nil {
|
|
||||||
var api *jetstream.APIError
|
|
||||||
if errors.Is(err, jetstream.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
return false, fmt.Errorf("the ask %s changed under every one of %d tries", id, askChangeTries)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b busAsked) All(ctx context.Context) ([]asked, error) {
|
|
||||||
kv, err := b.kv(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
lister, err := kv.ListKeys(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer func() { _ = lister.Stop() }()
|
|
||||||
var out []asked
|
|
||||||
for k := range lister.Keys() {
|
|
||||||
e, err := kv.Get(ctx, k)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var r asked
|
|
||||||
if json.Unmarshal(e.Value(), &r) == nil {
|
|
||||||
out = append(out, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// verbRefused is a verb's own refusal of an act: the seat answered, and said no. Any other error of callAction (a bus
|
|
||||||
// that did not answer, a grant that is refused) is not one: the verb may have run.
|
|
||||||
type verbRefused struct{ verb, words string }
|
|
||||||
|
|
||||||
func (e verbRefused) Error() string { return e.verb + " refused: " + e.words }
|
|
||||||
|
|
||||||
// callAction performs an action's verb as the controller, through the grant that names it.
|
|
||||||
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
|
||||||
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
|
||||||
seat, verb, ok := strings.Cut(a.Verb, ".")
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("%q names no seat and verb", a.Verb)
|
|
||||||
}
|
|
||||||
body := map[string]any{}
|
|
||||||
for k, v := range args {
|
|
||||||
body[k] = v
|
|
||||||
}
|
|
||||||
if seat == catalogue.DeliverySeat {
|
|
||||||
_, err := askDeliveryOwner(ctx, conn, verb, body)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
granted := false
|
|
||||||
for _, v := range broker.VerbsTheControllerActsOnAWarrant {
|
|
||||||
granted = granted || (v.Seat == seat && v.Verb == verb)
|
|
||||||
}
|
|
||||||
if !granted {
|
|
||||||
return fmt.Errorf("%s: %w", a.Verb, errNotGranted)
|
|
||||||
}
|
|
||||||
var answer link.Answer
|
|
||||||
var err error
|
|
||||||
if a.Machine != "" {
|
|
||||||
answer, err = link.AskSeatTool(ctx, conn, seat, verb, a.Machine, body, askWithin)
|
|
||||||
} else {
|
|
||||||
answer, err = link.AskMeshSeatTool(ctx, conn, seat, verb, body, askWithin)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if answer.Error != "" {
|
|
||||||
return verbRefused{verb: a.Verb, words: answer.Error}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tellThrough tells the operator one plain notice through the operator channel's `tell`, over the controller's own
|
|
||||||
// connection and grant (novox/hq ADR 0380); a holder's refusal is an error naming it.
|
|
||||||
func tellThrough(conn *nats.Conn) func(ctx context.Context, headline string, lines []string) error {
|
|
||||||
return func(ctx context.Context, headline string, lines []string) error {
|
|
||||||
answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell",
|
|
||||||
map[string]any{"headline": headline, "lines": lines}, tellWithin)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if answer.Error != "" {
|
|
||||||
return fmt.Errorf("operator-channel.tell refused: %s", answer.Error)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// routerRecordOf reads the router's record of one of the controller's asks, under its name, and answers
|
|
||||||
// how it ended when it did: the bucket is the one the asks seat's declarer names as its records.
|
|
||||||
func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.Context, id string) (*asks.Warrant, error) {
|
|
||||||
return func(ctx context.Context, id string) (*asks.Warrant, error) {
|
|
||||||
bucket, err := asksRecords(ctx, inv)
|
|
||||||
if err != nil || bucket == "" {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+askerName+"."+id, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if reply.Header.Get("Status") != "" {
|
|
||||||
return nil, nil // none, or not readable: the event says it
|
|
||||||
}
|
|
||||||
var rec struct {
|
|
||||||
State string `json:"state"`
|
|
||||||
Warrant *asks.Warrant `json:"warrant"`
|
|
||||||
}
|
|
||||||
if json.Unmarshal(reply.Data, &rec) != nil || rec.State == "open" || rec.Warrant == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return rec.Warrant, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
|
||||||
func asksRecords(ctx context.Context, inv catalogueReader) (string, error) {
|
|
||||||
declared, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
for _, m := range declared {
|
|
||||||
for _, s := range m.DefinesSeats {
|
|
||||||
if s.Name == broker.AsksSeat && len(s.Records) > 0 {
|
|
||||||
return broker.BucketName(m.Module, s.Records[0]), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// routerHereIn says whether a module declaring the asks seat, with its ask named by its caller, is assigned:
|
|
||||||
// without it nothing takes an ask, and asking would only fill a queue nobody reads.
|
|
||||||
func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, error) {
|
|
||||||
return func(ctx context.Context) (bool, error) {
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
for _, s := range e.Manifest.DefinesSeats {
|
|
||||||
if s.Name == broker.AsksSeat && s.NamedByCaller("ask") && len(e.On) > 0 {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353), judged on the two
|
|
||||||
// facts that make it so, and failing closed on each:
|
|
||||||
//
|
|
||||||
// 1. **composed**: the user list the mesh composes now grants the controller itself to publish its ask
|
|
||||||
// (controllerMayAsk) — the controller's own grant, not whether any user of the list changed;
|
|
||||||
// 2. **in force**: the bus's own module, on the machine holding the bus, says the user list there allows it
|
|
||||||
// and the server reloaded after that list was written (the nats module's `nats_user_can`, `in_force`) — a
|
|
||||||
// list sent is not a list the bus enforces until it reloads.
|
|
||||||
//
|
|
||||||
// Anything that cannot be worked out is not held, with why, so the conditions that need the operator are said
|
|
||||||
// as undelivered rather than asked into a refusal.
|
|
||||||
func grantHeldIn(open *stores, conn *nats.Conn) func(ctx context.Context) (bool, string, error) {
|
|
||||||
return grantHeldBy(composedGrantIn(open), busGrantIn(open, conn), time.Now)
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantJudge is one of the two facts: whether it holds, and why not.
|
|
||||||
type grantJudge func(ctx context.Context) (bool, string, error)
|
|
||||||
|
|
||||||
// grantHeldBy is the judgement over the two facts, remembered for grantLookEvery: composing the list resolves
|
|
||||||
// the bus's machine whole, and asking the bus's module is a call.
|
|
||||||
func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx context.Context) (bool, string, error) {
|
|
||||||
var mu sync.Mutex
|
|
||||||
var at time.Time
|
|
||||||
var held bool
|
|
||||||
var why string
|
|
||||||
return func(ctx context.Context) (bool, string, error) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if !at.IsZero() && now().Sub(at) < grantLookEvery {
|
|
||||||
return held, why, nil
|
|
||||||
}
|
|
||||||
held, why = judgeGrant(ctx, composed, inForce)
|
|
||||||
at = now()
|
|
||||||
return held, why, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgeGrant is held only when both facts hold; an error is a fact not known, never a yes. A fact that holds
|
|
||||||
// with a caveat (a reload not confirmed) gives that caveat as the why of a yes.
|
|
||||||
func judgeGrant(ctx context.Context, composed, inForce grantJudge) (bool, string) {
|
|
||||||
var caveat []string
|
|
||||||
for _, fact := range []struct {
|
|
||||||
judge grantJudge
|
|
||||||
what string
|
|
||||||
}{{composed, "whether the mesh composes the controller a grant to ask"}, {inForce, "whether the bus enforces it"}} {
|
|
||||||
if fact.judge == nil {
|
|
||||||
return false, fact.what + " is not judged here"
|
|
||||||
}
|
|
||||||
ok, why, err := fact.judge(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, fact.what + " could not be worked out: " + err.Error()
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
return false, why
|
|
||||||
}
|
|
||||||
if why != "" {
|
|
||||||
caveat = append(caveat, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true, strings.Join(caveat, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// controllerMayAsk says whether the controller's own grant, as composed from these records, lets it publish its
|
|
||||||
// ask (ADR 0259 §3: composed while a router that takes asks under its asker's name is assigned).
|
|
||||||
func controllerMayAsk(records broker.Records) (bool, error) {
|
|
||||||
users, err := broker.Users(records)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
for _, u := range users {
|
|
||||||
if u.Kind != broker.KindController {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
perms, err := broker.PermissionsFor(u)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return broker.MayPublish(perms, asks.AskSubject(askerName)), nil
|
|
||||||
}
|
|
||||||
return false, errors.New("the composition holds no controller")
|
|
||||||
}
|
|
||||||
|
|
||||||
// composedGrantIn is the first fact, from the mesh's records.
|
|
||||||
func composedGrantIn(open *stores) grantJudge {
|
|
||||||
return func(ctx context.Context) (bool, string, error) {
|
|
||||||
if open == nil || open.inventory == nil {
|
|
||||||
return false, "", errors.New("the mesh's records are not open")
|
|
||||||
}
|
|
||||||
records, err := open.inventory.BusRecords(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, "", err
|
|
||||||
}
|
|
||||||
may, err := controllerMayAsk(records)
|
|
||||||
if err != nil || may {
|
|
||||||
return may, "", err
|
|
||||||
}
|
|
||||||
return false, "the mesh composes the controller no grant to ask: no router that takes asks under its " +
|
|
||||||
"asker's name is assigned", nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// busGrantIn is the second fact, asked of the bus's own module on the machine holding the bus, with the check
|
|
||||||
// main made before it (novox/hq issue 353, PR 186: is the user list sent to the bus's machine the one composed
|
|
||||||
// now) kept for the cases the module's word does not settle.
|
|
||||||
func busGrantIn(open *stores, conn *nats.Conn) grantJudge {
|
|
||||||
ask := func(ctx context.Context) (string, json.RawMessage, error) {
|
|
||||||
if open == nil || open.inventory == nil || conn == nil {
|
|
||||||
return "", nil, errors.New("the mesh's records or the bus are not open")
|
|
||||||
}
|
|
||||||
holders, err := seatHolders(ctx, open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
return "", nil, err
|
|
||||||
}
|
|
||||||
h, held := holders[theBrokerSeat]
|
|
||||||
if !held || h.Node == "" || h.Module == "" {
|
|
||||||
return "", nil, nil
|
|
||||||
}
|
|
||||||
a, err := link.AskModuleToolOn(ctx, conn, h.Module, busUserCanTool, h.Node, map[string]any{
|
|
||||||
"user": broker.ControllerName, "action": "publish", "subject": asks.AskSubject(askerName)}, 15*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
return h.Node, nil, err
|
|
||||||
}
|
|
||||||
if a.Error != "" {
|
|
||||||
return h.Node, nil, errors.New(a.Error)
|
|
||||||
}
|
|
||||||
return h.Node, a.Result, nil
|
|
||||||
}
|
|
||||||
list := func(ctx context.Context) (busListState, error) {
|
|
||||||
if open == nil || open.inventory == nil {
|
|
||||||
return busListState{}, errors.New("the mesh's records are not open")
|
|
||||||
}
|
|
||||||
s, err := brokerListState(ctx, open, nil)
|
|
||||||
if err != nil || s.Machine == "" || !s.Behind {
|
|
||||||
return s, err
|
|
||||||
}
|
|
||||||
step, err := busHeld(ctx, open.inventory, []string{s.Machine})
|
|
||||||
if err != nil {
|
|
||||||
return busListState{}, err
|
|
||||||
}
|
|
||||||
s.BusStep = step[s.Machine]
|
|
||||||
return s, nil
|
|
||||||
}
|
|
||||||
return busGrantBy(ask, list)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busListState is the user list on the bus's machine as the controller's record knows it: whether the list
|
|
||||||
// sent there is behind the one composed now, why it could not be composed, and the bus step a send there waits
|
|
||||||
// on (that machine cannot be pushed until a person's `bus upgrade`).
|
|
||||||
type busListState struct {
|
|
||||||
Machine string
|
|
||||||
Behind bool
|
|
||||||
Uncomposed error
|
|
||||||
BusStep string
|
|
||||||
}
|
|
||||||
|
|
||||||
// remedy is the one thing that sends the bus's machine the list composed now: the plan's own reason when it
|
|
||||||
// cannot be composed, the bus upgrade while sends there wait on it, and otherwise a push of that machine.
|
|
||||||
func (s busListState) remedy(node string) string {
|
|
||||||
if s.Machine != "" {
|
|
||||||
node = s.Machine
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case s.Uncomposed != nil:
|
|
||||||
return fmt.Sprintf("the user list for %s cannot be composed (%v); `plan %s` says what to fix first",
|
|
||||||
node, s.Uncomposed, node)
|
|
||||||
case s.BusStep != "":
|
|
||||||
return fmt.Sprintf("sends to %s wait on the bus's planned step, so a push will not carry it: %s", node, s.BusStep)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("`push %s` carries it", node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busGrantBy judges the bus's word, and names for each way it is not held the one thing that makes it held:
|
|
||||||
//
|
|
||||||
// - **in force**: held;
|
|
||||||
// - **allowed, from a module older than `in_force`**: held, with the caveat that the reload is not confirmed.
|
|
||||||
// The file allows the publish, and the bus reloads within about 5 s of a change to it, so the worst case is
|
|
||||||
// one ask refused in those seconds, which is asked again (issue 369). The list check is not consulted:
|
|
||||||
// while a bus step is pending the bus's machine cannot be pushed, so any change to the composed list would
|
|
||||||
// read as behind for good and silence every ask until a person's bus upgrade (the review of PR 188);
|
|
||||||
// - **allowed, written, not reloaded** (the module gives both times): wait about 5 s;
|
|
||||||
// - **allowed, in force not known** (the module could not read a time): what is not known, said;
|
|
||||||
// - **not allowed, or the module cannot answer**: the list check decides — the remedy that sends the list
|
|
||||||
// when it is behind (busListState.remedy), and, for a module that cannot answer, held while the list sent
|
|
||||||
// is current, as main judged it.
|
|
||||||
//
|
|
||||||
// ask answers the machine holding the bus and its module's answer ("" when nothing holds the bus); list is the
|
|
||||||
// record's word on the list sent to that machine.
|
|
||||||
func busGrantBy(ask func(ctx context.Context) (string, json.RawMessage, error),
|
|
||||||
list func(ctx context.Context) (busListState, error)) grantJudge {
|
|
||||||
return func(ctx context.Context) (bool, string, error) {
|
|
||||||
node, raw, askErr := ask(ctx)
|
|
||||||
if askErr == nil && node == "" {
|
|
||||||
return false, "no module holds the bus, so nothing says what it enforces", nil
|
|
||||||
}
|
|
||||||
var answer busAnswer
|
|
||||||
if askErr == nil {
|
|
||||||
if answer, askErr = readBusAnswer(raw); askErr != nil {
|
|
||||||
askErr = fmt.Errorf("its answer could not be read: %w", askErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if askErr != nil {
|
|
||||||
if node == "" {
|
|
||||||
node = "the bus's machine"
|
|
||||||
}
|
|
||||||
s, err := list(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, "", fmt.Errorf("the bus module on %s could not answer (%v), and whether the list sent "+
|
|
||||||
"there is the one composed now could not be worked out: %w", node, askErr, err)
|
|
||||||
}
|
|
||||||
if s.Behind {
|
|
||||||
return false, fmt.Sprintf("the bus module on %s could not answer (%v), and the user list sent there is "+
|
|
||||||
"behind what the mesh composes; %s", node, askErr, s.remedy(node)), nil
|
|
||||||
}
|
|
||||||
return true, fmt.Sprintf("the bus module on %s could not answer (%v); judged by the user list sent there, "+
|
|
||||||
"which is the one composed now", node, askErr), nil
|
|
||||||
}
|
|
||||||
notAllowed := func(older string) (bool, string, error) {
|
|
||||||
s, err := list(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return false, "", fmt.Errorf("the user list on %s does not allow the controller to ask%s, and whether "+
|
|
||||||
"the list sent there is the one composed now could not be worked out: %w", node, older, err)
|
|
||||||
}
|
|
||||||
if s.Behind {
|
|
||||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s: it is behind "+
|
|
||||||
"what the mesh composes; %s", node, older, s.remedy(node)), nil
|
|
||||||
}
|
|
||||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s, though the list "+
|
|
||||||
"sent there is the one composed now; %s again", node, older, s.remedy(node)), nil
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case answer.InForce == nil && answer.Allowed:
|
|
||||||
return true, fmt.Sprintf("the user list on %s allows the controller to ask, but its reload is not "+
|
|
||||||
"confirmed: the bus module there is older than the answer whether a grant is in force, and the bus "+
|
|
||||||
"upgrade (`bus upgrade`) carries one that confirms it", node), nil
|
|
||||||
case answer.InForce == nil:
|
|
||||||
return notAllowed(" (the bus module there is older than the answer whether a grant is in force)")
|
|
||||||
case !answer.Allowed:
|
|
||||||
return notAllowed("")
|
|
||||||
case *answer.InForce:
|
|
||||||
return true, "", nil
|
|
||||||
case answer.FileWritten != "" && answer.ServerLoaded != "":
|
|
||||||
return false, fmt.Sprintf("the bus on %s has the controller's grant to ask written (%s) but has not "+
|
|
||||||
"reloaded since (last load %s); it reloads in place within about 5 s, so nothing is to be done but wait",
|
|
||||||
node, answer.FileWritten, answer.ServerLoaded), nil
|
|
||||||
}
|
|
||||||
why := answer.Why
|
|
||||||
if why == "" {
|
|
||||||
why = "it gives no reason"
|
|
||||||
}
|
|
||||||
return false, fmt.Sprintf("the user list on %s allows the controller to ask, but whether the bus enforces it "+
|
|
||||||
"is not known: %s", node, why), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// busUserCanTool is the bus module's answer to whether a user may do something, and whether it is in force.
|
|
||||||
const busUserCanTool = "nats_user_can"
|
|
||||||
|
|
||||||
// busAnswer is the bus module's answer to busUserCanTool; InForce is nil from a module older than that field,
|
|
||||||
// and FileWritten and ServerLoaded are given only when it could read both times (mesh-catalog's nats InForce).
|
|
||||||
type busAnswer struct {
|
|
||||||
Allowed bool `json:"allowed"`
|
|
||||||
InForce *bool `json:"in_force"`
|
|
||||||
Why string `json:"why"`
|
|
||||||
FileWritten string `json:"file_written"`
|
|
||||||
ServerLoaded string `json:"server_loaded"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func readBusAnswer(raw json.RawMessage) (busAnswer, error) {
|
|
||||||
var answer busAnswer
|
|
||||||
err := json.Unmarshal(raw, &answer)
|
|
||||||
return answer, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
|
||||||
const grantLookEvery = 30 * time.Second
|
|
||||||
|
|
||||||
// channelsIn is what the channels are now, as a fingerprint: each module claiming a kind of the channel
|
|
||||||
// bench, where, promising what, and whether of its own account. An ask the router refused is asked again
|
|
||||||
// once this changes.
|
|
||||||
func channelsIn(inv *inventory.Inventory) func(ctx context.Context) string {
|
|
||||||
return func(ctx context.Context) string {
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
var parts []string
|
|
||||||
for _, e := range entries {
|
|
||||||
for _, c := range e.Manifest.Claims {
|
|
||||||
if c.Kind == "" || !catalogue.KindedBenches[c.Name] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
on := append([]string(nil), e.On...)
|
|
||||||
sort.Strings(on)
|
|
||||||
caps := append([]string(nil), c.Capabilities...)
|
|
||||||
sort.Strings(caps)
|
|
||||||
parts = append(parts, fmt.Sprintf("%s/%s=%s@%s[%s]own:%t", c.Name, c.Kind, e.Manifest.Module,
|
|
||||||
strings.Join(on, ","), strings.Join(caps, ","), e.Manifest.RunsAs != ""))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(parts)
|
|
||||||
return strings.Join(parts, ";")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// startAsking makes the serving controller's asker and hands it the router's words.
|
|
||||||
func startAsking(ctx context.Context, open *stores, server *link.Server, conn *nats.Conn, keeper *conditions.Keeper) {
|
|
||||||
js, err := jetstream.New(conn)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a := servingAsker(open, conn, js, keeper)
|
|
||||||
if err := server.Decides(a); err != nil {
|
|
||||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
askerFrom = a
|
|
||||||
go a.keep(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
|
||||||
// records, and the judgements it asks by — the bus's grant to ask among them (novox/hq issue 353).
|
|
||||||
func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper *conditions.Keeper) *asker {
|
|
||||||
return &asker{
|
|
||||||
open: keeper.Open,
|
|
||||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
|
||||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
store: busAsked{conn: conn},
|
|
||||||
publish: func(ctx context.Context, subject string, body []byte, id string) error {
|
|
||||||
_, err := js.Publish(ctx, subject, body, jetstream.WithMsgID(id))
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
call: callAction(conn),
|
|
||||||
// A proposed settings layer is set by this controller itself on the warrant (novox/hq ADR 0277).
|
|
||||||
setLayer: setLayerIn(open),
|
|
||||||
record: func(ctx context.Context, act link.HandAct) error {
|
|
||||||
_, err := link.RecordHandAct(ctx, conn, act)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
tell: tellThrough(conn),
|
|
||||||
routerRecord: routerRecordOf(conn, open.inventory),
|
|
||||||
routerHere: routerHereIn(open.inventory),
|
|
||||||
grantHeld: grantHeldIn(open, conn),
|
|
||||||
channels: channelsIn(open.inventory),
|
|
||||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
|
||||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
|
||||||
},
|
|
||||||
now: time.Now,
|
|
||||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,186 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aKeeper())
|
|
||||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
|
||||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
|
||||||
_, _, err := composeQuestion(ctx, open, "laptop", compose.Reading, false)
|
|
||||||
var notMade *catalogue.NotMadeError
|
|
||||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
|
||||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
|
||||||
}
|
|
||||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
|
||||||
foreseen, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false)
|
|
||||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
|
||||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
|
||||||
}
|
|
||||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
|
||||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
|
||||||
if got := d1(t, open); len(got) != 0 {
|
|
||||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
|
||||||
before := awaitingPushBound
|
|
||||||
awaitingPushBound = -time.Minute
|
|
||||||
t.Cleanup(func() { awaitingPushBound = before })
|
|
||||||
got := d1(t, open)
|
|
||||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
|
||||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
|
||||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pushed: the secret is made and D1 says nothing.
|
|
||||||
pushedBy(t, open, "laptop")
|
|
||||||
if got := d1(t, open); len(got) != 0 {
|
|
||||||
t.Fatalf("a pushed machine still raised %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aTalker())
|
|
||||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got := d1(t, open)
|
|
||||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
|
||||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
|
||||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
|
||||||
// uncomposable for that — the stand-in hides nothing.
|
|
||||||
register(t, open, aKeeper())
|
|
||||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
one, two := rivals()
|
|
||||||
register(t, open, one)
|
|
||||||
register(t, open, two)
|
|
||||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
|
||||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
|
||||||
keys := map[string]conditions.Severity{}
|
|
||||||
for _, o := range d1(t, open) {
|
|
||||||
keys[o.Key()] = o.Severity
|
|
||||||
}
|
|
||||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
|
||||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aKeeper())
|
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got := d1(t, open)
|
|
||||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
|
||||||
t.Fatalf("a given secret under an old key: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aKeeper())
|
|
||||||
pushedBy(t, open, "laptop")
|
|
||||||
sent := time.Now()
|
|
||||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
|
||||||
if err != nil || since.After(sent) {
|
|
||||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
|
||||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
|
||||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
// What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only
|
|
||||||
// when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown.
|
|
||||||
if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sends, err := readDeliveries(ctx, open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if sends["laptop"].settled("keeper", time.Now()) {
|
|
||||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
|
||||||
}
|
|
||||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
|
||||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
|
||||||
}
|
|
||||||
pushedBy(t, open, "laptop")
|
|
||||||
sends, err = readDeliveries(ctx, open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if sends["laptop"].settled("keeper", time.Now()) {
|
|
||||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
|
||||||
}
|
|
||||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
|
||||||
t.Fatal("pushed past the grace, a holder is not expected")
|
|
||||||
}
|
|
||||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
|
||||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
|
||||||
}
|
|
||||||
pushedAndApplied(t, open, "laptop")
|
|
||||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
|
||||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
|
||||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
|
||||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
|
||||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
|
||||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
|
||||||
|
|
||||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
|
||||||
func aKeeper() catalogue.Manifest {
|
|
||||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
|
||||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
|
||||||
Resources: []map[string]any{
|
|
||||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
|
||||||
// are made — without a bus to carry it.
|
|
||||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
|
||||||
t.Helper()
|
|
||||||
ctx := t.Context()
|
|
||||||
plan, settings, err := resolveNode(ctx, open, node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, err := declared.Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, err := open.inventory.NodeByName(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
digest := digestOf(body)
|
|
||||||
// A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was
|
|
||||||
// built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109).
|
|
||||||
builds := declared.Builds
|
|
||||||
if builds == nil {
|
|
||||||
current, err := open.inventory.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
builds = map[string]string{}
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
builds[m.Module] = current[m.Module].Commit
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return digest
|
|
||||||
}
|
|
||||||
|
|
||||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
|
||||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
|
||||||
t.Helper()
|
|
||||||
digest := pushedBy(t, open, node)
|
|
||||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
|
||||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// d1 runs D1 once.
|
|
||||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
|
||||||
t.Helper()
|
|
||||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
|
||||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
|
||||||
|
|
||||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
|
||||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
|
||||||
|
|
||||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
|
||||||
// refused on it, so D1 is too.
|
|
||||||
|
|
||||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
|
||||||
|
|
||||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
|
||||||
// it and had time to report — never between assign and push.
|
|
||||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
|
||||||
now := time.Now()
|
|
||||||
sent := now.Add(-time.Minute)
|
|
||||||
long := now.Add(-time.Hour)
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
send lastSend
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"never sent", lastSend{}, false},
|
|
||||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
|
||||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
|
||||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
|
||||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
|
||||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
|
||||||
}
|
|
||||||
for _, c := range cases {
|
|
||||||
if got := c.send.settled("keeper", now); got != c.want {
|
|
||||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
|
||||||
// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send
|
|
||||||
// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent
|
|
||||||
// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so.
|
|
||||||
|
|
||||||
// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine
|
|
||||||
// is away once presence says so). A machine
|
|
||||||
// never heard from has not joined and is not away.
|
|
||||||
func awayOf(machines []machineFacts, now time.Time) map[string]bool {
|
|
||||||
out := map[string]bool{}
|
|
||||||
for _, m := range machines {
|
|
||||||
if m.silent(now) {
|
|
||||||
out[m.name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and
|
|
||||||
// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did
|
|
||||||
// before: a machine is not passed on a reading that was not made.
|
|
||||||
var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) {
|
|
||||||
nodes, err := open.inventory.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
address, err := broker.BusAddress()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
js, err := broker.Dial(address)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
machines := make([]machineFacts, len(nodes))
|
|
||||||
for i, n := range nodes {
|
|
||||||
machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen}
|
|
||||||
}
|
|
||||||
if err := withPresence(ctx, js.Conn(), machines, now); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return awayOf(machines, now), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// rememberAway keeps on the plan the machines of running that are away: sorted, each once.
|
|
||||||
func rememberAway(p *inventory.Plan, running []string, away map[string]bool) {
|
|
||||||
for _, n := range running {
|
|
||||||
if away[n] && !slices.Contains(p.Away, n) {
|
|
||||||
p.Away = append(p.Away, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(p.Away)
|
|
||||||
}
|
|
||||||
|
|
||||||
// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and
|
|
||||||
// those it did not pass as away.
|
|
||||||
func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string {
|
|
||||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool {
|
|
||||||
if !slices.Contains(p.Away, n) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_, sent := s.Rest[n]
|
|
||||||
return !sent
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// awaySaying is what a plan says of the machines it passed as away, empty when it passed none.
|
|
||||||
func awaySaying(p inventory.Plan) string {
|
|
||||||
var said []string
|
|
||||||
for _, n := range p.Away {
|
|
||||||
said = append(said, n+" was away: sent when it is heard again")
|
|
||||||
}
|
|
||||||
return strings.Join(said, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot.
|
|
||||||
func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool {
|
|
||||||
away, err := judgeAway(ctx, open, now)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err)
|
|
||||||
return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above
|
|
||||||
}
|
|
||||||
return away
|
|
||||||
}
|
|
||||||
@@ -1,263 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it
|
|
||||||
// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard
|
|
||||||
// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's
|
|
||||||
// builds, so the send it gets when it is heard again carries them (AC3).
|
|
||||||
func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
|
|
||||||
// laptop is away as every tier is sent; anchor is heard.
|
|
||||||
wasAway := judgeAway
|
|
||||||
judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) {
|
|
||||||
return map[string]bool{"laptop": true}, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { judgeAway = wasAway })
|
|
||||||
|
|
||||||
js, err := broker.Dial(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
if err := broker.AssertMeshStreams(js); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := js.EnsureControllerBuckets(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
legacy, err := js.Conn().JetStream()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
|
||||||
t.Cleanup(func() { checkEvents = nil })
|
|
||||||
sentHeard := make(chan *nats.Msg, 32)
|
|
||||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
|
||||||
|
|
||||||
wasSaved := inventory.PlanSaved
|
|
||||||
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
|
||||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
|
||||||
|
|
||||||
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
|
||||||
var mu sync.Mutex
|
|
||||||
var sends [][]string
|
|
||||||
everyKeyDone(t) // the builds of the tiers: each module is taken from a done key as soon as the walk reaches it
|
|
||||||
// Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look
|
|
||||||
// goes on through tier 1's send and overwrites the note of the move before the test reads it (the old build ask
|
|
||||||
// was where the walk paused).
|
|
||||||
wholeKeys := openPlanKeys
|
|
||||||
var holdB atomic.Bool
|
|
||||||
holdB.Store(true)
|
|
||||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
|
||||||
all, done, err := wholeKeys()
|
|
||||||
return keysHolding{all, "b", &holdB, ""}, done, err
|
|
||||||
}
|
|
||||||
digests := 0
|
|
||||||
wasSend := sendRollout
|
|
||||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
sends = append(sends, append([]string(nil), names...))
|
|
||||||
current, err := open.inventory.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
builds := map[string]string{}
|
|
||||||
for _, m := range []string{"a", "b"} {
|
|
||||||
builds[m] = current[m].Commit
|
|
||||||
}
|
|
||||||
for _, node := range names {
|
|
||||||
digests++
|
|
||||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { sendRollout = wasSend })
|
|
||||||
|
|
||||||
register := func(m, commit string, asked time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
|
||||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
|
||||||
Asked: asked}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
record := func(id, m, commit string, at time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
|
||||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
|
||||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
old := time.Now().Add(-2 * time.Hour)
|
|
||||||
for _, m := range []string{"a", "b"} {
|
|
||||||
record("build-"+m+"-1", m, "c1", old)
|
|
||||||
register(m, "c1", old)
|
|
||||||
for _, n := range []string{"anchor", "laptop"} {
|
|
||||||
if _, err := inv.Assign(ctx, n, m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, n := range []string{"anchor", "laptop"} {
|
|
||||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
|
||||||
register("a", "c2", time.Now().Add(-time.Minute))
|
|
||||||
|
|
||||||
// A machine's report on what it was sent, heard as the controller hears it.
|
|
||||||
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
|
||||||
seq := int64(0)
|
|
||||||
reports := func(node string) {
|
|
||||||
t.Helper()
|
|
||||||
last, found, err := inv.SentTo(ctx, node)
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("%s was sent nothing: %v", node, err)
|
|
||||||
}
|
|
||||||
seq++
|
|
||||||
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
|
||||||
"report_sequence": seq})
|
|
||||||
var report link.Report
|
|
||||||
if err := json.Unmarshal(body, &report); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := listener.Heard(ctx, report); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
plan := func() inventory.Plan {
|
|
||||||
t.Helper()
|
|
||||||
p, err := inv.PlanByID(ctx, "plan-1160")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
built := time.Now().UTC()
|
|
||||||
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
|
||||||
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{
|
|
||||||
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
|
||||||
"b": {},
|
|
||||||
}}
|
|
||||||
first, stopFirst := context.WithCancel(ctx)
|
|
||||||
defer stopFirst()
|
|
||||||
waitFirst := startRolloutEvents(first, open, js)
|
|
||||||
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
|
||||||
t.Cleanup(func() { stopFirst(); waitFirst() })
|
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
eventually(t, "tier 0 sent to anchor alone", func() bool {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor"
|
|
||||||
})
|
|
||||||
reports("anchor")
|
|
||||||
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
|
|
||||||
var moved string
|
|
||||||
eventually(t, "tier 1 reached on anchor's report alone", func() bool {
|
|
||||||
if q := plan(); q.Tier == 1 {
|
|
||||||
moved = q.Note
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
})
|
|
||||||
if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
|
|
||||||
t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
|
|
||||||
}
|
|
||||||
// Tier 1's module is taken from its key once that is built, which the walk's look does by itself.
|
|
||||||
holdB.Store(false)
|
|
||||||
askToLook()
|
|
||||||
eventually(t, "tier 1 sent to anchor alone", func() bool {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
|
|
||||||
})
|
|
||||||
// 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again.
|
|
||||||
was, known, err := inv.SentBuilds(ctx, "laptop")
|
|
||||||
if err != nil || !known {
|
|
||||||
t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known)
|
|
||||||
}
|
|
||||||
current, err := inv.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if was["a"] == current["a"].Commit || was["a"] != "c1" {
|
|
||||||
t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit)
|
|
||||||
}
|
|
||||||
if got := plan().Away; strings.Join(got, ",") != "laptop" {
|
|
||||||
t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// keysHolding is the stand-in builder's keys but for one module, whose key is not built while held; only the key of
|
|
||||||
// the fingerprint fp when that is given.
|
|
||||||
type keysHolding struct {
|
|
||||||
planKeyReads
|
|
||||||
module string
|
|
||||||
held *atomic.Bool
|
|
||||||
fp string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (k keysHolding) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) {
|
|
||||||
if module == k.module && k.held.Load() && (k.fp == "" || fp == k.fp) {
|
|
||||||
return builderkeys.Key{Module: module, Fingerprint: fp}, nil
|
|
||||||
}
|
|
||||||
return k.planKeyReads.Build(ctx, module, fp)
|
|
||||||
}
|
|
||||||
|
|
||||||
// holdKeys has the stand-in builder's keys of these modules (at any commit) unbuilt until the returned flag is cleared,
|
|
||||||
// so a walk that took them stays on its tier as it did while its build ran. It restores what it replaced at the end of
|
|
||||||
// the test, and the flag is set until the test lowers it.
|
|
||||||
func holdKeys(t *testing.T, modules ...string) *atomic.Bool {
|
|
||||||
t.Helper()
|
|
||||||
held := &atomic.Bool{}
|
|
||||||
held.Store(true)
|
|
||||||
was := openPlanKeys
|
|
||||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
|
||||||
all, done, err := was()
|
|
||||||
var keys planKeyReads = all
|
|
||||||
for _, m := range modules {
|
|
||||||
keys = keysHolding{keys, m, held, ""}
|
|
||||||
}
|
|
||||||
return keys, done, err
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { openPlanKeys = was })
|
|
||||||
return held
|
|
||||||
}
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
|
||||||
// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so.
|
|
||||||
|
|
||||||
// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away.
|
|
||||||
func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) {
|
|
||||||
now := time.Now().UTC()
|
|
||||||
machines := []machineFacts{
|
|
||||||
{name: "heard", lastHeard: now.Add(-time.Minute)},
|
|
||||||
{name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)},
|
|
||||||
{name: "stuck", lastHeard: now, presence: presenceSilent},
|
|
||||||
{name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard},
|
|
||||||
{name: "new"},
|
|
||||||
}
|
|
||||||
got := awayOf(machines, now)
|
|
||||||
for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false,
|
|
||||||
"new": false} {
|
|
||||||
if got[node] != want {
|
|
||||||
t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once.
|
|
||||||
func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) {
|
|
||||||
p := &inventory.Plan{}
|
|
||||||
away := map[string]bool{"g14": true, "ace": true}
|
|
||||||
rememberAway(p, []string{"anchor", "g14"}, away)
|
|
||||||
rememberAway(p, []string{"g14", "ace"}, away)
|
|
||||||
if !slices.Equal(p.Away, []string{"ace", "g14"}) {
|
|
||||||
t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1161/AC2: the plan says it, where its other passed machines are said.
|
|
||||||
func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) {
|
|
||||||
say := awaySaying(inventory.Plan{Away: []string{"g14"}})
|
|
||||||
if !strings.Contains(say, "g14 was away: sent when it is heard again") {
|
|
||||||
t.Fatalf("1161/AC2: the plan says %q", say)
|
|
||||||
}
|
|
||||||
if got := awaySaying(inventory.Plan{}); got != "" {
|
|
||||||
t.Fatalf("1161/AC2: a plan that passed no machine says %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for.
|
|
||||||
func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) {
|
|
||||||
p := inventory.Plan{Away: []string{"g14"}}
|
|
||||||
none := &inventory.PlanModule{}
|
|
||||||
if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) {
|
|
||||||
t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got)
|
|
||||||
}
|
|
||||||
sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}}
|
|
||||||
if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) {
|
|
||||||
t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,312 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and
|
|
||||||
// answers `backed-up` with the last night it is told (novox/hq ADR 0404).
|
|
||||||
type aBackupSeatOnTheBus struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
nows []string
|
|
||||||
night map[string]any // module → lastNight
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) {
|
|
||||||
t.Helper()
|
|
||||||
reply := func(m *nats.Msg, result any) {
|
|
||||||
body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node})
|
|
||||||
_ = m.Respond(body)
|
|
||||||
}
|
|
||||||
now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) {
|
|
||||||
var a struct {
|
|
||||||
Module string `json:"module"`
|
|
||||||
}
|
|
||||||
_ = json.Unmarshal(m.Data, &a)
|
|
||||||
f.mu.Lock()
|
|
||||||
f.nows = append(f.nows, a.Module)
|
|
||||||
f.mu.Unlock()
|
|
||||||
reply(m, map[string]any{"started": a.Module})
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
var out []map[string]any
|
|
||||||
for module, n := range f.night {
|
|
||||||
out = append(out, map[string]any{"module": module, "lastNight": n})
|
|
||||||
}
|
|
||||||
reply(m, out)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() })
|
|
||||||
if err := conn.Flush(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *aBackupSeatOnTheBus) asks() int {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
return len(f.nows)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits
|
|
||||||
// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the
|
|
||||||
// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the
|
|
||||||
// backups again, and the plan goes on. One bus and one store for the scenarios.
|
|
||||||
func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
js, err := broker.Dial(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
before := handActConn
|
|
||||||
handActConn = js.Conn()
|
|
||||||
t.Cleanup(func() { handActConn = before })
|
|
||||||
|
|
||||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
|
||||||
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
|
|
||||||
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
|
|
||||||
`"resources":[{"id":"files","type":"directory"}]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
|
||||||
Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, node := range []string{"anchor", "laptop"} {
|
|
||||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, node, "notes"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}}
|
|
||||||
for node, s := range seats {
|
|
||||||
s.serve(t, js.Conn(), node)
|
|
||||||
}
|
|
||||||
// Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup.
|
|
||||||
wasWithin, wasEvery := backupsWithin, backupsPollEvery
|
|
||||||
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
|
|
||||||
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
|
|
||||||
|
|
||||||
newPlan := func(id string) {
|
|
||||||
t.Helper()
|
|
||||||
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}}
|
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
read := func(id string) inventory.Plan {
|
|
||||||
t.Helper()
|
|
||||||
p, err := inv.PlanByID(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Both machines are asked; while one has not answered, no build is asked.
|
|
||||||
newPlan("plan-one")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 {
|
|
||||||
t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked)
|
|
||||||
}
|
|
||||||
seats["anchor"].answers("notes", true, "")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending {
|
|
||||||
t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups)
|
|
||||||
}
|
|
||||||
seats["laptop"].answers("notes", true, "")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone {
|
|
||||||
t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups)
|
|
||||||
}
|
|
||||||
finish := read("plan-one")
|
|
||||||
finish.State = inventory.PlanDone
|
|
||||||
if err := inv.SavePlan(ctx, &finish); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised.
|
|
||||||
seats["anchor"].night, seats["laptop"].night = nil, nil
|
|
||||||
newPlan("plan-two")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
seats["anchor"].answers("notes", true, "")
|
|
||||||
seats["laptop"].answers("notes", false, "no space left on device")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
held := read("plan-two")
|
|
||||||
if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 {
|
|
||||||
t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked)
|
|
||||||
}
|
|
||||||
var failed *inventory.PlanBackup
|
|
||||||
for i := range held.Backups.Asked {
|
|
||||||
if held.Backups.Asked[i].State == inventory.BackupFailed {
|
|
||||||
failed = &held.Backups.Asked[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" ||
|
|
||||||
strings.Contains(held.Note, "tried again") {
|
|
||||||
t.Fatalf("the failed backup: %+v note=%q", failed, held.Note)
|
|
||||||
}
|
|
||||||
advancePlans(ctx, open) // not retried on the next look
|
|
||||||
if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 ||
|
|
||||||
seats["laptop"].asks() != 2 {
|
|
||||||
t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks())
|
|
||||||
}
|
|
||||||
said, err := probeBackupHeld(ctx, &doctor{open: open})
|
|
||||||
if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") {
|
|
||||||
t.Fatalf("the condition: %+v %v", said, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on.
|
|
||||||
seats["laptop"].answers("notes", false, "no space left on device")
|
|
||||||
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by",
|
|
||||||
"mesh-delivery for jochen"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil {
|
|
||||||
t.Fatal("a plan with no failed backup was asked again")
|
|
||||||
}
|
|
||||||
seats["laptop"].night = nil
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 {
|
|
||||||
t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks())
|
|
||||||
}
|
|
||||||
seats["laptop"].answers("notes", true, "")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
// One key taken, not two: plan-one and plan-two carry the same module at the same commit, so they share one key
|
|
||||||
// build id (key_takein.go keyBuildID), and a walk finds a key already taken (keyTakes.taken, plan_keys.go) and takes
|
|
||||||
// it in no more. So that the smaller count hides no plan that stopped, plan-two's module must be built from that key.
|
|
||||||
if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 1 ||
|
|
||||||
got.Modules["notes"].Build == "" || got.Modules["notes"].State == "failed" || got.Modules["notes"].Key && got.Modules["notes"].State == "asked" {
|
|
||||||
t.Fatalf("the plan did not go on: %+v module=%+v builds=%v", got.Backups, got.Modules["notes"], *asked)
|
|
||||||
}
|
|
||||||
if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 {
|
|
||||||
t.Fatalf("the condition stays: %+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Issue 1086: a plan cut from a merge asked its first tier at the cut, so a module keeping data was sent without the
|
|
||||||
// backup the plan promised. The cut now takes the backups first (askFirstTierAtCut, batches.go), and the walk's own
|
|
||||||
// step asks the tier once they are done.
|
|
||||||
func TestAPlanCutFromAMergeOfABackedUpModuleAsksItsBackupAndNotItsTierUntilItIsDone(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
settings := t.TempDir() + "/merge-window.json"
|
|
||||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv(mergeWindowFileVar, settings)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
js, err := broker.Dial(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
before := handActConn
|
|
||||||
handActConn = js.Conn()
|
|
||||||
t.Cleanup(func() { handActConn = before })
|
|
||||||
|
|
||||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
|
|
||||||
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
|
|
||||||
`"resources":[{"id":"files","type":"directory"}]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/notes", Seat: "git", Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, m := range []string{"keeper", "notes"} {
|
|
||||||
if _, err := assign(ctx, open, "anchor", m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
seat := &aBackupSeatOnTheBus{}
|
|
||||||
seat.serve(t, js.Conn(), "anchor")
|
|
||||||
wasWithin, wasEvery := backupsWithin, backupsPollEvery
|
|
||||||
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
|
|
||||||
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
|
|
||||||
|
|
||||||
// 1. A merge of notes is heard and its window closes: the batch is cut through cutBatch.
|
|
||||||
hear(t, open, repoMerge("notes", "c-notes", t0), t0.Add(time.Second))
|
|
||||||
cutAt(t, open, t0.Add(time.Second+90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("%d walk(s)", len(ws))
|
|
||||||
}
|
|
||||||
// 2. Right after the cut no tier is asked, the record has anchor/notes pending, the seat heard one `now`.
|
|
||||||
cut, err := inv.PlanByID(ctx, ws[0].ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(*asked) != 0 || askedInTier(cut) {
|
|
||||||
t.Fatalf("the first tier was asked at the cut: %v %+v", *asked, cut.Modules)
|
|
||||||
}
|
|
||||||
if cut.Backups == nil || cut.Backups.State != inventory.BackupPending || len(cut.Backups.Asked) != 1 ||
|
|
||||||
cut.Backups.Asked[0].Machine != "anchor" || cut.Backups.Asked[0].Module != "notes" {
|
|
||||||
t.Fatalf("the backups of the cut: %+v", cut.Backups)
|
|
||||||
}
|
|
||||||
if seat.asks() != 1 {
|
|
||||||
t.Fatalf("the seat heard %d `now`", seat.asks())
|
|
||||||
}
|
|
||||||
// 3. The backup answers done: the walk's own step asks the first tier, once, and asks no second backup.
|
|
||||||
seat.answers("notes", true, "")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
after, err := inv.PlanByID(ctx, ws[0].ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(*asked) != 1 || after.Backups.State != inventory.BackupDone || seat.asks() != 1 {
|
|
||||||
t.Fatalf("builds=%v backups=%+v asks=%d", *asked, after.Backups, seat.asks())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,727 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
func windowed(t *testing.T) *stores {
|
|
||||||
t.Helper()
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
settings := t.TempDir() + "/merge-window.json"
|
|
||||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv(mergeWindowFileVar, settings)
|
|
||||||
noGroups(t)
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
|
||||||
Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, m := range []struct{ module, repository, path string }{
|
|
||||||
{"app", "novox/mesh-catalog", "modules/app"},
|
|
||||||
{"notes", "novox/mesh-catalog", "modules/notes"},
|
|
||||||
{"one", "novox/one", ""},
|
|
||||||
{"two", "novox/two", ""},
|
|
||||||
{"three", "novox/three", ""},
|
|
||||||
} {
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
|
||||||
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
|
||||||
Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return open
|
|
||||||
}
|
|
||||||
|
|
||||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func cutAt(t *testing.T, open *stores, now time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
|
||||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
|
||||||
hear(t, open, claude, t0.Add(time.Second))
|
|
||||||
hear(t, open, dunst, t0.Add(18*time.Second))
|
|
||||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
|
||||||
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
|
||||||
t.Fatalf("cut before the window closed: %+v", ws)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 {
|
|
||||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
w := ws[0]
|
|
||||||
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
|
||||||
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
|
||||||
}
|
|
||||||
for _, m := range []string{"app", "notes"} {
|
|
||||||
if _, in := w.Modules[m]; !in {
|
|
||||||
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
|
||||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
|
||||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
|
||||||
}
|
|
||||||
// The cut takes its first tier from the builder's keys (a look takes them in later; none runs here), and says at
|
|
||||||
// which commit: the one the branch holds, which is the later merge the walk carries.
|
|
||||||
for _, m := range w.Tiers[w.Tier] {
|
|
||||||
if s := w.Modules[m]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != dunst.Commit {
|
|
||||||
t.Fatalf("the walk did not take %s from its key at the commit it carries (%s): %+v", m, dunst.Commit, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
|
||||||
hear(t, open, claude, t0.Add(5*time.Minute))
|
|
||||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
|
||||||
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/three")
|
|
||||||
asksRecorded(t)
|
|
||||||
for i, r := range []string{"one", "two", "three"} {
|
|
||||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
|
||||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 {
|
|
||||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
w := ws[0]
|
|
||||||
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
|
||||||
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
|
||||||
}
|
|
||||||
for _, r := range []string{"one", "two", "three"} {
|
|
||||||
if w.CommitOf("novox/"+r) != "c-"+r {
|
|
||||||
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
|
||||||
}
|
|
||||||
if _, in := w.Modules[r]; !in {
|
|
||||||
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Each module of the first tier is taken from its key at its own repository's commit (the cut asks no build).
|
|
||||||
for _, r := range w.Tiers[w.Tier] {
|
|
||||||
if s := w.Modules[r]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != "c-"+r {
|
|
||||||
t.Fatalf("%s was not taken from its key at its own commit: %+v", r, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/both", "feat/busy")
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
|
||||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(100*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
|
||||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(170*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
|
||||||
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
|
||||||
}
|
|
||||||
|
|
||||||
busy := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/both", "feat/busy")
|
|
||||||
asksWithPaths(t)
|
|
||||||
var last time.Time
|
|
||||||
for i := 0; i < 12; i++ {
|
|
||||||
last = t0.Add(time.Duration(i) * time.Minute)
|
|
||||||
repo := []string{"one", "two", "three"}[i%3]
|
|
||||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
|
||||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
|
||||||
if i != 10 {
|
|
||||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ws, _ := walks(t, busy)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/553b7191claude")
|
|
||||||
asksWithPaths(t)
|
|
||||||
var said []inventory.Plan
|
|
||||||
was := inventory.PlanSaved
|
|
||||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
|
||||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
|
||||||
now := time.Now().UTC()
|
|
||||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
|
||||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
|
||||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
|
||||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
|
||||||
hear(t, open, claude, now.Add(-19*time.Second))
|
|
||||||
hear(t, open, dunst, now.Add(-time.Second))
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
|
||||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
|
||||||
lines := strings.Split(out, "\n")
|
|
||||||
first := lines[0]
|
|
||||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
|
||||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
|
||||||
if !strings.Contains(first, want) {
|
|
||||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
|
||||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
|
||||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
|
||||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
|
||||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
|
||||||
}
|
|
||||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
|
||||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
|
||||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func captured(t *testing.T, run func() error) string {
|
|
||||||
t.Helper()
|
|
||||||
r, w, err := os.Pipe()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
was := os.Stdout
|
|
||||||
os.Stdout = w
|
|
||||||
runErr := run()
|
|
||||||
os.Stdout = was
|
|
||||||
_ = w.Close()
|
|
||||||
var b bytes.Buffer
|
|
||||||
_, _ = io.Copy(&b, r)
|
|
||||||
if runErr != nil {
|
|
||||||
t.Fatal(runErr)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || !ws[0].Open() {
|
|
||||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
|
||||||
}
|
|
||||||
first := ws[0]
|
|
||||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
|
||||||
cutAt(t, open, t0.Add(5*time.Minute))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
|
||||||
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
|
||||||
t.Fatalf("the started walk was %s by the next merge", got.State)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
|
||||||
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
|
||||||
}
|
|
||||||
first.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(6*time.Minute))
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
|
||||||
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || !ws[0].Waiting() {
|
|
||||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
|
||||||
}
|
|
||||||
waiting := ws[0]
|
|
||||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(bs) != 0 || len(ws) != 2 {
|
|
||||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
newer := ws[0]
|
|
||||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
|
||||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
|
||||||
}
|
|
||||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" {
|
|
||||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
|
||||||
}
|
|
||||||
if _, in := newer.Modules["one"]; !in {
|
|
||||||
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
|
||||||
hear(t, open, later, t0.Add(18*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
w := ws[0]
|
|
||||||
w.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
|
||||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
|
||||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
|
||||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
|
||||||
Carried: later.Commit}) {
|
|
||||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
// Moves app, which that walk never built: the next batch walks it.
|
|
||||||
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
|
||||||
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
|
||||||
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
|
||||||
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
|
||||||
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
|
||||||
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
|
||||||
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
failed := ws[0]
|
|
||||||
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
|
||||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(3*time.Minute))
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
|
||||||
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
|
||||||
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
alone := ws[0]
|
|
||||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
|
||||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
|
||||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
|
||||||
}
|
|
||||||
// The cut takes the first tier from the builder's keys and asks no build, so nothing is recorded as asked: the
|
|
||||||
// module is taken from its key at the commit the merge was walked on.
|
|
||||||
if app := alone.Modules["app"]; app == nil || !app.Key || app.State != "asked" || app.KeyAt != middle.Commit {
|
|
||||||
t.Fatalf("the merge walked alone does not build app at its own commit: %v", alone.Modules)
|
|
||||||
}
|
|
||||||
// Retried, the failed walk would name merges the search answers now.
|
|
||||||
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
|
||||||
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
|
||||||
}
|
|
||||||
alone.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(4*time.Minute))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if ws[0].ID != alone.ID || len(bs) != 0 {
|
|
||||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
|
||||||
}
|
|
||||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
|
||||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
|
||||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
|
||||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
|
||||||
}
|
|
||||||
// A stopped walk starts no search: a person ended it.
|
|
||||||
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
|
||||||
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
|
||||||
cutAt(t, open, t0.Add(8*time.Minute))
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(9*time.Minute))
|
|
||||||
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
|
||||||
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
|
||||||
for _, grouped := range []bool{true, false} {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
for _, m := range []string{"mesh-host"} {
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
host := repoMerge("mesh-host", "h1", t0)
|
|
||||||
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
|
||||||
host.Head, ctl.Head = "feat/together", "feat/together"
|
|
||||||
if !grouped {
|
|
||||||
ctl.Head = "feat/alone"
|
|
||||||
}
|
|
||||||
hear(t, open, ctl, t0.Add(2*time.Second))
|
|
||||||
hear(t, open, host, t0.Add(3*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("%d walks", len(ws))
|
|
||||||
}
|
|
||||||
tiers := ws[0].Tiers
|
|
||||||
if grouped {
|
|
||||||
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
|
||||||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
|
||||||
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
|
||||||
}
|
|
||||||
} else if len(tiers) != 1 {
|
|
||||||
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABatchSurvivesARestart(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/both")
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
|
||||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
|
||||||
|
|
||||||
// A new controller: nothing of the first one's but the store.
|
|
||||||
again, err := openStores(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(again.Close)
|
|
||||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
|
||||||
cutAt(t, again, t0.Add(119*time.Second))
|
|
||||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
cutAt(t, again, t0.Add(120*time.Second))
|
|
||||||
ws, bs := walks(t, again)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
|
||||||
hear(t, open, c2, t0)
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
w := ws[0]
|
|
||||||
w.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
|
||||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
|
||||||
Commit: "c1", Carried: "c2"}) {
|
|
||||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk of two that built only two: a late merge of one is not its to name.
|
|
||||||
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
|
||||||
cutAt(t, open, t0.Add(18*time.Minute))
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
w = ws[0]
|
|
||||||
delete(w.Modules, "two")
|
|
||||||
w.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
|
||||||
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
|
||||||
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
|
||||||
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asked := asksWithPaths(t)
|
|
||||||
// The started walk's key (one at c1) is not built, so that walk stays started as it did while its build ran: with
|
|
||||||
// the stand-in builder's key done at once, one step would carry it to done and nothing would be beside the other.
|
|
||||||
wholeKeys, held := openPlanKeys, &atomic.Bool{}
|
|
||||||
held.Store(true)
|
|
||||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
|
||||||
all, done, err := wholeKeys()
|
|
||||||
return keysHolding{all, "one", held, fpOfModuleAt("one", "c1")}, done, err
|
|
||||||
}
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
started, _ := walks(t, open)
|
|
||||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0,
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}},
|
|
||||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
|
||||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
|
||||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
|
||||||
}
|
|
||||||
before := len(*asked)
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
|
||||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
|
||||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
|
||||||
}
|
|
||||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
|
||||||
later := time.Now().Add(time.Hour)
|
|
||||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
|
||||||
t.Fatalf("a deferred walk reads %q", line)
|
|
||||||
}
|
|
||||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, f := range facts {
|
|
||||||
if f.id == got.ID {
|
|
||||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Read again: the steps since wrote the started walk (what its key waits for), so the copy read at its cut is stale.
|
|
||||||
done, err := open.inventory.PlanByID(ctx, started[0].ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
done.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
if len(*asked) != before+1 {
|
|
||||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
|
||||||
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
|
||||||
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
var ownBatch, catalogueBatch inventory.Plan
|
|
||||||
for _, b := range bs {
|
|
||||||
if b.OwnPath() {
|
|
||||||
ownBatch = b
|
|
||||||
} else {
|
|
||||||
catalogueBatch = b
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
|
||||||
len(ownBatch.Delivery.Merges) != 1 {
|
|
||||||
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
|
||||||
}
|
|
||||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
|
||||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
|
||||||
}
|
|
||||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
|
||||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
|
||||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
|
||||||
}
|
|
||||||
ownWalk := ws[0]
|
|
||||||
for _, m := range []string{"app", "notes"} {
|
|
||||||
if _, in := ownWalk.Modules[m]; in {
|
|
||||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
|
||||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
|
||||||
}
|
|
||||||
ownWalk.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(3*time.Minute))
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
|
||||||
}
|
|
||||||
for _, m := range []string{"app", "notes"} {
|
|
||||||
if _, in := ws[0].Modules[m]; !in {
|
|
||||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, a := range *asked {
|
|
||||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
|
||||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
|
||||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
|
||||||
catalogueWalk := ws[0]
|
|
||||||
// The controller's second walk keeps its key unbuilt, so it stays started as it did while its build ran; with the
|
|
||||||
// stand-in builder's key done at once one step would carry it to done and nothing would be beside the catalogue walk.
|
|
||||||
wholeKeys, held := openPlanKeys, &atomic.Bool{}
|
|
||||||
held.Store(true)
|
|
||||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
|
||||||
all, done, err := wholeKeys()
|
|
||||||
return keysHolding{all, "mesh-controller", held, fpOfModuleAt("mesh-controller", "k2")}, done, err
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
|
||||||
cutAt(t, open, t0.Add(6*time.Minute))
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
|
||||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
|
||||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
|
||||||
}
|
|
||||||
if _, in := ws[0].Modules["app"]; in {
|
|
||||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
|
||||||
}
|
|
||||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
before := len(*asked)
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
if len(*asked) != before {
|
|
||||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
|
||||||
}
|
|
||||||
// Read again: the step above wrote the walk (what its key waits for), so the copy read at its cut is stale.
|
|
||||||
own2, err := open.inventory.PlanByID(ctx, ws[0].ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
own2.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
|
||||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
for _, m := range []struct {
|
|
||||||
module string
|
|
||||||
claims []catalogue.Claim
|
|
||||||
}{
|
|
||||||
{"nats", nil},
|
|
||||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
|
||||||
} {
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
|
||||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
|
||||||
ModuleDirsSaid: true}
|
|
||||||
hear(t, open, mixed, t0.Add(time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || ws[0].Waiting() {
|
|
||||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
|
||||||
}
|
|
||||||
done := ws[0]
|
|
||||||
done.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
|
||||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
|
||||||
got.Delivery.Merges, bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,293 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
|
|
||||||
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
|
|
||||||
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
|
|
||||||
func init() {
|
|
||||||
mergeWindowDefault = 0
|
|
||||||
// No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change
|
|
||||||
// (delivery_edges.go). A test that wants some replaces these (forgeHolding).
|
|
||||||
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
|
|
||||||
return catalogue.Manifest{}, false, nil
|
|
||||||
}
|
|
||||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
|
||||||
}
|
|
||||||
|
|
||||||
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
|
|
||||||
// settings file says them, and
|
|
||||||
// modules built from the catalogue (app, notes) and from three repositories of their own.
|
|
||||||
|
|
||||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
|
||||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
|
||||||
|
|
||||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull
|
|
||||||
// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group
|
|
||||||
// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test
|
|
||||||
// whose merges wait for the window as a group's says so (groupsWaiting).
|
|
||||||
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
|
||||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit,
|
|
||||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
|
||||||
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge).
|
|
||||||
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
|
||||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit,
|
|
||||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// noGroups stubs mesh-delivery's word on groups for a test: no head names a group.
|
|
||||||
func noGroups(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil }
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named
|
|
||||||
// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and
|
|
||||||
// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds
|
|
||||||
// its batch open however long it takes (ADR 0418 clause 3), is groupStates'.
|
|
||||||
func groupsWaiting(t *testing.T, heads ...string) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
|
|
||||||
if slices.Contains(heads, head) {
|
|
||||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
|
||||||
}
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
|
|
||||||
// before novox/hq ADR 0304: mesh-delivery cannot say any head's group.
|
|
||||||
func everyHeadWaits(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
|
||||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the
|
|
||||||
// members merge.
|
|
||||||
func groupStates(t *testing.T, head string, states map[string]string) {
|
|
||||||
t.Helper()
|
|
||||||
groupStatesFrom(t, head, states, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the
|
|
||||||
// composed check's: only the first holds the group's batch open for its last member.
|
|
||||||
func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) {
|
|
||||||
if asked != head {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
var members []groupMember
|
|
||||||
for repository, state := range states {
|
|
||||||
members = append(members, groupMember{Repository: repository, State: state, Composed: composed})
|
|
||||||
}
|
|
||||||
return members, true, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group,
|
|
||||||
// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share.
|
|
||||||
func fromHead(m link.SourceMoved, head string) link.SourceMoved {
|
|
||||||
m.Head = head
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is
|
|
||||||
// related to a batch assembling (ADR 0304 decision 1).
|
|
||||||
func lone(m link.SourceMoved) link.SourceMoved {
|
|
||||||
m.Head = ""
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// hear is a merge heard at a moment.
|
|
||||||
|
|
||||||
// cutAt is the cutter looking at a moment.
|
|
||||||
|
|
||||||
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
|
|
||||||
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
|
||||||
t.Helper()
|
|
||||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, p := range recent {
|
|
||||||
if p.Batch() {
|
|
||||||
batches = append(batches, p)
|
|
||||||
} else {
|
|
||||||
walks = append(walks, p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return walks, batches
|
|
||||||
}
|
|
||||||
|
|
||||||
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
|
|
||||||
// later commit, which names both merges — the earlier carried by the later.
|
|
||||||
|
|
||||||
// Merges of three repositories in one window are one walk, one commit for each.
|
|
||||||
|
|
||||||
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
|
|
||||||
// closes at most ten minutes after its first merge however busy.
|
|
||||||
|
|
||||||
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
|
|
||||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
|
||||||
|
|
||||||
// captured is what a command printed.
|
|
||||||
|
|
||||||
// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is
|
|
||||||
// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4).
|
|
||||||
|
|
||||||
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
|
|
||||||
// names the walk that took it over.
|
|
||||||
|
|
||||||
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
|
|
||||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
|
||||||
|
|
||||||
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
|
|
||||||
// first; the first delivered answers the older ones, and the search stops.
|
|
||||||
|
|
||||||
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
|
|
||||||
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
|
|
||||||
|
|
||||||
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
|
|
||||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
|
||||||
|
|
||||||
// The window's settings reach the controller in its settings file, read at every look; one that says no
|
|
||||||
// duration, or no file, is the default.
|
|
||||||
func TestTheWindowIsTheControllersSetting(t *testing.T) {
|
|
||||||
file := t.TempDir() + "/merge-window.json"
|
|
||||||
t.Setenv(mergeWindowFileVar, file)
|
|
||||||
for _, c := range []struct {
|
|
||||||
content string
|
|
||||||
window, atMost time.Duration
|
|
||||||
}{
|
|
||||||
{`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute},
|
|
||||||
{`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault},
|
|
||||||
{`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault},
|
|
||||||
} {
|
|
||||||
if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if w, m := mergeWindowOf(); w != c.window || m != c.atMost {
|
|
||||||
t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Setenv(mergeWindowFileVar, file+".gone")
|
|
||||||
if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault {
|
|
||||||
t.Errorf("no file reads as %s and %s", w, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362).
|
|
||||||
func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
|
||||||
f := calm(t0)
|
|
||||||
f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery",
|
|
||||||
since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"},
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}}
|
|
||||||
got := watchWaits(f)
|
|
||||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") ||
|
|
||||||
!strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") {
|
|
||||||
t.Fatalf("the waiting walk does not name both merges: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
|
|
||||||
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
|
|
||||||
// nothing, and the merge joins the next batch.
|
|
||||||
|
|
||||||
// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to
|
|
||||||
// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before.
|
|
||||||
|
|
||||||
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
|
|
||||||
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
|
||||||
ev := func(commit string, paths, removed []string) inventory.BatchedMerge {
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed}
|
|
||||||
b, _ := json.Marshal(m)
|
|
||||||
return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b,
|
|
||||||
Merged: t0.Add(time.Duration(len(commit)) * time.Second)}
|
|
||||||
}
|
|
||||||
got := combinedMerges([]inventory.BatchedMerge{
|
|
||||||
ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}),
|
|
||||||
ev("bb", []string{"x/module.json"}, nil),
|
|
||||||
ev("ccc", []string{"z.go"}, nil)})
|
|
||||||
if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) ||
|
|
||||||
len(got[0].Paths) != 3 {
|
|
||||||
t.Fatalf("combined as %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
|
||||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
|
||||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
|
||||||
// still waits for the word, so no catalogue delivery skips its turn.
|
|
||||||
|
|
||||||
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
|
||||||
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
|
||||||
// request is named by its commit, and a title is cut at fifty runes.
|
|
||||||
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
|
|
||||||
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
|
|
||||||
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
|
|
||||||
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
|
|
||||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
|
|
||||||
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
|
|
||||||
running := func(module string) []string {
|
|
||||||
if module == "messenger" || module == "telegram" {
|
|
||||||
return []string{"novox"}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if got, want := planLineOn(p, now, tierAtLeast, running),
|
|
||||||
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
|
|
||||||
t.Fatalf("the line reads %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
|
|
||||||
t.Fatalf("without the machines the line reads %q", got)
|
|
||||||
}
|
|
||||||
old := p
|
|
||||||
old.Commits, old.Delivery = nil, nil
|
|
||||||
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
|
|
||||||
t.Fatalf("a record naming no pull request reads %q", got)
|
|
||||||
}
|
|
||||||
long := p
|
|
||||||
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
|
|
||||||
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
|
|
||||||
t.Fatalf("a long title is not cut at fifty runes: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
|
|
||||||
func sameMerge(a, b inventory.PlanMerge) bool {
|
|
||||||
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
|
|
||||||
}
|
|
||||||
|
|
||||||
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
|
||||||
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
|
||||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
|
||||||
@@ -1,153 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
|
|
||||||
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
|
|
||||||
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
|
|
||||||
|
|
||||||
// The kinds S18 and S19 raise.
|
|
||||||
const (
|
|
||||||
kindBatchNotCut = "batch-not-cut"
|
|
||||||
kindBatchBehindWalk = "batch-behind-walk"
|
|
||||||
// kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after
|
|
||||||
// its first (novox/hq ADR 0418 clause 3).
|
|
||||||
kindGroupNotMerged = "group-not-merged"
|
|
||||||
)
|
|
||||||
|
|
||||||
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
|
||||||
type batchFacts struct {
|
|
||||||
id, state, grouped string
|
|
||||||
// atMost is when its window closes at the latest; closed when it closed.
|
|
||||||
atMost, closed time.Time
|
|
||||||
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
|
|
||||||
// tiers.
|
|
||||||
behind string
|
|
||||||
walkBound time.Duration
|
|
||||||
// holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first
|
|
||||||
// merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting).
|
|
||||||
holds map[string]string
|
|
||||||
created time.Time
|
|
||||||
atMostWindow time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
|
||||||
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
|
|
||||||
batches, err := inv.Batches(ctx)
|
|
||||||
if err != nil || len(batches) == 0 {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
bounds, err := measuredTierBounds(ctx, inv, now)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []batchFacts
|
|
||||||
for _, b := range batches {
|
|
||||||
_, atMostWindow := mergeWindowOf()
|
|
||||||
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow}
|
|
||||||
if w := b.Delivery; w != nil && w.Batch != nil {
|
|
||||||
f.holds = w.Batch.Holds
|
|
||||||
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
|
||||||
if f.atMost.Before(f.closed) {
|
|
||||||
f.closed = f.atMost
|
|
||||||
}
|
|
||||||
if w.Batch.CutAtOnce && !b.Created.IsZero() {
|
|
||||||
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
|
|
||||||
f.atMost, f.closed = b.Created, b.Created
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if f.behind != "" {
|
|
||||||
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
|
||||||
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out = append(out, f)
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
|
|
||||||
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, b := range f.batches {
|
|
||||||
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter ||
|
|
||||||
len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
late := f.now.Sub(b.atMost)
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
|
|
||||||
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
|
|
||||||
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
|
|
||||||
Headline: "Merged changes are not being delivered",
|
|
||||||
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
|
|
||||||
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
|
|
||||||
Resolved: "Merged changes are being delivered again"})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
|
|
||||||
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, b := range f.batches {
|
|
||||||
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
in := f.now.Sub(b.closed)
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
|
|
||||||
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
|
|
||||||
b.grouped, b.behind),
|
|
||||||
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
|
|
||||||
Headline: "Merged changes wait behind a slow delivery",
|
|
||||||
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
|
|
||||||
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
|
|
||||||
Resolved: "Merged changes no longer wait behind a slow delivery"})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge
|
|
||||||
// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of
|
|
||||||
// the group is delivered while it waits.
|
|
||||||
func watchGroupsNotMerged(f *signalFacts) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, b := range f.batches {
|
|
||||||
if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
heads := make([]string, 0, len(b.holds))
|
|
||||||
for h := range b.holds {
|
|
||||||
heads = append(heads, h)
|
|
||||||
}
|
|
||||||
sort.Strings(heads)
|
|
||||||
var why []string
|
|
||||||
for _, h := range heads {
|
|
||||||
why = append(why, b.holds[h])
|
|
||||||
}
|
|
||||||
waited := f.now.Sub(b.created)
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id,
|
|
||||||
Kind: kindGroupNotMerged, Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+
|
|
||||||
"of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id),
|
|
||||||
Said: "a group waits for a member that has not merged",
|
|
||||||
Headline: "A change waits for the rest of its group to merge",
|
|
||||||
Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+
|
|
||||||
"One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.",
|
|
||||||
humanDuration(waited)),
|
|
||||||
Resolved: "The group's last change merged"})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,117 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once
|
|
||||||
// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery.
|
|
||||||
func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
// Both modules' keys are not built, so each walk stays open as it did while its build ran: with the stand-in
|
|
||||||
// builder's key done at once, one step would carry a one-module walk to done.
|
|
||||||
wholeKeys, held := openPlanKeys, &atomic.Bool{}
|
|
||||||
held.Store(true)
|
|
||||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
|
||||||
all, done, err := wholeKeys()
|
|
||||||
return keysHolding{keysHolding{all, "one", held, ""}, "two", held, ""}, done, err
|
|
||||||
}
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() {
|
|
||||||
t.Fatalf("the first walk is not open: %+v", ws)
|
|
||||||
}
|
|
||||||
heard := t0.Add(100 * time.Second)
|
|
||||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 0 {
|
|
||||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
for _, w := range ws {
|
|
||||||
if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) {
|
|
||||||
t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(),
|
|
||||||
deferred(w), askedAny(w), w.Note)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is
|
|
||||||
// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973).
|
|
||||||
func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }()
|
|
||||||
heard := t0.Add(100 * time.Second)
|
|
||||||
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
|
||||||
hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 1 {
|
|
||||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
b := bs[0]
|
|
||||||
words := batchWords(b, heard)
|
|
||||||
if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" ||
|
|
||||||
!strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") {
|
|
||||||
t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words)
|
|
||||||
}
|
|
||||||
if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() {
|
|
||||||
t.Fatalf("the started walk was %s by the related merge", got.State)
|
|
||||||
}
|
|
||||||
first.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, heard.Add(2*time.Second))
|
|
||||||
if _, bs := walks(t, open); len(bs) != 0 {
|
|
||||||
t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever
|
|
||||||
// the two share (issue 973): it replaces the controller, so it goes alone.
|
|
||||||
func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
first := ws[0]
|
|
||||||
heard := t0.Add(100 * time.Second)
|
|
||||||
hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard)
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued ||
|
|
||||||
bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") {
|
|
||||||
t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
first.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, heard.Add(time.Second))
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
|
||||||
t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws)
|
|
||||||
}
|
|
||||||
own := ws[0]
|
|
||||||
// Unrelated to the own-path walk by every footprint, a merge still waits for it.
|
|
||||||
later := heard.Add(time.Minute)
|
|
||||||
hear(t, open, lone(repoMerge("two", "c2", later)), later)
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID {
|
|
||||||
t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A binding to data moves only by a person (novox/hq ADR 0232, issue 273).
|
|
||||||
//
|
|
||||||
// The resolver keeps each consumer of a provision that keeps its data at the provider it was last
|
|
||||||
// sent (catalogue/bound.go) and says what it would have moved. This is where that is said: on the
|
|
||||||
// push that composed it, and by the self-check's D12 every run, as an urgent condition naming the
|
|
||||||
// consumer, both providers and the pin that confirms the move.
|
|
||||||
|
|
||||||
// The condition kinds of D12.
|
|
||||||
const (
|
|
||||||
// kindBindingKept is a move the resolver refused: the consumer is still where its data is.
|
|
||||||
kindBindingKept = "binding-kept"
|
|
||||||
// kindBindingMoved is a consumer about to be sent another provider than the one on record with
|
|
||||||
// no pin naming it — what the resolver exists to make impossible, said if it ever is not.
|
|
||||||
kindBindingMoved = "binding-moved"
|
|
||||||
// kindBindingMoving is a move a pin asked for, not yet sent: a person's act, said so that the
|
|
||||||
// data is moved before the push that carries it.
|
|
||||||
kindBindingMoving = "binding-moving"
|
|
||||||
)
|
|
||||||
|
|
||||||
// probeBindingsID is the self-check's id for this probe, and the source of what it raises.
|
|
||||||
const probeBindingsID = "D12"
|
|
||||||
|
|
||||||
// keptObservation is the urgent condition for one refused move.
|
|
||||||
func keptObservation(k catalogue.KeptBinding) conditions.Observation {
|
|
||||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: bindingID(k.Machine, k.Consumer, k.Provision),
|
|
||||||
Token: kindBindingKept, Kind: kindBindingKept, Machine: k.Machine, Also: otherMachines(k.Machine, k.Bound.Node, k.Would.Node),
|
|
||||||
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
|
||||||
Summary: fmt.Sprintf("on %s, the mesh %s", k.Machine, k.String())}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bindingID(machine, consumer, provision string) string {
|
|
||||||
return machine + "." + consumer + "." + provision
|
|
||||||
}
|
|
||||||
|
|
||||||
func otherMachines(machine string, nodes ...string) []string {
|
|
||||||
var out []string
|
|
||||||
seen := map[string]bool{machine: true}
|
|
||||||
for _, n := range nodes {
|
|
||||||
if n != "" && !seen[n] {
|
|
||||||
seen[n] = true
|
|
||||||
out = append(out, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
|
|
||||||
// condition at once where this process keeps the conditions: a push is when a person is looking.
|
|
||||||
func reportKept(ctx context.Context, plan catalogue.Resolution) {
|
|
||||||
for _, k := range plan.Kept {
|
|
||||||
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
|
|
||||||
if conditionsFrom != nil {
|
|
||||||
o := keptObservation(k)
|
|
||||||
o.Source = probeBindingsID
|
|
||||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
|
||||||
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
|
|
||||||
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
|
|
||||||
// moves it (said, so the data goes first), and never anything else.
|
|
||||||
func probeBindings(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
||||||
// One reading of the mesh for every machine this looks at.
|
|
||||||
ctx = withInputCache(ctx)
|
|
||||||
inv := d.open.inventory
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, n := range nodes {
|
|
||||||
plan, _, err := resolveNode(ctx, d.open, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
if unresolvable(err) {
|
|
||||||
// D1 says it, with the binding that refused it when that is why.
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
|
||||||
}
|
|
||||||
bound, err := inv.BindingsFor(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
pins, err := inv.PinsFor(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out = append(out, bindingFindings(plan, bound, pins)...)
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// bindingFindings is what one machine's resolution says against its record.
|
|
||||||
func bindingFindings(plan catalogue.Resolution, bound map[string]map[string]catalogue.Chosen,
|
|
||||||
pins map[string]catalogue.Chosen) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, k := range plan.Kept {
|
|
||||||
out = append(out, keptObservation(k))
|
|
||||||
}
|
|
||||||
said := map[string]bool{}
|
|
||||||
for _, need := range plan.Needs {
|
|
||||||
if !need.KeepsData || need.ByRecord {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
was, recorded := bound[need.For][need.Name]
|
|
||||||
now := catalogue.Chosen{Node: need.From, Module: need.Module}
|
|
||||||
if !recorded || was == now || (was.Module == "" && was.Node == now.Node) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
id := bindingID(plan.Node, need.For, need.Name)
|
|
||||||
if said[id] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
said[id] = true
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Machine: plan.Node,
|
|
||||||
Also: otherMachines(plan.Node, was.Node, now.Node), Resolver: conditions.ResolverOperator}
|
|
||||||
if pin, pinned := pins[need.Name]; pinned && pin.Node == now.Node && (pin.Module == "" || pin.Module == now.Module) {
|
|
||||||
o.Token, o.Kind, o.Severity = kindBindingMoving, kindBindingMoving, conditions.Warning
|
|
||||||
o.Summary = fmt.Sprintf("on %s, %s's %s moves from %s to %s at the next push, by the pin — its data "+
|
|
||||||
"is on %s: move it first", plan.Node, need.For, need.Name, was, now, was)
|
|
||||||
} else {
|
|
||||||
o.Token, o.Kind, o.Severity = kindBindingMoved, kindBindingMoved, conditions.Urgent
|
|
||||||
o.Summary = fmt.Sprintf("on %s, %s's %s would be sent %s, and it is bound to %s, where its data is, "+
|
|
||||||
"with no pin naming %s — a move nothing asked for", plan.Node, need.For, need.Name, now, was, now)
|
|
||||||
}
|
|
||||||
out = append(out, o)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,102 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, m := range storeManifests() {
|
|
||||||
register(t, open, m)
|
|
||||||
}
|
|
||||||
assignAll := func(pairs ...[2]string) {
|
|
||||||
for _, a := range pairs {
|
|
||||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
|
||||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
|
|
||||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
|
||||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
|
||||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
|
||||||
[2]string{"laptop", "board"})
|
|
||||||
|
|
||||||
plan, _, err := resolveNode(ctx, open, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
|
|
||||||
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
|
|
||||||
}
|
|
||||||
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
|
|
||||||
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sent, and recorded: only the binding to data.
|
|
||||||
bindings := boundToData(plan, nil)
|
|
||||||
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
|
|
||||||
t.Fatalf("recorded %+v", bindings)
|
|
||||||
}
|
|
||||||
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
|
||||||
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The laptop's store taken away: refused, not moved to the anchor's empty one.
|
|
||||||
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := resolveNode(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
|
|
||||||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
|
|
||||||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
|
|
||||||
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
|
|
||||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
plan, _, err = resolveNode(ctx, open, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
|
||||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
|
||||||
}
|
|
||||||
found, err := probeBindings(ctx, &doctor{open: open})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
|
|
||||||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
|
|
||||||
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
|
|
||||||
}
|
|
||||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
all, err := inv.Bindings(ctx)
|
|
||||||
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
|
|
||||||
t.Fatalf("%+v, %v", all, err)
|
|
||||||
}
|
|
||||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
|
||||||
t.Fatalf("a move sent is still said: %+v, %v", found, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 273, ADR 0232: a consumer of a provision that keeps its data moves only by a pin.
|
|
||||||
|
|
||||||
func storeManifests() []catalogue.Manifest {
|
|
||||||
return []catalogue.Manifest{
|
|
||||||
{Module: "store", Version: "1",
|
|
||||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
|
||||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
|
||||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
|
||||||
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
|
|
||||||
{Module: "resolver", Version: "1",
|
|
||||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
|
||||||
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
|
|
||||||
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
|
|
||||||
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) catalogue.Needed {
|
|
||||||
t.Helper()
|
|
||||||
for _, n := range plan.Needs {
|
|
||||||
if n.For == consumer && n.Name == provision {
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("no %s for %s: %+v", provision, consumer, plan.Needs)
|
|
||||||
return catalogue.Needed{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
|
|
||||||
// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
|
|
||||||
// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
|
|
||||||
|
|
||||||
// What one machine's resolution says against its record.
|
|
||||||
func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
|
|
||||||
home, anchor := catalogue.Chosen{Node: "home", Module: "store"}, catalogue.Chosen{Node: "anchor", Module: "store"}
|
|
||||||
plan := catalogue.Resolution{Node: "laptop",
|
|
||||||
Kept: []catalogue.KeptBinding{{Machine: "laptop", Consumer: "board", Provision: "postgres-database",
|
|
||||||
Bound: home, Would: anchor}},
|
|
||||||
Needs: []catalogue.Needed{
|
|
||||||
{Name: "postgres-database", For: "board", From: "home", Module: "store", KeepsData: true},
|
|
||||||
{Name: "postgres-database", For: "game", From: "anchor", Module: "store", KeepsData: true},
|
|
||||||
{Name: "wildcard-resolution", For: "network", From: "anchor", Module: "resolver"},
|
|
||||||
}}
|
|
||||||
bound := map[string]map[string]catalogue.Chosen{
|
|
||||||
"board": {"postgres-database": home},
|
|
||||||
"game": {"postgres-database": home},
|
|
||||||
"network": {"wildcard-resolution": {Node: "home", Module: "resolver"}},
|
|
||||||
}
|
|
||||||
found := linted(bindingFindings(plan, bound, nil))
|
|
||||||
if len(found) != 2 {
|
|
||||||
t.Fatalf("found %+v", found)
|
|
||||||
}
|
|
||||||
kept, moved := found[0], found[1]
|
|
||||||
if kept.Kind != kindBindingKept || kept.Severity != conditions.Urgent || kept.Machine != "laptop" ||
|
|
||||||
!strings.Contains(kept.Summary, "would move board's postgres-database from home/store to anchor/store") ||
|
|
||||||
!strings.Contains(kept.Summary, "its data is on home/store") ||
|
|
||||||
!strings.Contains(kept.Summary, "`pin laptop postgres-database anchor store` to confirm a move (and move the data first)") {
|
|
||||||
t.Errorf("kept: %+v", kept)
|
|
||||||
}
|
|
||||||
if moved.Kind != kindBindingMoved || moved.Severity != conditions.Urgent ||
|
|
||||||
!strings.Contains(moved.Summary, "game's postgres-database would be sent anchor/store") {
|
|
||||||
t.Errorf("moved: %+v", moved)
|
|
||||||
}
|
|
||||||
if kept.Key() == moved.Key() {
|
|
||||||
t.Error("two consumers, one condition")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A push says the move it refused, and raises its condition at once.
|
|
||||||
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
|
|
||||||
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
|
|
||||||
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
|
|
||||||
open[0].Source != probeBindingsID {
|
|
||||||
t.Fatalf("raised %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,324 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"html/template"
|
|
||||||
"net/http"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// boardCommand serves the three questions as a page.
|
|
||||||
//
|
|
||||||
// **It reads through the same functions everything else does and holds nothing**
|
|
||||||
// (novox/hq 03-DESIGN/01-to-be/11-a-board.md). The board being replaced is one service that reads
|
|
||||||
// every context's database directly — [ADR 0008](novox/hq) violated by the one component with a
|
|
||||||
// reason to violate it, and the cost is that a boundary nothing may cross can move, while one
|
|
||||||
// thing crossing it is enough to freeze it. A board that reads the provisioning tables is a board
|
|
||||||
// that breaks when provisioning changes its tables, and the change then gets weighed against the
|
|
||||||
// board.
|
|
||||||
//
|
|
||||||
// **It stores nothing of its own.** No cache that can disagree, no table of what the mesh looked
|
|
||||||
// like last time. Every request reads the mesh now; if that is slow, the answer belongs in the
|
|
||||||
// context that owns it, where everything else asking gets it too.
|
|
||||||
//
|
|
||||||
// **Reading is the whole of it.** Every action a board could offer already exists as a command,
|
|
||||||
// and a button that does something no command does is a second implementation of a decision.
|
|
||||||
func boardCommand(ctx context.Context, args []string) error {
|
|
||||||
set := flag.NewFlagSet("board", flag.ContinueOnError)
|
|
||||||
// The private network, not everything. A board says which machines are broken and what they
|
|
||||||
// are running, which is exactly the map somebody attacking this would like — and there is no
|
|
||||||
// reason for it to be reachable from further away than the mesh.
|
|
||||||
listen := set.String("listen", "127.0.0.1:8080", "where to serve it")
|
|
||||||
if _, err := parseAround(set, args); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
server := &http.Server{
|
|
||||||
Addr: *listen,
|
|
||||||
ReadHeaderTimeout: 10 * time.Second,
|
|
||||||
Handler: board(),
|
|
||||||
}
|
|
||||||
fmt.Printf("the board is on http://%s\n", *listen)
|
|
||||||
fmt.Printf(" it reads the mesh on every request and keeps nothing\n")
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
<-ctx.Done()
|
|
||||||
closing, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
_ = server.Shutdown(closing)
|
|
||||||
}()
|
|
||||||
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// board is the handler, separate so a test can drive it without a listener.
|
|
||||||
func board() http.Handler {
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path != "/" {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
asked, err := ask(r.Context())
|
|
||||||
if err != nil {
|
|
||||||
// **Said, not blank.** A board that cannot reach the mesh and renders an empty page
|
|
||||||
// says "nothing is wrong" in the one situation where nobody can know that.
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
||||||
w.WriteHeader(http.StatusServiceUnavailable)
|
|
||||||
_ = page.Execute(w, view{Unreachable: err.Error()})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
||||||
if err := page.Execute(w, asked); err != nil {
|
|
||||||
// The page is half-written by now; there is nothing useful left to say to the
|
|
||||||
// browser, and saying it here is what stops the failure being silent.
|
|
||||||
fmt.Printf("the board could not render: %v\n", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
// The same answers for something that is not a person, from the same read. A board and a
|
|
||||||
// script disagreeing about which machine is broken would be worse than either alone.
|
|
||||||
mux.HandleFunc("/mesh.json", func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
open, err := openStores(r.Context())
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusServiceUnavailable)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
asked, err := theThreeQuestions(r.Context(), open)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusServiceUnavailable)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
body, err := statusAsJSON(asked)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
_, _ = w.Write(append(body, '\n'))
|
|
||||||
})
|
|
||||||
return mux
|
|
||||||
}
|
|
||||||
|
|
||||||
// ask reads the mesh for one request.
|
|
||||||
func ask(ctx context.Context) (view, error) {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return view{}, err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
|
|
||||||
asked, err := theThreeQuestions(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return view{}, err
|
|
||||||
}
|
|
||||||
return viewOf(asked), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// view is what the page is given. Nothing is derived here that the reader could not derive.
|
|
||||||
type view struct {
|
|
||||||
Unreachable string
|
|
||||||
Machines int
|
|
||||||
Broken []brokenMachine
|
|
||||||
Quiet []quietMachine
|
|
||||||
Behind []staleModule
|
|
||||||
Waiting []waitingMachine
|
|
||||||
// Unresolved is every machine that cannot be worked out at all. Shown above everything else,
|
|
||||||
// because a machine here is in none of the other lists: nothing was computed for it, so it is
|
|
||||||
// not broken, not quiet and not behind — and a page without this said "all well" about a mesh
|
|
||||||
// where nothing could be sent anywhere.
|
|
||||||
Unresolved []blockedMachine
|
|
||||||
// Network is why the private network could not be computed, when it could not.
|
|
||||||
Network string
|
|
||||||
// Adopted is every node still adopted (novox/hq ADR 0100). Not broken: nothing forces the
|
|
||||||
// flip, so a node left adopted is shown rather than read as converged.
|
|
||||||
Adopted []string
|
|
||||||
At string
|
|
||||||
}
|
|
||||||
|
|
||||||
type blockedMachine struct {
|
|
||||||
Node string
|
|
||||||
// Said is the mesh's own words, a line at a time. Every unmet requirement, not the first:
|
|
||||||
// a machine is usually blocked by more than one and fixing one of them changes nothing.
|
|
||||||
Said []string
|
|
||||||
}
|
|
||||||
|
|
||||||
type waitingMachine struct {
|
|
||||||
Node string
|
|
||||||
// Never told is not out of date: nobody has ever asked this machine to be anything. Same
|
|
||||||
// remedy, different situation, and the page says which.
|
|
||||||
Never bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type brokenMachine struct {
|
|
||||||
Node string
|
|
||||||
// Outcome is refused or failed, and stays distinct all the way to the page. **Refused means
|
|
||||||
// the machine is exactly as it was and what is wrong is in what was sent; failed means it is
|
|
||||||
// in a state nobody declared and what is wrong is on the machine.** They are fixed in
|
|
||||||
// different places, so one word for both would send half the readers to the wrong one.
|
|
||||||
Outcome string
|
|
||||||
Said []string
|
|
||||||
When string
|
|
||||||
}
|
|
||||||
|
|
||||||
type quietMachine struct {
|
|
||||||
Node string
|
|
||||||
// Heard is "never" or how long ago. Never heard from is not the same as quiet for a while:
|
|
||||||
// one may be a machine that was never sent anything.
|
|
||||||
Heard string
|
|
||||||
}
|
|
||||||
|
|
||||||
type staleModule struct {
|
|
||||||
Module string
|
|
||||||
Holds string
|
|
||||||
Source string
|
|
||||||
Running []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func viewOf(asked answers) view {
|
|
||||||
out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
|
|
||||||
Network: asked.network, Adopted: adoptedNodes(asked.nodes)}
|
|
||||||
var blocked []string
|
|
||||||
for name := range asked.refused {
|
|
||||||
blocked = append(blocked, name)
|
|
||||||
}
|
|
||||||
sort.Strings(blocked)
|
|
||||||
for _, name := range blocked {
|
|
||||||
one := blockedMachine{Node: name}
|
|
||||||
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
|
|
||||||
one.Said = append(one.Said, strings.TrimSpace(line))
|
|
||||||
}
|
|
||||||
out.Unresolved = append(out.Unresolved, one)
|
|
||||||
}
|
|
||||||
for _, d := range asked.wrong {
|
|
||||||
one := brokenMachine{Node: d.Node, Outcome: d.Outcome,
|
|
||||||
When: d.At.Local().Format("2006-01-02 15:04")}
|
|
||||||
if d.Refused != "" {
|
|
||||||
// The host's own words. It says exactly what it could not accept, and nothing
|
|
||||||
// written here would say it better.
|
|
||||||
one.Said = append(one.Said, firstLine(d.Refused))
|
|
||||||
}
|
|
||||||
for _, f := range d.Failed {
|
|
||||||
one.Said = append(one.Said, f.ID+": "+firstLine(f.Error))
|
|
||||||
}
|
|
||||||
out.Broken = append(out.Broken, one)
|
|
||||||
}
|
|
||||||
for _, n := range asked.quiet {
|
|
||||||
out.Quiet = append(out.Quiet, quietMachine{Node: n.Name, Heard: heardFrom(n)})
|
|
||||||
}
|
|
||||||
for _, m := range asked.waiting {
|
|
||||||
out.Waiting = append(out.Waiting, waitingMachine{Node: m.Node, Never: m.Never})
|
|
||||||
}
|
|
||||||
for module, on := range asked.behind {
|
|
||||||
from := asked.sources[module]
|
|
||||||
out.Behind = append(out.Behind, staleModule{
|
|
||||||
Module: module, Holds: short(from.BuiltFrom), Source: short(from.Head), Running: on,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// The page. Deliberately one file with no assets: a board that cannot render without fetching
|
|
||||||
// something is a board that is blank exactly when the mesh is unwell.
|
|
||||||
var page = template.Must(template.New("board").Parse(`<!doctype html>
|
|
||||||
<meta charset="utf-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
||||||
<title>the mesh</title>
|
|
||||||
<style>
|
|
||||||
:root { color-scheme: light dark; }
|
|
||||||
body { font: 15px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; margin: 2rem auto;
|
|
||||||
max-width: 52rem; padding: 0 1rem; }
|
|
||||||
h1 { font-size: 1.1rem; font-weight: 600; margin: 0 0 1.5rem; }
|
|
||||||
h2 { font-size: 1rem; font-weight: 600; margin: 2rem 0 .5rem; }
|
|
||||||
.quiet { opacity: .65; }
|
|
||||||
.said { opacity: .8; padding-left: 1.5rem; }
|
|
||||||
.outcome { display: inline-block; min-width: 4.5rem; }
|
|
||||||
.refused { color: #b26b00; }
|
|
||||||
.failed { color: #c0392b; }
|
|
||||||
ul { list-style: none; padding: 0; margin: 0; }
|
|
||||||
li { padding: .15rem 0; }
|
|
||||||
footer { margin-top: 3rem; opacity: .6; font-size: .85rem; }
|
|
||||||
</style>
|
|
||||||
{{if .Unreachable}}
|
|
||||||
<h1>the mesh cannot be read</h1>
|
|
||||||
<p class="failed">{{.Unreachable}}</p>
|
|
||||||
<p class="quiet">This says nothing about whether the mesh is well — only that this page could not
|
|
||||||
find out.</p>
|
|
||||||
{{else}}
|
|
||||||
<h1>{{.Machines}} machine{{if ne .Machines 1}}s{{end}}</h1>
|
|
||||||
|
|
||||||
{{if .Unresolved}}
|
|
||||||
<h2>Can everything be worked out?</h2>
|
|
||||||
<ul>
|
|
||||||
{{range .Unresolved}}
|
|
||||||
<li><span class="outcome failed">blocked</span> <strong>{{.Node}}</strong>
|
|
||||||
{{range .Said}}<div class="said">{{.}}</div>{{end}}
|
|
||||||
</li>
|
|
||||||
{{end}}
|
|
||||||
</ul>
|
|
||||||
<p class="quiet">Nothing can be sent to a machine here, and it appears in none of the lists below:
|
|
||||||
nothing was computed for it, so there is nothing it can be behind.</p>
|
|
||||||
{{end}}
|
|
||||||
{{if .Network}}
|
|
||||||
<p class="failed">The private network could not be computed: {{.Network}}</p>
|
|
||||||
{{end}}
|
|
||||||
|
|
||||||
<h2>Is anything broken?</h2>
|
|
||||||
{{if .Broken}}
|
|
||||||
<ul>
|
|
||||||
{{range .Broken}}
|
|
||||||
<li>
|
|
||||||
<span class="outcome {{.Outcome}}">{{.Outcome}}</span>
|
|
||||||
<strong>{{.Node}}</strong> <span class="quiet">{{.When}}</span>
|
|
||||||
{{range .Said}}<div class="said">{{.}}</div>{{end}}
|
|
||||||
</li>
|
|
||||||
{{end}}
|
|
||||||
</ul>
|
|
||||||
{{else}}<p class="quiet">No. Every machine is doing what it was told.</p>{{end}}
|
|
||||||
|
|
||||||
<h2>Is anything not answering?</h2>
|
|
||||||
{{if .Quiet}}
|
|
||||||
<ul>{{range .Quiet}}<li><strong>{{.Node}}</strong> <span class="quiet">{{.Heard}}</span></li>{{end}}</ul>
|
|
||||||
<p class="quiet">Not heard from is not the same as tried and could not — a machine here may be
|
|
||||||
new, switched off, or unreachable.</p>
|
|
||||||
{{else}}<p class="quiet">No. Every machine has been heard from.</p>{{end}}
|
|
||||||
|
|
||||||
<h2>Is anything out of date?</h2>
|
|
||||||
{{if .Behind}}
|
|
||||||
<ul>
|
|
||||||
{{range .Behind}}
|
|
||||||
<li><strong>{{.Module}}</strong> holds {{.Holds}}, source has {{.Source}}
|
|
||||||
{{if .Running}}<div class="said">running on {{range $i, $n := .Running}}{{if $i}}, {{end}}{{$n}}{{end}}</div>
|
|
||||||
{{else}}<div class="said quiet">assigned to nothing</div>{{end}}
|
|
||||||
</li>
|
|
||||||
{{end}}
|
|
||||||
</ul>
|
|
||||||
{{else}}<p class="quiet">No. Every module is what its source last had.</p>{{end}}
|
|
||||||
|
|
||||||
{{if .Waiting}}
|
|
||||||
<ul>
|
|
||||||
{{range .Waiting}}
|
|
||||||
<li><strong>{{.Node}}</strong>
|
|
||||||
{{if .Never}}<span class="quiet">has never been sent anything</span>
|
|
||||||
{{else}}<span class="quiet">is not running what the mesh would send it</span>{{end}}
|
|
||||||
</li>
|
|
||||||
{{end}}
|
|
||||||
</ul>
|
|
||||||
<p class="quiet">Never told is not out of date: nobody has asked that machine to be anything yet.
|
|
||||||
Both are sent by <code>push --behind</code>.</p>
|
|
||||||
{{else}}<p class="quiet">Every machine is running what the mesh would send it.</p>{{end}}
|
|
||||||
{{if .Adopted}}
|
|
||||||
<h2>Which machines are adopted?</h2>
|
|
||||||
<ul>{{range .Adopted}}<li><strong>{{.}}</strong> <span class="quiet">adopted — what was found on it is kept until each module is taken</span></li>{{end}}</ul>
|
|
||||||
{{end}}
|
|
||||||
{{end}}
|
|
||||||
|
|
||||||
<footer>Read at {{.At}}. This page holds nothing and changes nothing.</footer>
|
|
||||||
`))
|
|
||||||
@@ -1,154 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Refused and failed stay distinct all the way to the page.
|
|
||||||
//
|
|
||||||
// **Refused means the machine is exactly as it was and what is wrong is in what was sent; failed
|
|
||||||
// means it is in a state nobody declared and what is wrong is on the machine.** They are fixed in
|
|
||||||
// different places, so a page saying "error" for both sends half its readers to the wrong one.
|
|
||||||
func TestRefusedAndFailedReachThePageAsDifferentThings(t *testing.T) {
|
|
||||||
rendered := render(t, viewOf(answers{
|
|
||||||
nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
|
||||||
wrong: []inventory.Doing{
|
|
||||||
{Node: "anchor", Outcome: "refused", Refused: "not a declaration this host speaks",
|
|
||||||
At: time.Now()},
|
|
||||||
{Node: "laptop", Outcome: "failed", At: time.Now(),
|
|
||||||
Failed: []inventory.FailedResource{{ID: "web.container", Error: "no such image"}}},
|
|
||||||
},
|
|
||||||
}))
|
|
||||||
// The rendered outcome, not the word anywhere on the page: both words appear in the
|
|
||||||
// stylesheet, so a plain Contains passes whatever the machine actually said. It did.
|
|
||||||
for _, want := range []string{`<span class="outcome refused">refused</span>`,
|
|
||||||
`<span class="outcome failed">failed</span>`} {
|
|
||||||
if !strings.Contains(rendered, want) {
|
|
||||||
t.Fatalf("the page does not distinguish the two — missing %s:\n%s", want, rendered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// And the host's own words, which say exactly what it could not accept.
|
|
||||||
for _, want := range []string{"not a declaration this host speaks", "web.container", "no such image"} {
|
|
||||||
if !strings.Contains(rendered, want) {
|
|
||||||
t.Fatalf("the page does not say %q, so a reader must go and ask:\n%s", want, rendered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing wrong is said, not left blank. An empty page and a well mesh must not look alike.
|
|
||||||
func TestAWellMeshSaysSoRatherThanShowingNothing(t *testing.T) {
|
|
||||||
rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}}))
|
|
||||||
for _, want := range []string{
|
|
||||||
"Every machine is doing what it was told",
|
|
||||||
"Every machine has been heard from",
|
|
||||||
"Every module is what its source last had",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(rendered, want) {
|
|
||||||
t.Fatalf("a well mesh does not say %q:\n%s", want, rendered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A board that cannot reach the mesh must not render an empty page: that says "nothing is wrong"
|
|
||||||
// in the one situation where nobody can know it.
|
|
||||||
func TestABoardThatCannotReadTheMeshSaysSo(t *testing.T) {
|
|
||||||
rendered := render(t, view{Unreachable: "the store did not answer"})
|
|
||||||
if !strings.Contains(rendered, "the store did not answer") {
|
|
||||||
t.Fatalf("the reason is not on the page:\n%s", rendered)
|
|
||||||
}
|
|
||||||
if strings.Contains(rendered, "Every machine is doing what it was told") {
|
|
||||||
t.Fatal("a board that could not read the mesh reported that the mesh is well")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Quiet is not broken, and the page says which it is.
|
|
||||||
func TestQuietIsNotReportedAsBroken(t *testing.T) {
|
|
||||||
rendered := render(t, viewOf(answers{
|
|
||||||
nodes: []inventory.Node{{Name: "laptop"}},
|
|
||||||
quiet: []inventory.Node{{Name: "laptop"}},
|
|
||||||
}))
|
|
||||||
if !strings.Contains(rendered, "not the same as tried and could not") {
|
|
||||||
t.Fatalf("the page does not separate quiet from broken:\n%s", rendered)
|
|
||||||
}
|
|
||||||
if !strings.Contains(rendered, "Every machine is doing what it was told") {
|
|
||||||
t.Fatalf("a quiet machine was counted as broken:\n%s", rendered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The page renders what a machine said, and a hostile string in it is not markup.
|
|
||||||
//
|
|
||||||
// What is on this page comes from machines, and a machine's own words are the whole reason the
|
|
||||||
// page is useful. They are also the one thing here that nobody in this repository wrote.
|
|
||||||
func TestWhatAMachineSaidIsNotMarkup(t *testing.T) {
|
|
||||||
rendered := render(t, viewOf(answers{
|
|
||||||
nodes: []inventory.Node{{Name: "anchor"}},
|
|
||||||
wrong: []inventory.Doing{{Node: "anchor", Outcome: "refused",
|
|
||||||
Refused: `<script>alert("from the machine")</script>`, At: time.Now()}},
|
|
||||||
}))
|
|
||||||
if strings.Contains(rendered, "<script>alert") {
|
|
||||||
t.Fatalf("a machine's words were rendered as markup:\n%s", rendered)
|
|
||||||
}
|
|
||||||
if !strings.Contains(rendered, "<script>") {
|
|
||||||
t.Fatalf("the words were not shown at all, so the reader cannot see what it said:\n%s", rendered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module behind its source names the machines running the old one — the part with consequences.
|
|
||||||
func TestBeingBehindNamesTheMachinesRunningTheOldOne(t *testing.T) {
|
|
||||||
rendered := render(t, viewOf(answers{
|
|
||||||
nodes: []inventory.Node{{Name: "laptop"}},
|
|
||||||
behind: map[string][]string{"web": {"laptop"}},
|
|
||||||
sources: map[string]inventory.Source{"web": {BuiltFrom: "aaaaaaaaaa", Head: "bbbbbbbbbb"}},
|
|
||||||
}))
|
|
||||||
for _, want := range []string{"web", "aaaaaaaa", "bbbbbbbb", "running on laptop"} {
|
|
||||||
if !strings.Contains(rendered, want) {
|
|
||||||
t.Fatalf("the page does not say %q:\n%s", want, rendered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func render(t *testing.T, v view) string {
|
|
||||||
t.Helper()
|
|
||||||
var out strings.Builder
|
|
||||||
if err := page.Execute(&out, v); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return out.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// The board and the command must answer the same question the same way.
|
|
||||||
//
|
|
||||||
// A machine not running what the mesh would send it was added to `status` and would have been
|
|
||||||
// missing here — which is the one thing this page's design forbids: two answers to the same
|
|
||||||
// question, disagreeing, with a person in front of each.
|
|
||||||
func TestTheBoardSaysWhichMachinesHaveNotBeenSentWhatTheyShouldBe(t *testing.T) {
|
|
||||||
rendered := render(t, viewOf(answers{
|
|
||||||
nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
|
||||||
waiting: []inventory.Machine{
|
|
||||||
{Node: "anchor"},
|
|
||||||
{Node: "laptop", Never: true},
|
|
||||||
},
|
|
||||||
}))
|
|
||||||
if !strings.Contains(rendered, "is not running what the mesh would send it") {
|
|
||||||
t.Fatalf("the page does not say a machine is out of date:\n%s", rendered)
|
|
||||||
}
|
|
||||||
// Never told is a different situation with the same remedy, and the page keeps them apart.
|
|
||||||
if !strings.Contains(rendered, "has never been sent anything") {
|
|
||||||
t.Fatalf("the page does not separate never told from out of date:\n%s", rendered)
|
|
||||||
}
|
|
||||||
if !strings.Contains(rendered, "push --behind") {
|
|
||||||
t.Fatalf("the page does not say what sends them:\n%s", rendered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And says so when there is nothing waiting, rather than leaving the question unanswered.
|
|
||||||
func TestAMeshWithNothingWaitingSaysSo(t *testing.T) {
|
|
||||||
rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}}))
|
|
||||||
if !strings.Contains(rendered, "Every machine is running what the mesh would send it") {
|
|
||||||
t.Fatalf("the page leaves the question unanswered:\n%s", rendered)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,207 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A filling bucket or log is said before it is full** (novox/hq ADR 0297 §6, issue 501).
|
|
||||||
//
|
|
||||||
// A module's bucket and its log each have a cap on the bus, and when either is full the bus refuses
|
|
||||||
// the next write: the module stops doing what it writes for, and nothing said so before. On
|
|
||||||
// 2026-10-11 the issue tracker's bucket held a sixth of its cap after two days, growing toward a stop
|
|
||||||
// nobody would have heard coming. So the self-check reads every module's bucket and log, and one at
|
|
||||||
// fillRaiseAt of its cap or more raises a condition naming the module, the bucket or log, and how full
|
|
||||||
// it is.
|
|
||||||
//
|
|
||||||
// **Cleared by observation below fillClearBelow, not below fillRaiseAt**: a bucket or log hovering at
|
|
||||||
// its threshold would otherwise be raised and cleared on every run. Between the two, an open condition
|
|
||||||
// is kept and none is raised.
|
|
||||||
//
|
|
||||||
// **One that cannot be read is said, and the rest are still judged.** An unanswered question about one
|
|
||||||
// stream is no reason to know nothing of the others: it is a finding of its own, naming the bucket or
|
|
||||||
// log and why, and a fill condition already open for it is kept, because not knowing is not a pass.
|
|
||||||
|
|
||||||
// The fill thresholds, in percent of a bucket's or log's cap.
|
|
||||||
const (
|
|
||||||
fillRaiseAt = 75
|
|
||||||
fillClearBelow = 70
|
|
||||||
)
|
|
||||||
|
|
||||||
// The conditions the fill probe raises: one filling, and one that could not be read.
|
|
||||||
const (
|
|
||||||
kindBucketOrLogFilling = "bucket-or-log-filling"
|
|
||||||
kindBucketOrLogUnread = "bucket-or-log-unread"
|
|
||||||
)
|
|
||||||
|
|
||||||
// probeFillID is the probe's id in the registry.
|
|
||||||
const probeFillID = "D-fill"
|
|
||||||
|
|
||||||
// bucketOrLog is one module's bucket or log as the fill probe reads it.
|
|
||||||
type bucketOrLog struct {
|
|
||||||
Module string
|
|
||||||
// Kind is `bucket` or `log`; Name its local name; Stream the stream it is on the bus.
|
|
||||||
Kind, Name, Stream string
|
|
||||||
}
|
|
||||||
|
|
||||||
// filled is how full one bucket or log is, read from the bus.
|
|
||||||
type filled struct {
|
|
||||||
Bytes, Max uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// percent is how full, in whole percent, rounded down.
|
|
||||||
func (f filled) percent() uint64 {
|
|
||||||
if f.Max == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return f.Bytes * 100 / f.Max
|
|
||||||
}
|
|
||||||
|
|
||||||
// fillKey is where a bucket's or log's fill condition is kept.
|
|
||||||
func fillKey(s bucketOrLog) string { return conditions.Key(conditions.ScopeBus, s.Stream, "filling") }
|
|
||||||
|
|
||||||
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and,
|
|
||||||
// while its condition is open, at fillClearBelow or above too; nothing otherwise, which is what clears
|
|
||||||
// it. The operator's words are the kind's (plain_words.go); the summary and the evidence name the
|
|
||||||
// module, the bucket or log, and the fill.
|
|
||||||
func fillObservation(s bucketOrLog, f filled, open bool) (conditions.Observation, bool) {
|
|
||||||
if f.Max == 0 {
|
|
||||||
return conditions.Observation{}, false // one with no cap cannot fill
|
|
||||||
}
|
|
||||||
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
|
||||||
atRaise := f.Bytes*100 >= f.Max*fillRaiseAt
|
|
||||||
aboveClear := f.Bytes*100 >= f.Max*fillClearBelow
|
|
||||||
if !atRaise && !(open && aboveClear) {
|
|
||||||
return conditions.Observation{}, false
|
|
||||||
}
|
|
||||||
pct := f.percent()
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
|
|
||||||
s.Module, s.Kind, s.Name, mibWords(f.Bytes), mibWords(f.Max), pct),
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", s.Module, s.Kind, s.Name,
|
|
||||||
s.Stream, f.Bytes, f.Max, pct),
|
|
||||||
}, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// unreadObservation says one bucket or log could not be read, and why. Asked over the network, so one
|
|
||||||
// look can be wrong: raised on the second look in a row (confirm.go).
|
|
||||||
func unreadObservation(s bucketOrLog, why error) conditions.Observation {
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "unread", Kind: kindBucketOrLogUnread,
|
|
||||||
Severity: conditions.Warning, Confirm: true,
|
|
||||||
Summary: fmt.Sprintf("%s's %s %s could not be read, so how full it is is not known", s.Module, s.Kind, s.Name),
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %v", s.Module, s.Kind, s.Name, s.Stream, why),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptFilling is an open fill condition said again for a bucket or log that could not be read this
|
|
||||||
// time: not knowing how full it is now is no reason to say it has room. Only ever made from a condition
|
|
||||||
// read back as open — its own summary and severity, never words made up for it.
|
|
||||||
func keptFilling(s bucketOrLog, open conditions.Condition, why error) conditions.Observation {
|
|
||||||
return conditions.Observation{
|
|
||||||
Scope: conditions.ScopeBus, ID: s.Stream, Token: "filling", Kind: kindBucketOrLogFilling,
|
|
||||||
Severity: open.Severity, Summary: open.Summary,
|
|
||||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): not read this time (%v); kept open as it was",
|
|
||||||
s.Module, s.Kind, s.Name, s.Stream, why),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// mibWords is a size as a person reads it, in MiB with one decimal.
|
|
||||||
func mibWords(b uint64) string {
|
|
||||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024))
|
|
||||||
}
|
|
||||||
|
|
||||||
// readFill is how full one bucket or log is on the bus; found false when it is not on the bus.
|
|
||||||
type readFill func(ctx context.Context, s bucketOrLog) (f filled, found bool, err error)
|
|
||||||
|
|
||||||
// openFill is the fill condition open under a key, if one is, or why it could not be read.
|
|
||||||
type openFill func(ctx context.Context, key string) (conditions.Condition, bool, error)
|
|
||||||
|
|
||||||
// judgeFills judges every bucket and log on its own: one that cannot be read is said as unread, with
|
|
||||||
// its fill condition kept only when that condition was read back and is open — when it cannot be read
|
|
||||||
// either, nothing is said of its fill, which the unread finding already covers. Every other is judged
|
|
||||||
// by its fill; for one of those, a condition that cannot be read is taken as open, so an unknown never
|
|
||||||
// clears a fill measured between fillClearBelow and fillRaiseAt.
|
|
||||||
func judgeFills(ctx context.Context, all []bucketOrLog, read readFill, open openFill) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, s := range all {
|
|
||||||
f, found, err := read(ctx, s)
|
|
||||||
if err != nil {
|
|
||||||
out = append(out, unreadObservation(s, err))
|
|
||||||
if c, isOpen, cerr := open(ctx, fillKey(s)); cerr == nil && isOpen {
|
|
||||||
out = append(out, keptFilling(s, c, err))
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
continue // not on the bus: created on the controller's next raise, and nothing there can fill
|
|
||||||
}
|
|
||||||
_, isOpen, cerr := open(ctx, fillKey(s))
|
|
||||||
isOpen = isOpen || cerr != nil
|
|
||||||
if o, said := fillObservation(s, f, isOpen); said {
|
|
||||||
out = append(out, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// declaredBucketsAndLogs is every module's bucket and log the catalogue declares, by its stream.
|
|
||||||
func declaredBucketsAndLogs(ctx context.Context, d *doctor) ([]bucketOrLog, error) {
|
|
||||||
buckets, err := d.open.inventory.DeclaredBuckets(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
logs, err := d.open.inventory.DeclaredLogs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []bucketOrLog
|
|
||||||
for _, b := range buckets {
|
|
||||||
out = append(out, bucketOrLog{Module: b.Module, Kind: "bucket", Name: b.Name, Stream: "KV_" + b.Bucket()})
|
|
||||||
}
|
|
||||||
for _, l := range logs {
|
|
||||||
out = append(out, bucketOrLog{Module: l.Module, Kind: "log", Name: l.Name, Stream: l.Stream()})
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeFill reads every module's bucket and log on the bus and says each one filling toward its cap,
|
|
||||||
// and each one that could not be read.
|
|
||||||
func probeFill(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
||||||
all, err := declaredBucketsAndLogs(ctx, d)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
js := d.js.Context()
|
|
||||||
read := func(ctx context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return filled{}, false, err
|
|
||||||
}
|
|
||||||
info, err := js.StreamInfo(s.Stream, nats.Context(ctx))
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, nats.ErrStreamNotFound):
|
|
||||||
return filled{}, false, nil
|
|
||||||
case err != nil:
|
|
||||||
return filled{}, false, err
|
|
||||||
case info.Config.MaxBytes <= 0:
|
|
||||||
return filled{}, true, nil
|
|
||||||
}
|
|
||||||
return filled{Bytes: info.State.Bytes, Max: uint64(info.Config.MaxBytes)}, true, nil
|
|
||||||
}
|
|
||||||
open := func(ctx context.Context, key string) (conditions.Condition, bool, error) {
|
|
||||||
if d.keeper == nil {
|
|
||||||
return conditions.Condition{}, false, nil
|
|
||||||
}
|
|
||||||
return d.keeper.Get(ctx, key)
|
|
||||||
}
|
|
||||||
return judgeFills(ctx, all, read, open), nil
|
|
||||||
}
|
|
||||||
@@ -1,165 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A filling log or bucket is said at three quarters of its cap and cleared below seven tenths**
|
|
||||||
// (novox/hq ADR 0297 §6): raised at 75% naming the module, the bucket or log and its fill; not at 74.9%;
|
|
||||||
// kept between 70% and 75% while it is open, and not raised there when it is not; and cleared — said no
|
|
||||||
// more — once it reads below 70%, open or not.
|
|
||||||
func TestAFillingBucketOrLogIsRaisedAtThreeQuartersAndClearedBelowSevenTenths(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
log := bucketOrLog{Module: "mesh-issues", Kind: "log", Name: "changes", Stream: "LOG_mesh-issues_changes"}
|
|
||||||
bucket := bucketOrLog{Module: "mesh-issues", Kind: "bucket", Name: "issues", Stream: "KV_mesh-issues_issues"}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
of bucketOrLog
|
|
||||||
bytes uint64
|
|
||||||
max uint64
|
|
||||||
open bool
|
|
||||||
said bool
|
|
||||||
}{
|
|
||||||
{"a log at exactly 75%", log, 768 * mib, 1024 * mib, false, true},
|
|
||||||
{"a bucket at exactly 75%", bucket, 48 * mib, 64 * mib, false, true},
|
|
||||||
{"a bucket full", bucket, 64 * mib, 64 * mib, false, true},
|
|
||||||
{"a log just under 75%", log, 768*mib - 1, 1024 * mib, false, false},
|
|
||||||
{"a bucket at 72%, not open", bucket, 64 * mib * 72 / 100, 64 * mib, false, false},
|
|
||||||
{"a bucket at 72%, open: kept", bucket, 64 * mib * 72 / 100, 64 * mib, true, true},
|
|
||||||
{"a log at exactly 70%, open: kept", log, 700 * mib, 1000 * mib, true, true},
|
|
||||||
{"a log just under 70%, open: cleared", log, 700*mib - 1, 1000 * mib, true, false},
|
|
||||||
{"a bucket at 10%, open: cleared", bucket, 64 * mib / 10, 64 * mib, true, false},
|
|
||||||
{"a bucket with no cap", bucket, 100, 0, true, false},
|
|
||||||
} {
|
|
||||||
o, said := fillObservation(c.of, filled{Bytes: c.bytes, Max: c.max}, c.open)
|
|
||||||
if said != c.said {
|
|
||||||
t.Errorf("%s: said %v, want %v", c.name, said, c.said)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !said {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if o.Key() != fillKey(c.of) || o.Kind != kindBucketOrLogFilling || o.Severity != conditions.Warning {
|
|
||||||
t.Errorf("%s: raised as %s (%s, %s)", c.name, o.Key(), o.Kind, o.Severity)
|
|
||||||
}
|
|
||||||
for _, part := range []string{c.of.Module, c.of.Kind + " " + c.of.Name, "%", " of "} {
|
|
||||||
if !strings.Contains(o.Summary, part) {
|
|
||||||
t.Errorf("%s: does not name %q: %q", c.name, part, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Said, "bytes") {
|
|
||||||
t.Errorf("%s: the evidence does not say the fill in bytes: %q", c.name, o.Said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
o, _ := fillObservation(log, filled{Bytes: 800 * mib, Max: 1024 * mib}, false)
|
|
||||||
if !strings.Contains(o.Summary, "800.0 MiB of 1024.0 MiB, 78%") {
|
|
||||||
t.Errorf("the fill is said as %q", o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **One bucket or log that cannot be read does not stop the others being judged** (review of #231): it is
|
|
||||||
// said as unread with its reason, a fill condition open for it is kept, and every other bucket and log
|
|
||||||
// is judged by its own fill.
|
|
||||||
func TestAnUnreadableBucketOrLogIsSaidAndTheRestAreStillJudged(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
broken := bucketOrLog{Module: "a", Kind: "bucket", Name: "broken", Stream: "KV_a_broken"}
|
|
||||||
brokenOpen := bucketOrLog{Module: "a", Kind: "log", Name: "was-filling", Stream: "LOG_a_was-filling"}
|
|
||||||
full := bucketOrLog{Module: "b", Kind: "log", Name: "changes", Stream: "LOG_b_changes"}
|
|
||||||
empty := bucketOrLog{Module: "c", Kind: "bucket", Name: "quiet", Stream: "KV_c_quiet"}
|
|
||||||
absent := bucketOrLog{Module: "d", Kind: "bucket", Name: "not-yet", Stream: "KV_d_not-yet"}
|
|
||||||
refusal := errors.New("the bus did not answer")
|
|
||||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
switch s {
|
|
||||||
case broken, brokenOpen:
|
|
||||||
return filled{}, false, refusal
|
|
||||||
case full:
|
|
||||||
return filled{Bytes: 900 * mib, Max: 1000 * mib}, true, nil
|
|
||||||
case empty:
|
|
||||||
return filled{Bytes: 1, Max: 64 * mib}, true, nil
|
|
||||||
}
|
|
||||||
return filled{}, false, nil
|
|
||||||
}
|
|
||||||
open := func(_ context.Context, key string) (conditions.Condition, bool, error) {
|
|
||||||
if key == fillKey(brokenOpen) {
|
|
||||||
return conditions.Condition{Key: key, Severity: conditions.Warning,
|
|
||||||
Summary: "a's log was-filling holds 800.0 MiB of 1024.0 MiB, 78%"}, true, nil
|
|
||||||
}
|
|
||||||
return conditions.Condition{}, false, nil
|
|
||||||
}
|
|
||||||
got := map[string]conditions.Observation{}
|
|
||||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, brokenOpen, full, empty, absent}, read, open) {
|
|
||||||
got[o.Key()] = o
|
|
||||||
}
|
|
||||||
for _, s := range []bucketOrLog{broken, brokenOpen} {
|
|
||||||
o, said := got[conditions.Key(conditions.ScopeBus, s.Stream, "unread")]
|
|
||||||
if !said || o.Kind != kindBucketOrLogUnread || !o.Confirm || !strings.Contains(o.Said, refusal.Error()) {
|
|
||||||
t.Errorf("%s could not be read and was said as %+v", s.Stream, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if o, kept := got[fillKey(brokenOpen)]; !kept || !strings.Contains(o.Said, "kept open") {
|
|
||||||
t.Errorf("an open fill condition of a log not read this time was not kept: %+v", o)
|
|
||||||
}
|
|
||||||
if _, said := got[fillKey(broken)]; said {
|
|
||||||
t.Error("a bucket not read and not filling was said filling")
|
|
||||||
}
|
|
||||||
if o, said := got[fillKey(full)]; !said || o.Kind != kindBucketOrLogFilling {
|
|
||||||
t.Errorf("a log at 90%% beside an unreadable one was not judged: %+v", got)
|
|
||||||
}
|
|
||||||
if len(got) != 4 {
|
|
||||||
t.Errorf("said %d findings, want 4 (two unread, one kept, one filling): %v", len(got), got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both kinds have plain words of their own, which hold to the plain rule (novox/hq ADR 0253).
|
|
||||||
func TestAFillingOrUnreadBucketOrLogIsSaidInPlainWords(t *testing.T) {
|
|
||||||
for _, kind := range []string{kindBucketOrLogFilling, kindBucketOrLogUnread} {
|
|
||||||
wording, has := plainWordings[kind]
|
|
||||||
if !has {
|
|
||||||
t.Errorf("%s has no plain words", kind)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if why, ok := conditions.PlainWords(wording(conditions.Observation{Kind: kind})); !ok {
|
|
||||||
t.Errorf("%s: %s", kind, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **When neither the bucket or log nor its condition can be read, nothing is said of its fill** (second
|
|
||||||
// review of #231): the unread finding covers it, and no fill is invented for it. For one that is read and
|
|
||||||
// measured between seven tenths and three quarters, a condition that cannot be read is taken as open, so
|
|
||||||
// an unknown never clears it.
|
|
||||||
func TestNothingIsSaidOfAFillWhenNeitherItNorItsConditionCanBeRead(t *testing.T) {
|
|
||||||
const mib = 1024 * 1024
|
|
||||||
broken := bucketOrLog{Module: "a", Kind: "log", Name: "changes", Stream: "LOG_a_changes"}
|
|
||||||
between := bucketOrLog{Module: "b", Kind: "bucket", Name: "issues", Stream: "KV_b_issues"}
|
|
||||||
read := func(_ context.Context, s bucketOrLog) (filled, bool, error) {
|
|
||||||
if s == broken {
|
|
||||||
return filled{}, false, errors.New("the bus did not answer")
|
|
||||||
}
|
|
||||||
return filled{Bytes: 72 * mib, Max: 100 * mib}, true, nil
|
|
||||||
}
|
|
||||||
open := func(context.Context, string) (conditions.Condition, bool, error) {
|
|
||||||
return conditions.Condition{}, false, errors.New("the conditions bucket did not answer")
|
|
||||||
}
|
|
||||||
got := map[string]conditions.Observation{}
|
|
||||||
for _, o := range judgeFills(context.Background(), []bucketOrLog{broken, between}, read, open) {
|
|
||||||
got[o.Key()] = o
|
|
||||||
}
|
|
||||||
if o, said := got[fillKey(broken)]; said {
|
|
||||||
t.Fatalf("a fill was said for a log whose fill and condition could not be read: %+v", o)
|
|
||||||
}
|
|
||||||
if _, said := got[conditions.Key(conditions.ScopeBus, broken.Stream, "unread")]; !said {
|
|
||||||
t.Error("the log that could not be read was not said as unread")
|
|
||||||
}
|
|
||||||
if _, kept := got[fillKey(between)]; !kept {
|
|
||||||
t.Error("a bucket at 72% whose condition could not be read was cleared")
|
|
||||||
}
|
|
||||||
if len(got) != 2 {
|
|
||||||
t.Errorf("said %d findings, want 2: %v", len(got), got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,294 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// asking a build machine for a module, and what came back.
|
|
||||||
//
|
|
||||||
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
|
||||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
|
||||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
|
||||||
|
|
||||||
// buildsCommand says what has been built lately.
|
|
||||||
func buildsCommand(ctx context.Context, args []string) error {
|
|
||||||
set := flag.NewFlagSet("builds", flag.ContinueOnError)
|
|
||||||
limit := set.Int("n", 20, "how many to show")
|
|
||||||
positionals, err := parseAround(set, args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
module := ""
|
|
||||||
if len(positionals) == 1 {
|
|
||||||
module = positionals[0]
|
|
||||||
} else if len(positionals) > 1 {
|
|
||||||
return errors.New("builds [<module>] [-n N]")
|
|
||||||
}
|
|
||||||
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
inv := open.inventory
|
|
||||||
|
|
||||||
builds, err := inv.Builds(ctx, module, *limit)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(builds) == 0 {
|
|
||||||
// Said rather than printed as nothing: an empty list and a failed read must never look
|
|
||||||
// the same, and getting here means the store answered.
|
|
||||||
if module != "" {
|
|
||||||
fmt.Printf("nothing has been built for %s\n", module)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
fmt.Println("nothing has been built yet")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, b := range builds {
|
|
||||||
printBuild(os.Stdout, b)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// printBuild says one build: what, its outcome, where and when, what it made and published, its reuse key, and its
|
|
||||||
// source fingerprint (novox/hq issue 1204: the one tool that reads what the builder recorded it was made from).
|
|
||||||
func printBuild(w io.Writer, b inventory.Build) {
|
|
||||||
what := b.Module
|
|
||||||
if what == "" {
|
|
||||||
// It failed before knowing what it was building, which is most of the interesting
|
|
||||||
// failures. The repository is what a person has to go and look at.
|
|
||||||
what = "?"
|
|
||||||
}
|
|
||||||
outcome := "built " + short(b.Commit)
|
|
||||||
if !b.Worked() {
|
|
||||||
outcome = "failed"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, "%-18s %-14s %-10s %s %s\n",
|
|
||||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
|
||||||
fmt.Fprintf(w, " %s", b.Repository)
|
|
||||||
if b.Ref != "" {
|
|
||||||
fmt.Fprintf(w, " at %s", b.Ref)
|
|
||||||
}
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
for _, made := range b.Made {
|
|
||||||
fmt.Fprintf(w, " %-10s %s\n", made.Kind, made.Reference)
|
|
||||||
}
|
|
||||||
// What it published and under what reuse key (novox/hq ADR 0408 §1).
|
|
||||||
if p := b.Published; p != nil {
|
|
||||||
if p.Artifact != "" {
|
|
||||||
fmt.Fprintf(w, " published %s\n", p.Artifact)
|
|
||||||
}
|
|
||||||
if p.Package != nil {
|
|
||||||
fmt.Fprintf(w, " published %s@%s\n", p.Package.Name, p.Package.Version)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if b.ReuseKey != "" {
|
|
||||||
fmt.Fprintf(w, " reuse key %s\n", b.ReuseKey)
|
|
||||||
}
|
|
||||||
if b.SourceFingerprint != "" {
|
|
||||||
fmt.Fprintf(w, " source %s\n", b.SourceFingerprint)
|
|
||||||
}
|
|
||||||
if !b.Worked() {
|
|
||||||
// The builder's own first line. The whole failure is often a build log, and printing
|
|
||||||
// it here would bury every other row.
|
|
||||||
fmt.Fprintf(w, " %s\n", firstLine(b.Failed))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// takenBuild is a build the controller judges and registers, whatever told it of the build: build-agent's outcome (the
|
|
||||||
// builder's own module's, until R4g, ADR 0453 §5) or the builder's done key (ADR 0501 §1). **One judging for both**:
|
|
||||||
// the rules of who may register, the gate a build failed, the predefined keys and the rule that an unchanged build
|
|
||||||
// source moves nothing are the same whichever part built it (ADR 0501 §1, §3).
|
|
||||||
type takenBuild struct {
|
|
||||||
kept inventory.Build
|
|
||||||
manifest catalogue.Manifest
|
|
||||||
recorded inventory.Source
|
|
||||||
ask registrationAsk
|
|
||||||
path string
|
|
||||||
trunk string
|
|
||||||
// by and what name the build in what is said: who built it, and what was built ("ace", "novox/mesh-catalog").
|
|
||||||
by, what, commit string
|
|
||||||
// published refuses a build that may not become a module's version (off the trunk, in the scratch namespace);
|
|
||||||
// nil for a key, which is a trunk fingerprint's by construction (ADR 0501 §2).
|
|
||||||
published func() error
|
|
||||||
// carried publishes a list of predefined keys the build carried; nil when it carried none.
|
|
||||||
carried func() error
|
|
||||||
}
|
|
||||||
|
|
||||||
// registerJudged judges a recorded build and registers its module, or says why not.
|
|
||||||
func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild) (catalogue.Manifest, error) {
|
|
||||||
manifest, kept := j.manifest, j.kept
|
|
||||||
notRegistered := func(err error) (catalogue.Manifest, error) {
|
|
||||||
return manifest, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", j.by, manifest.Module,
|
|
||||||
short(j.commit), err)
|
|
||||||
}
|
|
||||||
if err := namesNoInstallation(manifest); err != nil {
|
|
||||||
return manifest, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
|
||||||
j.by, j.what, short(j.commit), err)
|
|
||||||
}
|
|
||||||
// **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk
|
|
||||||
// below is the trunk of whatever repository was built, which may be one an agent made — and a module named
|
|
||||||
// `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal.
|
|
||||||
repoID, err := mayRegisterFrom(ctx, inv, manifest.Module, j.recorded, j.ask, j.path, j.trunk)
|
|
||||||
if err != nil {
|
|
||||||
return notRegistered(err)
|
|
||||||
}
|
|
||||||
if j.published != nil {
|
|
||||||
if err := j.published(); err != nil {
|
|
||||||
return notRegistered(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
|
||||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
|
||||||
// and the next push would send it.
|
|
||||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
|
||||||
return manifest, err
|
|
||||||
} else if failed {
|
|
||||||
return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
|
||||||
"back: it is recorded and not registered again — a newer build is", j.by, manifest.Module,
|
|
||||||
short(j.commit))
|
|
||||||
}
|
|
||||||
// Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a
|
|
||||||
// key that list does not hold, or of another type, is refused here and never enters the catalogue.
|
|
||||||
if j.carried != nil {
|
|
||||||
if err := j.carried(); err != nil {
|
|
||||||
return manifest, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if manifest.NamesPredefined() {
|
|
||||||
if _, err := catalogue.ParseManifest(kept.Manifest); err != nil {
|
|
||||||
return notRegistered(fmt.Errorf("judged against the list of predefined keys the mesh publishes, %w", err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
|
||||||
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
|
||||||
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
|
||||||
// byte-reproducible, and the bus rebuilt for another module's merge demanded a planned upgrade.
|
|
||||||
if kept.SourceFingerprint != "" {
|
|
||||||
stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID)
|
|
||||||
if err != nil {
|
|
||||||
return manifest, err
|
|
||||||
}
|
|
||||||
if stands != "" && stands != kept.ID && len(raw) > 0 {
|
|
||||||
if was, err := catalogue.ParseManifest(raw); err == nil && was.Module == manifest.Module {
|
|
||||||
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
|
|
||||||
manifest.Module, short(j.commit), stands)
|
|
||||||
manifest = was
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := inv.RegisterModule(ctx, manifest, j.recorded); err != nil {
|
|
||||||
if errors.Is(err, inventory.ErrSuperseded) {
|
|
||||||
return notRegistered(err)
|
|
||||||
}
|
|
||||||
return manifest, err
|
|
||||||
}
|
|
||||||
// A bus process the new definition gains gets its account now, on each machine assigned the module, so the
|
|
||||||
// next push composes a real account (issue 1226).
|
|
||||||
issueGainedProcessAccounts(ctx, inv, manifest)
|
|
||||||
// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
|
|
||||||
if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
|
|
||||||
return manifest, err
|
|
||||||
}
|
|
||||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
|
||||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
|
||||||
// worked and the module is registered.
|
|
||||||
collect(ctx, inv)
|
|
||||||
return manifest, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// answers is what the three questions came back with, read once.
|
|
||||||
type answers struct {
|
|
||||||
// notAnnounced are the owners some module reads the state of that have announced nothing yet (novox/hq ADR 0495,
|
|
||||||
// issue 1790): read from `reads` as before until they do; notAnnouncedWhy why that could not be read.
|
|
||||||
notAnnounced []string
|
|
||||||
notAnnouncedWhy string
|
|
||||||
wrong []inventory.Doing
|
|
||||||
nodes []inventory.Node
|
|
||||||
quiet []inventory.Node
|
|
||||||
behind map[string][]string
|
|
||||||
sources map[string]inventory.Source
|
|
||||||
// waiting is every machine not running what the mesh would send it.
|
|
||||||
waiting []inventory.Machine
|
|
||||||
// recordedWaits is, per machine, every move of a recorded module that waits there for a person (novox/hq ADR
|
|
||||||
// 0377 §6): the machine is behind for these, and `push <node>` sends them.
|
|
||||||
recordedWaits map[string][]string
|
|
||||||
// reported is every machine's last word beside when it was last sent a declaration — the
|
|
||||||
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
|
||||||
// recorded at send, not at apply).
|
|
||||||
reported []inventory.Reported
|
|
||||||
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
|
||||||
plans []inventory.Plan
|
|
||||||
// tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296).
|
|
||||||
tierBounds tierBounds
|
|
||||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
|
||||||
// other answer here: those are about a machine that was told something, and this is about one
|
|
||||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
|
||||||
// to compare against, so without this a wholly blocked mesh reads as a well one.
|
|
||||||
refused map[string]string
|
|
||||||
// network is why the private network could not be computed, when it could not. Almost always
|
|
||||||
// a consequence of the refusals above: a node that does not resolve is not on the network, and
|
|
||||||
// a mesh whose hub is that node has no hub.
|
|
||||||
network string
|
|
||||||
// filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
|
||||||
// 0168): what filters it beyond the mesh's own, the runtime's plumbing and bans, by name — a
|
|
||||||
// predecessor's chain, a found firewall in force again. Such a machine is not "all well".
|
|
||||||
filtered map[string]inventory.Filtering
|
|
||||||
// untaken is, per machine, each assigned module whose resources the machine is holding as it
|
|
||||||
// found them, and how many — a module that was assigned, sent, and is running none of what it
|
|
||||||
// declares because nothing has taken it (novox/hq ADR 0100, 04-ISSUES/125).
|
|
||||||
//
|
|
||||||
// **Its absence cost an outage.** The module was assigned, the push reported success, this
|
|
||||||
// command said the machine was doing everything it was told, and the module's three containers
|
|
||||||
// did not exist. On the strength of those reports the predecessor's proxy was stopped and every
|
|
||||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
|
||||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
|
||||||
untaken map[string]map[string]int
|
|
||||||
// leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue
|
|
||||||
// 380): assigned and not applied, which every push said only in passing. Not well while there is any.
|
|
||||||
leftOut map[string][]leftOutModule
|
|
||||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
|
||||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
|
||||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
|
||||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
|
||||||
unheld []catalogue.Unheld
|
|
||||||
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
|
|
||||||
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
|
|
||||||
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
|
|
||||||
// not be read when they could not — never read as none.
|
|
||||||
conditions []conditions.Condition
|
|
||||||
conditionsUnread string
|
|
||||||
// overflowing is every module whose identity overflows the bound of a provision it requires
|
|
||||||
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
|
||||||
// this is the one place it is said across the mesh. Not well while there is any.
|
|
||||||
overflowing []catalogue.Overflow
|
|
||||||
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
|
||||||
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
|
||||||
handActs *int
|
|
||||||
handActsUnread string
|
|
||||||
// heals is what the healers did in the last seven days (novox/hq to-be 45 §7): acts, and conditions
|
|
||||||
// handed to the operator; healsUnread why it could not be read.
|
|
||||||
heals *healsCount
|
|
||||||
healsUnread string
|
|
||||||
// pending is every assignment waiting for its module's build, and every one ended in the last day
|
|
||||||
// (novox/hq issue 325); pendingUnread why they could not be read. One waiting, or one ended other than
|
|
||||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
|
||||||
pending []inventory.PendingAssignment
|
|
||||||
pendingUnread string
|
|
||||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
|
||||||
// issue 485); lastSentUnread why it could not be read.
|
|
||||||
lastSent map[string]inventory.Send
|
|
||||||
lastSentUnread string
|
|
||||||
}
|
|
||||||
@@ -1,131 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
|
||||||
m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{
|
|
||||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
|
||||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
|
||||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
|
||||||
}))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if m.Module != "shop" {
|
|
||||||
t.Fatalf("registered %q", m.Module)
|
|
||||||
}
|
|
||||||
shelf, err := open.inventory.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, held := shelf["shop"]; !held {
|
|
||||||
t.Fatal("the module a heard build produced is not in the catalogue")
|
|
||||||
}
|
|
||||||
src, err := open.inventory.SourceOf(ctx, "shop")
|
|
||||||
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
|
|
||||||
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
|
|
||||||
}
|
|
||||||
builds, err := open.inventory.Builds(ctx, "shop", 5)
|
|
||||||
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
|
|
||||||
t.Fatalf("the build is not recorded once: %v %v", builds, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
|
|
||||||
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
|
|
||||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
|
||||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
|
||||||
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
|
||||||
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
|
||||||
}
|
|
||||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
|
||||||
t.Fatal("the refused module was registered anyway")
|
|
||||||
}
|
|
||||||
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
|
|
||||||
t.Fatalf("the refused build was not recorded: %v", builds)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "no compiler") {
|
|
||||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
|
||||||
result := func(id, ref, commit string) link.BuildResult {
|
|
||||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
|
||||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
|
||||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
|
||||||
}
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
|
||||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
|
||||||
}
|
|
||||||
|
|
||||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
|
||||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
|
||||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
|
||||||
r.Manifest, r.Path = other, "modules/letta"
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
|
||||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
|
||||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
|
||||||
result := func(asked time.Time, image string) link.BuildResult {
|
|
||||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
|
||||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
|
||||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
|
||||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
|
||||||
}
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
|
||||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
|
||||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
|
||||||
}
|
|
||||||
shelf, err := open.inventory.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
|
||||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
|
||||||
}
|
|
||||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
|
||||||
t.Errorf("the late build was not recorded: %v", builds)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,706 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"regexp"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Where a build may register a module from (novox/hq ADR 0266).
|
|
||||||
//
|
|
||||||
// A build's outcome registers its module, and a registered module is what the next push sends. Before this,
|
|
||||||
// an outcome registered whatever its manifest named, from whichever repository it was built from: an agent
|
|
||||||
// that made a repository of its own, committed `modules/sudo/module.json` granting itself a rule without a
|
|
||||||
// password, and asked the `build` verb for it had its repository registered as the module `sudo` — whose next
|
|
||||||
// push made the agent root on every node. A fork of the node-engine did the same everywhere. The trunk rule
|
|
||||||
// (ADR 0238) did not stop it: the trunk it checked was the trunk of the repository built, which was the
|
|
||||||
// agent's own.
|
|
||||||
//
|
|
||||||
// So **an outcome registers a module only from the repository the catalogue already builds that module
|
|
||||||
// from**; and a module new to the catalogue only from a repository the catalogue already builds another
|
|
||||||
// module from — whose trunk takes a reviewed, approved merge, which is how a merge adds a module (novox/hq
|
|
||||||
// issue 300). Anything else — a module moved to another repository, a module from a repository the
|
|
||||||
// catalogue has never built — is the operator's, at the controller's terminal: allowed only when the build
|
|
||||||
// request was kept as asked there. Judged at the take-in, which every outcome reaches whoever hears it and
|
|
||||||
// whichever verb asked it (`build`, `rebuild`, `replay --register`, `assign` with build), and before the ask
|
|
||||||
// for a call through a verb, so an agent cannot have a build node run a repository the catalogue does not
|
|
||||||
// build from at all.
|
|
||||||
|
|
||||||
// errNotItsSource is an outcome refused for where it was built from.
|
|
||||||
var errNotItsSource = errors.New("not built from the repository the catalogue builds it from")
|
|
||||||
|
|
||||||
// servedVar marks every process the serving controller starts — each verb's command, each child — and the serving
|
|
||||||
// process itself, so none of them can read as the operator at the terminal (novox/hq ADR 0266). Set by serve
|
|
||||||
// before it answers anything, inherited by every child through os.Environ.
|
|
||||||
const servedVar = "MESH_SERVED_BY_THE_CONTROLLER"
|
|
||||||
|
|
||||||
// startedAtTheTerminal says this process was started at the controller's terminal: not the serving controller,
|
|
||||||
// not anything it started, not a verb's command, not a seat call's. The serving controller marks its own
|
|
||||||
// environment (servedVar), so a build asked in it, or by any process it starts, never reads as the terminal's;
|
|
||||||
// runVerb also names the verb and the caller.
|
|
||||||
//
|
|
||||||
// **And a line mesh-cli asked as the controller's terminal** (novox/hq ADR 0272 §4): the serving controller runs it
|
|
||||||
// without the served mark and without a verb, names its caller, and marks it with cliTerminalVar — a mark only the
|
|
||||||
// mesh-cli path sets and every other command line the controller runs is stripped of (commandEnvironment).
|
|
||||||
func startedAtTheTerminal() bool {
|
|
||||||
if os.Getenv(servedVar) != "" || os.Getenv(verbVar) != "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return os.Getenv(link.CallerVar) == "" || os.Getenv(cliTerminalVar) != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// cliTerminalVar marks a command line the serving controller runs as the controller's terminal for mesh-cli.
|
|
||||||
const cliTerminalVar = "MESH_CLI_TERMINAL"
|
|
||||||
|
|
||||||
// markServed marks this process, and so everything it starts, as the serving controller's.
|
|
||||||
func markServed() {
|
|
||||||
if err := os.Setenv(servedVar, "1"); err != nil {
|
|
||||||
panic("the serving controller could not mark its environment: " + err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceForms compares sources however each is spelled: a path on a seat's holder (ADR 0111) or a URL — a
|
|
||||||
// build asked of a seat's path is registered with the URL composed from it when its outcome does not echo
|
|
||||||
// the seat. A seat's path is composed into its URL where the seat's holder is known, so both spellings of
|
|
||||||
// one repository are one; where it is not, a seat's path matches only the same seat's same path.
|
|
||||||
type sourceForms struct {
|
|
||||||
bases map[string]string // the seat's clone base, `scheme://host:port`, by seat; "" where it is not known
|
|
||||||
base func(seat string) string
|
|
||||||
// read is base with the reason it is unknown; nil where only base is given. again forces a fresh read past a
|
|
||||||
// failure kept for failedKept.
|
|
||||||
read func(ctx context.Context, seat string, again bool) (string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// failedKept is how long an unreadable base is not read again for the callers that loop over every catalogue
|
|
||||||
// entry: one composition of the mesh serves them all (novox/hq issue 1277).
|
|
||||||
const failedKept = 5 * time.Second
|
|
||||||
|
|
||||||
// newSourceForms reads the seats' bases from the mesh when first needed. A base that could not be read is not
|
|
||||||
// kept as if it were known: it is read again at the next ask (novox/hq issue 1277).
|
|
||||||
func newSourceForms(ctx context.Context, inv *inventory.Inventory) *sourceForms {
|
|
||||||
return sourceFormsRead(ctx, time.Now, func(ctx context.Context) (catalogue.World, error) {
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.World{}, fmt.Errorf("the catalogue cannot be read: %w", err)
|
|
||||||
}
|
|
||||||
read, err := restOfTheMeshOf(ctx, inv, shelf, "")
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.World{}, fmt.Errorf("the mesh cannot be read: %w", err)
|
|
||||||
}
|
|
||||||
return read, nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceFormsRead is newSourceForms on how the mesh is read, handed in.
|
|
||||||
func sourceFormsRead(ctx context.Context, now func() time.Time,
|
|
||||||
readWorld func(context.Context) (catalogue.World, error)) *sourceForms {
|
|
||||||
f := &sourceForms{bases: map[string]string{}}
|
|
||||||
var failedAt time.Time // when the last read failed; zero when it did not
|
|
||||||
var failure error
|
|
||||||
f.read = func(ctx context.Context, seat string, again bool) (string, error) {
|
|
||||||
if b, known := f.bases[seat]; known {
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
if failure != nil && !again && now().Sub(failedAt) < failedKept {
|
|
||||||
return "", failure
|
|
||||||
}
|
|
||||||
world, err := readWorld(ctx)
|
|
||||||
if err == nil {
|
|
||||||
var b string
|
|
||||||
if b, err = seatBase(world, seat); err == nil {
|
|
||||||
failure = nil
|
|
||||||
f.bases[seat] = b
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
failedAt, failure = now(), err // the holder may be changing now: only an unread base is kept, for a moment
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
f.base = func(seat string) string {
|
|
||||||
b, _ := f.read(ctx, seat, false)
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeAddressWait is the registration's wait for the forge's address (novox/hq issue 1277): a fresh read every
|
|
||||||
// 5 s for at most 60 s, as the wait for the forge's answer (issue 486). A variable so a test waits in no real time.
|
|
||||||
var forgeAddressWait = waitOut{bound: 60 * time.Second, first: 5 * time.Second, most: 5 * time.Second,
|
|
||||||
now: time.Now, sleep: sleepOrDone}
|
|
||||||
|
|
||||||
// forgeAddress is the git seat's base, read again within w's bound while it cannot be read. Where the forms have
|
|
||||||
// no reader of their own (a test's), base says it.
|
|
||||||
func (f *sourceForms) forgeAddress(ctx context.Context, w waitOut) (string, error) {
|
|
||||||
if f.read == nil {
|
|
||||||
return f.base(gitSeat), nil
|
|
||||||
}
|
|
||||||
began := w.now()
|
|
||||||
for {
|
|
||||||
b, err := f.read(ctx, gitSeat, true)
|
|
||||||
if err == nil {
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if w.now().Sub(began)+w.first > w.bound {
|
|
||||||
return "", fmt.Errorf("after %s: %w", w.now().Sub(began).Round(time.Second), err)
|
|
||||||
}
|
|
||||||
fmt.Printf("the forge's address cannot be read now (%v); reading it again in %s, for at most %s in all "+
|
|
||||||
"(novox/hq issue 1277)\n", err, w.first, w.bound)
|
|
||||||
if serr := w.sleep(ctx, w.first); serr != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// canonical is one spelling of a repository: lower case, no `.git`, no trailing slash, and a seat's path as
|
|
||||||
// the URL its holder serves it at where that is known.
|
|
||||||
func (f *sourceForms) canonical(repository, seat string) string {
|
|
||||||
trim := func(s string) string {
|
|
||||||
s = strings.TrimSpace(strings.ToLower(s))
|
|
||||||
s = strings.TrimRight(s, "/")
|
|
||||||
return strings.TrimRight(strings.TrimSuffix(s, ".git"), "/")
|
|
||||||
}
|
|
||||||
if seat == "" {
|
|
||||||
return trim(repository)
|
|
||||||
}
|
|
||||||
if b := f.base(seat); b != "" {
|
|
||||||
return trim(b + "/" + strings.Trim(repository, "/"))
|
|
||||||
}
|
|
||||||
return "seat:" + seat + ":" + trim(strings.Trim(repository, "/"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// same says two sources are one repository.
|
|
||||||
func (f *sourceForms) same(aRepository, aSeat, bRepository, bSeat string) bool {
|
|
||||||
if aRepository == "" || bRepository == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return f.canonical(aRepository, aSeat) == f.canonical(bRepository, bSeat)
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeFacts is what the mesh's forge says of a repository a module is registered from (novox/hq ADR 0266).
|
|
||||||
type forgeFacts struct {
|
|
||||||
// ID is the forge's own id of the repository: what tells it from one deleted and made again by its name.
|
|
||||||
ID int64
|
|
||||||
// Guarded is whether the branch is protected as a module's trunk must be: no direct push, a merge whitelist naming
|
|
||||||
// the tools' account and no agent's account, and no administrator merging past the rule (novox/hq ADR 0534 §6; no
|
|
||||||
// required status, which the forge no longer asks, §1). Why says what is missing when it is not.
|
|
||||||
Guarded bool
|
|
||||||
Why string
|
|
||||||
}
|
|
||||||
|
|
||||||
// askTheForge asks the forge, through its module's tools on the bus, for a repository's id and its branch's
|
|
||||||
// protection, waiting out a forge not served for a moment (novox/hq issue 486). A variable so a test needs no
|
|
||||||
// forge.
|
|
||||||
var askTheForge = func(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
|
||||||
return askedWithinBudget(ctx, owner+"/"+repo, func(ctx context.Context) (forgeFacts, error) {
|
|
||||||
return askTheForgeOnce(ctx, owner, repo, branch)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
type forgeBudgetKey struct{}
|
|
||||||
|
|
||||||
// withForgeBudget says how much of the registration's one wait is left for the forge's answer: the wait for its
|
|
||||||
// address has used the rest (novox/hq issue 1277). The registration holds the controller's one receive loop, so the
|
|
||||||
// two waits together keep to one bound.
|
|
||||||
func withForgeBudget(ctx context.Context, left time.Duration) context.Context {
|
|
||||||
return context.WithValue(ctx, forgeBudgetKey{}, max(left, 0))
|
|
||||||
}
|
|
||||||
|
|
||||||
// askedWithinBudget is forgeWaitedOut with forgeWait's bound, or what is left of it when the address was waited for.
|
|
||||||
func askedWithinBudget(ctx context.Context, repository string, ask func(context.Context) (forgeFacts, error)) (forgeFacts, error) {
|
|
||||||
w := forgeWait
|
|
||||||
if left, ok := ctx.Value(forgeBudgetKey{}).(time.Duration); ok {
|
|
||||||
w.bound = min(w.bound, left)
|
|
||||||
}
|
|
||||||
return forgeWaitedOut(ctx, w, repository, ask)
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitOut bounds how long a question is asked again while its answer is "not now": a pause of first, doubling
|
|
||||||
// to most, for at most bound in all. now and sleep are the clock, so a test waits in no real time.
|
|
||||||
type waitOut struct {
|
|
||||||
bound, first, most time.Duration
|
|
||||||
now func() time.Time
|
|
||||||
sleep func(context.Context, time.Duration) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeWait is the registration's wait for the forge (novox/hq issue 486). On 2026-10-11 a build's registration
|
|
||||||
// asked the forge while node-tools on the control-node re-served after a rollout of the tool runner, heard
|
|
||||||
// "nothing serves gitea.gitea_api", and failed its plan; the forge answered again seconds later. A re-serve takes
|
|
||||||
// about 10 s, so 60 s waits one out with room to spare. It is a cap on the whole wait, an ask in flight included:
|
|
||||||
// the registration holds the controller's one receive loop while it waits, and a loop held two minutes reads as
|
|
||||||
// deaf. The pauses are those the build machine waits the registry with (novox/hq issue 457).
|
|
||||||
var forgeWait = waitOut{bound: 60 * time.Second, first: time.Second, most: 10 * time.Second,
|
|
||||||
now: time.Now, sleep: sleepOrDone}
|
|
||||||
|
|
||||||
// sleepOrDone pauses for d, or less when ctx ends first.
|
|
||||||
func sleepOrDone(ctx context.Context, d time.Duration) error {
|
|
||||||
t := time.NewTimer(d)
|
|
||||||
defer t.Stop()
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
case <-t.C:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeNotNow is an answer that says the forge's tool is not served now — the tool runner re-serving — not that
|
|
||||||
// the forge refused. Only that is asked again: a refusal, an unreadable answer or a repository with no id is the
|
|
||||||
// forge's word, and a tool that did not answer in time has already held the receive loop its whole wait.
|
|
||||||
func forgeNotNow(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
|
||||||
|
|
||||||
// forgeWaitedOut asks the forge about repository, asking again while it answers "not served now", all of it
|
|
||||||
// within w's bound, an ask in flight included (novox/hq issue 486). Past the bound it fails, saying it waited,
|
|
||||||
// for how long, and what the forge last said; when the caller's own context ends, it says that instead.
|
|
||||||
func forgeWaitedOut(ctx context.Context, w waitOut, repository string,
|
|
||||||
ask func(context.Context) (forgeFacts, error)) (forgeFacts, error) {
|
|
||||||
capped, cancel := context.WithTimeout(ctx, w.bound)
|
|
||||||
defer cancel()
|
|
||||||
began := w.now()
|
|
||||||
pause := w.first
|
|
||||||
for asked := 1; ; asked++ {
|
|
||||||
facts, err := ask(capped)
|
|
||||||
if err == nil {
|
|
||||||
if asked > 1 {
|
|
||||||
fmt.Printf("the forge answered about %s again, after %s (novox/hq issue 486)\n", repository,
|
|
||||||
w.now().Sub(began).Round(time.Second))
|
|
||||||
}
|
|
||||||
return facts, nil
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return forgeFacts{}, err // the caller's own context ended: no wait of ours to speak of
|
|
||||||
}
|
|
||||||
gaveUp := func() (forgeFacts, error) {
|
|
||||||
return forgeFacts{}, fmt.Errorf("waited %s for the forge to answer about %s, asking %d times, and it "+
|
|
||||||
"did not (novox/hq issue 486): %w", w.now().Sub(began).Round(time.Second), repository, asked, err)
|
|
||||||
}
|
|
||||||
if capped.Err() != nil {
|
|
||||||
return gaveUp() // the bound ended an ask in flight
|
|
||||||
}
|
|
||||||
if !forgeNotNow(err) {
|
|
||||||
return forgeFacts{}, err
|
|
||||||
}
|
|
||||||
if w.now().Sub(began)+pause > w.bound {
|
|
||||||
return gaveUp()
|
|
||||||
}
|
|
||||||
fmt.Printf("the forge cannot be asked about %s now (%v); asking again in %s, for at most %s in all "+
|
|
||||||
"(novox/hq issue 486)\n", repository, err, pause, w.bound)
|
|
||||||
if serr := w.sleep(capped, pause); serr != nil {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository,
|
|
||||||
ctx.Err())
|
|
||||||
}
|
|
||||||
return gaveUp()
|
|
||||||
}
|
|
||||||
pause = min(2*pause, w.most)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// askTheForgeOnce asks the forge once, through its module's tools on the bus.
|
|
||||||
func askTheForgeOnce(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
|
|
||||||
js, err := aBus()
|
|
||||||
if err != nil {
|
|
||||||
return forgeFacts{}, err
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
bus := link.OverNATS{Conn: js.Conn()}
|
|
||||||
ask := func(tool string, args map[string]any, into any) error {
|
|
||||||
raw, _ := json.Marshal(args)
|
|
||||||
answer, err := link.Ask(ctx, bus, "gitea", tool, raw, 30*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if answer.Error != "" {
|
|
||||||
return fmt.Errorf("gitea.%s: %s", tool, answer.Error)
|
|
||||||
}
|
|
||||||
return json.Unmarshal(answer.Result, into)
|
|
||||||
}
|
|
||||||
var found struct {
|
|
||||||
Result struct {
|
|
||||||
ID int64 `json:"id"`
|
|
||||||
FullName string `json:"full_name"`
|
|
||||||
} `json:"result"`
|
|
||||||
}
|
|
||||||
if err := ask("gitea_api", map[string]any{"path": "/repos/" + owner + "/" + repo}, &found); err != nil {
|
|
||||||
return forgeFacts{}, err
|
|
||||||
}
|
|
||||||
if found.Result.ID == 0 {
|
|
||||||
return forgeFacts{}, fmt.Errorf("the forge named no id for %s/%s", owner, repo)
|
|
||||||
}
|
|
||||||
var rules struct {
|
|
||||||
Rules []protectionRule `json:"rules"`
|
|
||||||
}
|
|
||||||
if err := ask("gitea_branch_protection_get", map[string]any{"owner": owner, "repo": repo}, &rules); err != nil {
|
|
||||||
return forgeFacts{}, err
|
|
||||||
}
|
|
||||||
facts := judgedRule(rules.Rules, branch)
|
|
||||||
facts.ID = found.Result.ID
|
|
||||||
return facts, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeToolsAccount is the forge tools' account, which main's merge whitelist names for mesh-delivery (novox/hq ADR 0398
|
|
||||||
// §2, the gitea module's tools-account.ts TOOLS_ACCOUNT); beside it the whitelist names only people's own logins, the
|
|
||||||
// operator's (ADR 0534 §2), never an agent's account.
|
|
||||||
const forgeToolsAccount = "mesh-tools"
|
|
||||||
|
|
||||||
// isAgentAccount is whether a forge login is a node's agent's account, agent-<node> (novox/hq ADR 0398 §1): one that
|
|
||||||
// may never merge into a module's trunk by itself.
|
|
||||||
func isAgentAccount(login string) bool { return strings.HasPrefix(login, "agent-") }
|
|
||||||
|
|
||||||
// protectionRule is a branch protection rule as the forge's tool summarises it.
|
|
||||||
type protectionRule struct {
|
|
||||||
Rule string `json:"rule"`
|
|
||||||
Push bool `json:"push"`
|
|
||||||
RequiredStatuses []string `json:"required_statuses"`
|
|
||||||
AdminMayOverride bool `json:"admin_may_override"`
|
|
||||||
// MergeWhitelist is who may merge into the branch; nil when the rule keeps no merge whitelist (anyone who writes).
|
|
||||||
MergeWhitelist []string `json:"merge_whitelist"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgedRule judges the one rule the forge applies to a branch, as the forge picks it: the rule named for the
|
|
||||||
// branch, else the first glob rule, in the forge's order, that covers it (gitea's first matching rule). Never a
|
|
||||||
// later rule that happens to be stronger: the forge does not read it. A trunk is trusted for who can land a commit on
|
|
||||||
// it, not for a status it requires (novox/hq ADR 0534 §6): no push, a merge whitelist naming the tools' account and
|
|
||||||
// beside it only people's own logins, never an agent's, and no administrator merging past the rule; then a commit lands
|
|
||||||
// there through mesh-delivery, on a passing mesh/check and the operator's Approve (ADR 0534 §4), or by the operator's
|
|
||||||
// own hand at the forge (§2).
|
|
||||||
func judgedRule(rules []protectionRule, branch string) forgeFacts {
|
|
||||||
switch r := appliedRule(rules, branch); {
|
|
||||||
case r == nil:
|
|
||||||
return forgeFacts{Why: fmt.Sprintf("no protection rule covers %s", branch)}
|
|
||||||
case r.Push:
|
|
||||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets a person push to %s directly", r.Rule, branch)}
|
|
||||||
case r.MergeWhitelist == nil || len(r.MergeWhitelist) == 0:
|
|
||||||
return forgeFacts{Why: fmt.Sprintf("the rule %s keeps no merge whitelist naming %s alone for %s", r.Rule, forgeToolsAccount, branch)}
|
|
||||||
case !slices.Contains(r.MergeWhitelist, forgeToolsAccount):
|
|
||||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets %s merge into %s, and not %s", r.Rule,
|
|
||||||
strings.Join(r.MergeWhitelist, ", "), branch, forgeToolsAccount)}
|
|
||||||
case slices.ContainsFunc(r.MergeWhitelist, isAgentAccount):
|
|
||||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets an agent's account merge into %s: %s", r.Rule, branch,
|
|
||||||
strings.Join(r.MergeWhitelist, ", "))}
|
|
||||||
case r.AdminMayOverride:
|
|
||||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets an administrator merge into %s past it", r.Rule, branch)}
|
|
||||||
}
|
|
||||||
return forgeFacts{Guarded: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
// appliedRule is the one rule the forge applies to a branch: the rule named for it, else the first glob rule, in the
|
|
||||||
// forge's order, that covers it; nil when none does.
|
|
||||||
func appliedRule(rules []protectionRule, branch string) *protectionRule {
|
|
||||||
for i := range rules {
|
|
||||||
if rules[i].Rule == branch {
|
|
||||||
return &rules[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := range rules {
|
|
||||||
if ruleCovers(rules[i].Rule, branch) {
|
|
||||||
return &rules[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ruleCovers says a protection rule's name — a branch, or a glob of them — covers a branch, as the forge reads it.
|
|
||||||
func ruleCovers(rule, branch string) bool {
|
|
||||||
if rule == branch {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if !strings.ContainsAny(rule, "*?[") {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
var re strings.Builder
|
|
||||||
re.WriteString("^")
|
|
||||||
for i := 0; i < len(rule); i++ {
|
|
||||||
switch c := rule[i]; {
|
|
||||||
case c == '*' && i+1 < len(rule) && rule[i+1] == '*':
|
|
||||||
re.WriteString(".*")
|
|
||||||
i++
|
|
||||||
case c == '*':
|
|
||||||
re.WriteString("[^/]*")
|
|
||||||
case c == '?':
|
|
||||||
re.WriteString("[^/]")
|
|
||||||
default:
|
|
||||||
re.WriteString(regexp.QuoteMeta(string(c)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
re.WriteString("$")
|
|
||||||
ok, _ := regexp.MatchString(re.String(), branch)
|
|
||||||
return ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// onTheForge is a source's owner and name on the mesh's own forge (the git seat's holder), and whether it is
|
|
||||||
// there at all.
|
|
||||||
func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, ok bool) {
|
|
||||||
var rest string
|
|
||||||
switch {
|
|
||||||
case seat == gitSeat:
|
|
||||||
rest = strings.Trim(repository, "/")
|
|
||||||
case seat == "":
|
|
||||||
base := f.base(gitSeat)
|
|
||||||
if base == "" {
|
|
||||||
return "", "", false
|
|
||||||
}
|
|
||||||
url, prefix := f.canonical(repository, ""), f.canonical(base, "")+"/"
|
|
||||||
if !strings.HasPrefix(url, prefix) {
|
|
||||||
return "", "", false
|
|
||||||
}
|
|
||||||
// The case the forge spells it with: the URL as given, past the base.
|
|
||||||
rest = strings.TrimSuffix(strings.Trim(repository[len(prefix):], "/"), ".git")
|
|
||||||
default:
|
|
||||||
return "", "", false
|
|
||||||
}
|
|
||||||
parts := strings.Split(rest, "/")
|
|
||||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
|
||||||
return "", "", false
|
|
||||||
}
|
|
||||||
return parts[0], parts[1], true
|
|
||||||
}
|
|
||||||
|
|
||||||
// mayRegisterFrom says whether a build's outcome may register module from the source it was built from, and the
|
|
||||||
// forge's id of that repository to record (novox/hq ADR 0266). Through any verb, only:
|
|
||||||
//
|
|
||||||
// - from the module's registered repository — the same repository by the forge's own id, not only its name; or,
|
|
||||||
// for a module new to the catalogue, from a repository the catalogue builds another module from;
|
|
||||||
// - and from a repository on the mesh's forge whose trunk is protected as a trunk must be: no direct push, at
|
|
||||||
// a merge whitelist naming the tools' account and no agent's, no administrator merging past the rule (ADR 0534 §6).
|
|
||||||
//
|
|
||||||
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
|
||||||
// repository and path; or, for the build of a merge, when mesh-delivery announced it on the operator's Approve of
|
|
||||||
// that head (novox/hq ADR 0420 §7 for a moved module, ADR 0432 for a new one), the forge's checks still holding. path is the module's directory as built; branch the trunk it is registered from.
|
|
||||||
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
|
||||||
ask registrationAsk, path, branch string) (int64, error) {
|
|
||||||
return mayRegisterIn(ctx, inv, newSourceForms(ctx, inv), module, built, ask, path, branch)
|
|
||||||
}
|
|
||||||
|
|
||||||
// registrationAsk is what a registration's warrant is read from (novox/hq ADR 0501 §3): the build an outcome answers,
|
|
||||||
// whose kept request says whether it was asked at the terminal or for a merge (build-agent's outcome, the builder's own
|
|
||||||
// module until R4g); or, for a take-in from the builder's done key, the merge the indexer's trunk value names and
|
|
||||||
// whether the operator asked the registration at the controller's terminal (ADR 0501 §4). A key asks no build, so it
|
|
||||||
// keeps no request.
|
|
||||||
type registrationAsk struct {
|
|
||||||
// Build is the build whose kept request is read; empty for a take-in from a key.
|
|
||||||
Build string
|
|
||||||
// Merge is the commit of the merge a key's module is registered from (the indexer's trunk value); empty for an
|
|
||||||
// outcome, whose request names it.
|
|
||||||
Merge string
|
|
||||||
// Terminal is a key registration the operator asked at the controller's terminal for this repository and path.
|
|
||||||
Terminal bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// String is the ask as the registration's words name it.
|
|
||||||
func (a registrationAsk) String() string {
|
|
||||||
switch {
|
|
||||||
case a.Build != "":
|
|
||||||
return a.Build
|
|
||||||
case a.Terminal:
|
|
||||||
return "the registration asked at the terminal"
|
|
||||||
case a.Merge != "":
|
|
||||||
return fmt.Sprintf("the merge %.8s", a.Merge)
|
|
||||||
}
|
|
||||||
return "a key"
|
|
||||||
}
|
|
||||||
|
|
||||||
// catalogueReader is the catalogue's manifests, as the router's records are found in them.
|
|
||||||
type catalogueReader interface {
|
|
||||||
Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// registrations is what a registration's decision reads of the mesh: *inventory.Inventory, or a test's stand-in.
|
|
||||||
type registrations interface {
|
|
||||||
catalogueReader
|
|
||||||
SourceOf(ctx context.Context, module string) (inventory.Source, error)
|
|
||||||
SourceIdentity(ctx context.Context, module string) (int64, error)
|
|
||||||
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
|
||||||
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
|
||||||
MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
|
||||||
func mayRegisterIn(ctx context.Context, inv registrations, forms *sourceForms, module string, built inventory.Source,
|
|
||||||
ask registrationAsk, path, branch string) (int64, error) {
|
|
||||||
buildID := ask.String()
|
|
||||||
was, err := inv.SourceOf(ctx, module)
|
|
||||||
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
|
||||||
if err != nil && !isNew {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
terminal := ask.Terminal
|
|
||||||
if ask.Build != "" {
|
|
||||||
var err error
|
|
||||||
if terminal, err = askedHereFor(ctx, inv, forms, ask.Build, built, path); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
refuse := func(why string) (int64, error) {
|
|
||||||
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
|
||||||
"controller's terminal, or by the build of a merge mesh-delivery announced on the operator's Approve of that "+
|
|
||||||
"head, never through a verb: a registered module is what the next push sends, and whoever "+
|
|
||||||
"may call a verb includes agents (novox/hq ADR 0266)", errNotItsSource, why)
|
|
||||||
}
|
|
||||||
|
|
||||||
var why string
|
|
||||||
var alongside []inventory.Entry // the modules a new one's repository already builds
|
|
||||||
switch {
|
|
||||||
case !isNew && forms.same(was.Repository, was.Seat, built.Repository, built.Seat):
|
|
||||||
case !isNew:
|
|
||||||
registered := was.Repository
|
|
||||||
if registered == "" {
|
|
||||||
registered = "no repository (it was handed over by hand)"
|
|
||||||
}
|
|
||||||
why = fmt.Sprintf("%s is built from %s, and this build is of %s", module, sourceWords(registered, was.Seat),
|
|
||||||
sourceWords(built.Repository, built.Seat))
|
|
||||||
default:
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
if !e.Provided && e.Source.Repository != "" &&
|
|
||||||
forms.same(e.Source.Repository, e.Source.Seat, built.Repository, built.Seat) {
|
|
||||||
alongside = append(alongside, e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(alongside) == 0 {
|
|
||||||
why = fmt.Sprintf("%s is new to the catalogue, and %s is no repository the catalogue builds a module from",
|
|
||||||
module, sourceWords(built.Repository, built.Seat))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **Or the move the operator approved** (novox/hq ADR 0420 §7, issue 1025): a known module built from another
|
|
||||||
// repository by a merge, whose pull request mesh-delivery merged on the operator's Approve of that head.
|
|
||||||
// A verb alone, or a merge with no such warrant, is refused as before.
|
|
||||||
// **Or a new module from a repository nothing registered is built from, on the same warrant** (novox/hq ADR 0432,
|
|
||||||
// issue 1146): the build of a merge mesh-delivery announced on the operator's Approve of that head registers it.
|
|
||||||
// The forge's protection and the repository's identity are checked below as for every registration.
|
|
||||||
movedWhy := ""
|
|
||||||
if why != "" && !terminal {
|
|
||||||
accepted, err := approvedMove(ctx, inv, forms, ask, built)
|
|
||||||
switch {
|
|
||||||
case err == nil:
|
|
||||||
movedWhy, why = accepted, ""
|
|
||||||
case !errors.Is(err, errNotAMerge):
|
|
||||||
fmt.Printf("%s: %s is built from another repository, and no approved move covers this build: %v\n",
|
|
||||||
buildID, module, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if why != "" && !terminal {
|
|
||||||
return refuse(why)
|
|
||||||
}
|
|
||||||
|
|
||||||
if built.Seat == "" {
|
|
||||||
// A URL is judged against the forge's address: an address that cannot be read is no proof the source is
|
|
||||||
// elsewhere (novox/hq issue 1277).
|
|
||||||
began := forgeAddressWait.now()
|
|
||||||
addressCtx, cancel := context.WithTimeout(ctx, forgeAddressWait.bound)
|
|
||||||
_, err := forms.forgeAddress(addressCtx, forgeAddressWait)
|
|
||||||
cancel()
|
|
||||||
ctx = withForgeBudget(ctx, forgeAddressWait.bound-forgeAddressWait.now().Sub(began))
|
|
||||||
if err != nil {
|
|
||||||
if terminal {
|
|
||||||
fmt.Printf("%s: the forge's address could not be read (%v) — registered, as asked at the controller's "+
|
|
||||||
"terminal\n", buildID, err)
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
return refuse(fmt.Sprintf("the forge's address could not be read (%v), so whether %s's trunk is "+
|
|
||||||
"protected cannot be read", err, sourceWords(built.Repository, built.Seat)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
owner, name, onForge := forms.onTheForge(built.Repository, built.Seat)
|
|
||||||
if !onForge {
|
|
||||||
if terminal {
|
|
||||||
fmt.Printf("%s: %s is not on the mesh's forge — registered, as asked at the controller's terminal\n",
|
|
||||||
buildID, sourceWords(built.Repository, built.Seat))
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
return refuse(fmt.Sprintf("%s is not on the mesh's forge, so whether its trunk is protected cannot be read",
|
|
||||||
sourceWords(built.Repository, built.Seat)))
|
|
||||||
}
|
|
||||||
facts, err := askTheForge(ctx, owner, name, branch)
|
|
||||||
if err != nil {
|
|
||||||
if terminal {
|
|
||||||
fmt.Printf("%s: the forge could not be asked about %s/%s (%v) — registered, as asked at the controller's "+
|
|
||||||
"terminal\n", buildID, owner, name, err)
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
return refuse(fmt.Sprintf("the forge could not say whether %s/%s's %s is protected: %v", owner, name, branch, err))
|
|
||||||
}
|
|
||||||
if terminal {
|
|
||||||
if why != "" || !facts.Guarded {
|
|
||||||
fmt.Printf("%s: %s — registered, as asked at the controller's terminal\n", buildID,
|
|
||||||
strings.Trim(why+"; "+facts.Why, "; "))
|
|
||||||
}
|
|
||||||
return facts.ID, nil
|
|
||||||
}
|
|
||||||
if !facts.Guarded {
|
|
||||||
return refuse(fmt.Sprintf("%s/%s's %s is not protected as a module's trunk must be: %s", owner, name, branch,
|
|
||||||
facts.Why))
|
|
||||||
}
|
|
||||||
// The same repository by the forge's id, not only its name: one deleted and made again is another.
|
|
||||||
recorded := map[string]int64{}
|
|
||||||
// A moved module is registered from the repository the operator approved, not the one it was registered from.
|
|
||||||
if !isNew && movedWhy == "" {
|
|
||||||
id, err := inv.SourceIdentity(ctx, module)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
recorded[module] = id
|
|
||||||
}
|
|
||||||
for _, e := range alongside {
|
|
||||||
id, err := inv.SourceIdentity(ctx, e.Manifest.Module)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
recorded[e.Manifest.Module] = id
|
|
||||||
}
|
|
||||||
for m, id := range recorded {
|
|
||||||
if id != 0 && id != facts.ID {
|
|
||||||
return refuse(fmt.Sprintf("%s/%s is not the repository %s was registered from: the forge knows it as "+
|
|
||||||
"repository %d, and %s was registered from repository %d — one of that name deleted and made again",
|
|
||||||
owner, name, m, facts.ID, m, id))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if movedWhy != "" && isNew {
|
|
||||||
fmt.Printf("%s: %s is new, from %s/%s — registered at the operator's Approve: %s\n", buildID, module, owner, name,
|
|
||||||
movedWhy)
|
|
||||||
} else if movedWhy != "" {
|
|
||||||
fmt.Printf("%s: %s moved from %s to %s/%s — registered at the operator's Approve: %s\n", buildID, module,
|
|
||||||
sourceWords(was.Repository, was.Seat), owner, name, movedWhy)
|
|
||||||
}
|
|
||||||
return facts.ID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// askedHereFor says the build was asked at the controller's terminal, for this repository and this path: a kept
|
|
||||||
// request marked so, whose source is the outcome's (novox/hq ADR 0266).
|
|
||||||
func askedHereFor(ctx context.Context, inv registrations, forms *sourceForms, buildID string,
|
|
||||||
built inventory.Source, path string) (bool, error) {
|
|
||||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
|
||||||
if err != nil || !found || !r.AtTerminal {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if !forms.same(r.Repository, r.Seat, built.Repository, built.Seat) ||
|
|
||||||
strings.Trim(r.Path, "/") != strings.Trim(path, "/") {
|
|
||||||
fmt.Printf("%s was asked at the terminal of %s at %q, and its outcome is of %s at %q: not the terminal's\n",
|
|
||||||
buildID, sourceWords(r.Repository, r.Seat), r.Path, sourceWords(built.Repository, built.Seat), path)
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceWords is a source as a person reads it.
|
|
||||||
func sourceWords(repository, seat string) string {
|
|
||||||
if seat == "" {
|
|
||||||
return repository
|
|
||||||
}
|
|
||||||
return buildSource{Repository: repository, Seat: seat}.String()
|
|
||||||
}
|
|
||||||
@@ -1,332 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
func theCatalogue(t *testing.T) *stores {
|
|
||||||
t.Helper()
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
for _, b := range []link.BuildResult{
|
|
||||||
onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}),
|
|
||||||
onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}),
|
|
||||||
} {
|
|
||||||
keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true)
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, b); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return open
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
for _, c := range []struct {
|
|
||||||
name, repository, path, module string
|
|
||||||
}{
|
|
||||||
{"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"},
|
|
||||||
{"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"},
|
|
||||||
{"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"},
|
|
||||||
} {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
id := "build-" + strings.ReplaceAll(c.repository, "/", "-")
|
|
||||||
keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb
|
|
||||||
evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"})
|
|
||||||
_, _, err := takeIn(ctx, open.inventory, evil)
|
|
||||||
if !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err)
|
|
||||||
}
|
|
||||||
shelf, err := open.inventory.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := shelf[c.module].Version; got != "1" {
|
|
||||||
t.Fatalf("%s is now %q, from %s", c.module, got, c.repository)
|
|
||||||
}
|
|
||||||
if _, found, _ := open.inventory.BuildByID(ctx, id); !found {
|
|
||||||
t.Errorf("the refused build %s is not recorded", id)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
keptAsked(t, open.inventory, "build-new", "agent/tools", "", false)
|
|
||||||
_, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "",
|
|
||||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
|
||||||
if !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("a new module from an agent's repository was taken in: %v", err)
|
|
||||||
}
|
|
||||||
// And one asked of nobody here — an outcome on the bus no request was kept for — the same.
|
|
||||||
_, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "",
|
|
||||||
map[string]any{"module": "agent-tools", "version": "1"}))
|
|
||||||
if !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("an outcome nobody asked for was taken in: %v", err)
|
|
||||||
}
|
|
||||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" {
|
|
||||||
t.Fatal("the refused module is in the catalogue")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true)
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "",
|
|
||||||
map[string]any{"module": "sudo", "version": "2"})); err != nil {
|
|
||||||
t.Fatalf("a move the operator asked for at the terminal was refused: %v", err)
|
|
||||||
}
|
|
||||||
if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" {
|
|
||||||
t.Fatalf("sudo is built from %q", src.Repository)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external",
|
|
||||||
Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "",
|
|
||||||
map[string]any{"module": "app", "version": "1"})); err != nil {
|
|
||||||
t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
|
||||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID,
|
|
||||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil {
|
|
||||||
t.Fatalf("a plan's build of the module's own repository was refused: %v", err)
|
|
||||||
}
|
|
||||||
added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"})
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, added); err != nil {
|
|
||||||
t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err)
|
|
||||||
}
|
|
||||||
shelf, _ := open.inventory.Catalogue(ctx)
|
|
||||||
if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" {
|
|
||||||
t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
|
||||||
map[string]any{"module": "sudo", "version": "evil"})
|
|
||||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
|
||||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
|
||||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("the fork's build was taken in: %v", err)
|
|
||||||
}
|
|
||||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
|
||||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
|
||||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
|
||||||
map[string]any{"module": "sudo", "version": "2"})
|
|
||||||
failed.Commit = "badbadbad0123456"
|
|
||||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if found && previous.ID == fork.ID {
|
|
||||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
|
||||||
}
|
|
||||||
if !found || previous.ID != "build-sudo" {
|
|
||||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"})
|
|
||||||
t.Cleanup(func() { theForge.Delete("novox/unguarded") })
|
|
||||||
first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"})
|
|
||||||
keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true)
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
|
||||||
t.Fatalf("at the terminal: %v", err)
|
|
||||||
}
|
|
||||||
again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"})
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) ||
|
|
||||||
!strings.Contains(err.Error(), "push to main directly") {
|
|
||||||
t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err)
|
|
||||||
}
|
|
||||||
theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api"))
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "",
|
|
||||||
map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("a forge that could not say was read as a protected trunk: %v", err)
|
|
||||||
}
|
|
||||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" {
|
|
||||||
t.Fatalf("unguarded is %q", shelf["unguarded"].Version)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true})
|
|
||||||
t.Cleanup(func() { theForge.Delete("novox/remade") })
|
|
||||||
first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"})
|
|
||||||
keptAsked(t, open.inventory, first.ID, "novox/remade", "", true)
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, first); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 {
|
|
||||||
t.Fatalf("the forge's id was not recorded: %d", id)
|
|
||||||
}
|
|
||||||
theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "",
|
|
||||||
map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) ||
|
|
||||||
!strings.Contains(err.Error(), "made again") {
|
|
||||||
t.Fatalf("a repository made again under the name was taken in: %v", err)
|
|
||||||
}
|
|
||||||
// And a new module from it, beside the one registered from the first, the same.
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other",
|
|
||||||
map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("a new module from a repository made again was taken in: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app",
|
|
||||||
Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"})
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err)
|
|
||||||
}
|
|
||||||
elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"})
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheServingControllerIsNeverTheTerminal(t *testing.T) {
|
|
||||||
t.Setenv(verbVar, "")
|
|
||||||
t.Setenv(link.CallerVar, "")
|
|
||||||
t.Setenv(servedVar, "")
|
|
||||||
if !startedAtTheTerminal() {
|
|
||||||
t.Fatal("a process started by hand is not the terminal")
|
|
||||||
}
|
|
||||||
markServed()
|
|
||||||
if startedAtTheTerminal() {
|
|
||||||
t.Fatal("the serving controller reads as the terminal")
|
|
||||||
}
|
|
||||||
child := exec.Command(os.Args[0], "-test.run=^$")
|
|
||||||
child.Env = os.Environ()
|
|
||||||
if !slices.Contains(child.Env, servedVar+"=1") {
|
|
||||||
t.Fatal("what the serving controller starts does not carry its mark")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 {
|
|
||||||
t.Fatalf("the recorded id became %d", id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
|
||||||
const commit = "c0ffee0123456789"
|
|
||||||
for _, c := range []struct {
|
|
||||||
what string
|
|
||||||
change func(*anApprovedMerge)
|
|
||||||
unkept bool
|
|
||||||
request string // who asked: "merge", or "build" through a verb
|
|
||||||
moved bool
|
|
||||||
}{
|
|
||||||
{what: "the operator's Approve of the merged head", request: "merge", moved: true},
|
|
||||||
{what: "no merge mesh-delivery announced", unkept: true, request: "merge"},
|
|
||||||
{what: "a warrant for other heads", request: "merge", change: func(a *anApprovedMerge) {
|
|
||||||
a.merge.Heads = map[string]string{"novox/mesh-host@aaaaaaaaaaaa": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"}
|
|
||||||
}},
|
|
||||||
{what: "a Decline", request: "merge", change: func(a *anApprovedMerge) {
|
|
||||||
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
|
|
||||||
}},
|
|
||||||
{what: "a person who is not the operator", request: "merge", change: func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }},
|
|
||||||
{what: "a build asked through a verb", request: "build"},
|
|
||||||
} {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
a := approvedMerge(t)
|
|
||||||
a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-host", 7, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
|
||||||
if c.change != nil {
|
|
||||||
c.change(&a)
|
|
||||||
}
|
|
||||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
|
||||||
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
|
||||||
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
|
||||||
}
|
|
||||||
routerAskOf = func(_ context.Context, _ catalogueReader, _, id string) (*asks.Ask, *asks.Warrant, error) {
|
|
||||||
if id != a.merge.AskID {
|
|
||||||
t.Errorf("%s: the router was asked for %q", c.what, id)
|
|
||||||
}
|
|
||||||
return &a.ask, &a.warrant, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
|
||||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
|
||||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
|
||||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
|
||||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
|
||||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
|
||||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"})
|
|
||||||
moved.Commit = commit
|
|
||||||
_, _, err := takeIn(ctx, inv, moved)
|
|
||||||
src, _ := inv.SourceOf(ctx, "sudo")
|
|
||||||
switch {
|
|
||||||
case c.moved && err != nil:
|
|
||||||
t.Errorf("%s: the move was refused: %v", c.what, err)
|
|
||||||
case c.moved && src.Repository != "novox/mesh-host":
|
|
||||||
t.Errorf("%s: sudo is built from %q", c.what, src.Repository)
|
|
||||||
case !c.moved && !errors.Is(err, errNotItsSource):
|
|
||||||
t.Errorf("%s: the move was not refused as ADR 0266 has it: %v", c.what, err)
|
|
||||||
case !c.moved && src.Repository != "novox/mesh-catalog":
|
|
||||||
t.Errorf("%s: a refused move changed sudo's source to %q", c.what, src.Repository)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,162 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"hash/fnv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The routes to root a review of ADR 0266 found (novox/hq ADR 0266 §7): an agent makes a repository of its
|
|
||||||
// own — or forks one the mesh builds from — commits a module.json naming a module the mesh runs everywhere
|
|
||||||
// (`sudo`, granting itself a rule without a password; `mesh-host`, the node-engine), and asks the `build` verb
|
|
||||||
// for it. Its outcome was registered under that name from the agent's repository, and the next push sent it.
|
|
||||||
|
|
||||||
// onTrunk is an outcome of a commit on its repository's trunk, as the build seat says it.
|
|
||||||
func onTrunk(id, repository, seat, path string, manifest map[string]any) link.BuildResult {
|
|
||||||
raw, _ := json.Marshal(manifest)
|
|
||||||
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/" + repository + ".git", Path: path,
|
|
||||||
Ref: "main", On: "anchor", Commit: "c0ffee0123456789", Manifest: raw,
|
|
||||||
Trunk: "main", OnTrunk: true, Branches: []string{"main"}}
|
|
||||||
if seat != "" {
|
|
||||||
r.Source = &link.SourceOnSeat{Seat: seat, Repository: repository}
|
|
||||||
}
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptAsked keeps a build request as the asker would: through a verb, or at the terminal.
|
|
||||||
func keptAsked(t *testing.T, inv *inventory.Inventory, id, repository, path string, atTerminal bool) {
|
|
||||||
t.Helper()
|
|
||||||
if err := inv.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: id, Repository: repository, Seat: "git",
|
|
||||||
Path: path, For: "build", AtTerminal: atTerminal}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// theCatalogue is a mesh whose sudo is built from the catalogue repository and whose node-engine from its own.
|
|
||||||
|
|
||||||
// The operator at the terminal may still move a module, or add one from a new repository.
|
|
||||||
|
|
||||||
// The delivery's flow is untouched: a merge's rebuild of a module from its own repository, and a merge adding
|
|
||||||
// a module to a repository the catalogue builds from (novox/hq issue 300), are registered with no terminal.
|
|
||||||
|
|
||||||
// Through a verb, a build of a repository the catalogue builds nothing from is not even asked: the build node
|
|
||||||
// would run what the agent wrote.
|
|
||||||
|
|
||||||
// asTheOperator keeps a build as asked at the controller's terminal, as the operator's first build of a module
|
|
||||||
// from a repository the catalogue does not yet build from is (novox/hq ADR 0266), and hands it back.
|
|
||||||
func asTheOperator(t *testing.T, inv *inventory.Inventory, b link.BuildResult) link.BuildResult {
|
|
||||||
t.Helper()
|
|
||||||
repository, seat := b.Repository, ""
|
|
||||||
if b.Source != nil {
|
|
||||||
repository, seat = b.Source.Repository, b.Source.Seat
|
|
||||||
}
|
|
||||||
if err := inv.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: b.ID, Repository: repository, Seat: seat,
|
|
||||||
Path: b.Path, For: "build", AtTerminal: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
// A rollback puts back only a build of the module's own repository: an agent's build of the module's name,
|
|
||||||
// recorded and refused, at the very commit the machine ran before (a fork carries it), is never registered by
|
|
||||||
// the back door of a failed gate.
|
|
||||||
|
|
||||||
// theForge is what the forge says in a test, by owner/name: a repository not named here is protected as a
|
|
||||||
// trunk must be, with an id of its own.
|
|
||||||
var theForge sync.Map
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
askTheForge = func(_ context.Context, owner, repo, _ string) (forgeFacts, error) {
|
|
||||||
if said, ok := theForge.Load(owner + "/" + repo); ok {
|
|
||||||
switch f := said.(type) {
|
|
||||||
case error:
|
|
||||||
return forgeFacts{}, f
|
|
||||||
case forgeFacts:
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
h := fnv.New32a()
|
|
||||||
_, _ = h.Write([]byte(owner + "/" + repo))
|
|
||||||
return forgeFacts{ID: int64(h.Sum32()), Guarded: true}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module's trunk the forge does not protect — direct pushes, no required status — or a forge that cannot say,
|
|
||||||
// registers nothing through a verb: the trunk rule means nothing on a branch anyone pushes to.
|
|
||||||
|
|
||||||
// A repository deleted and made again under the module's repository's name is another repository: the forge's
|
|
||||||
// id, recorded at registration, tells them apart.
|
|
||||||
|
|
||||||
// The terminal's mark is the operator's for the repository and path they asked: an outcome of another, under that
|
|
||||||
// build's id, is not theirs.
|
|
||||||
|
|
||||||
// The serving controller, and everything it starts, is never the terminal: a build asked in it reads as asked
|
|
||||||
// through the mesh even when no verb and no caller is named.
|
|
||||||
|
|
||||||
// The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger
|
|
||||||
// rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review).
|
|
||||||
func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) {
|
|
||||||
tools := []string{forgeToolsAccount}
|
|
||||||
guarded := protectionRule{Rule: "*", MergeWhitelist: tools}
|
|
||||||
open := protectionRule{Rule: "main", Push: true, MergeWhitelist: tools}
|
|
||||||
if f := judgedRule([]protectionRule{guarded, open}, "main"); f.Guarded {
|
|
||||||
t.Error("an exact rule letting pushes was passed over for a glob that guards")
|
|
||||||
}
|
|
||||||
if f := judgedRule([]protectionRule{{Rule: "ma*", MergeWhitelist: nil}, {Rule: "*", MergeWhitelist: tools}},
|
|
||||||
"main"); f.Guarded || !strings.Contains(f.Why, "ma*") {
|
|
||||||
t.Errorf("the first glob covering the branch was not the one judged: %+v", f)
|
|
||||||
}
|
|
||||||
if f := judgedRule([]protectionRule{{Rule: "release/*"}, {Rule: "main", MergeWhitelist: tools}}, "main"); !f.Guarded {
|
|
||||||
t.Errorf("the rule named for the branch was not judged: %+v", f)
|
|
||||||
}
|
|
||||||
if f := judgedRule(nil, "main"); f.Guarded {
|
|
||||||
t.Error("no rule is no protection")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A module's trunk is trusted when it takes no push, only the tools' account and people's own logins merge into it,
|
|
||||||
// and no administrator merges past its rule** (novox/hq ADR 0534 §2, §6): the forge requires no status to merge (§1), so
|
|
||||||
// a rule requiring none is trusted; one with no merge whitelist, no tools' account in it, an agent's account in it, a
|
|
||||||
// push allowed or an administrator's override is not.
|
|
||||||
func TestATrunkIsTrustedForItsMergersNotForAStatus(t *testing.T) {
|
|
||||||
tools := []string{forgeToolsAccount}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
rule protectionRule
|
|
||||||
want bool
|
|
||||||
why string
|
|
||||||
}{
|
|
||||||
{"no status, no push, the tools alone merge", protectionRule{Rule: "main", MergeWhitelist: tools}, true, ""},
|
|
||||||
{"a status required as well", protectionRule{Rule: "main", MergeWhitelist: tools, RequiredStatuses: []string{"mesh/check"}}, true, ""},
|
|
||||||
{"no merge whitelist", protectionRule{Rule: "main"}, false, "merge whitelist"},
|
|
||||||
{"an empty merge whitelist", protectionRule{Rule: "main", MergeWhitelist: []string{}}, false, "merge whitelist"},
|
|
||||||
{"the operator's own login beside the tools", protectionRule{Rule: "main", MergeWhitelist: []string{forgeToolsAccount, "jschoubben"}}, true, ""},
|
|
||||||
{"an agent's account in the whitelist", protectionRule{Rule: "main", MergeWhitelist: []string{forgeToolsAccount, "agent-ace"}}, false, "agent-ace"},
|
|
||||||
{"an agent's account alone", protectionRule{Rule: "main", MergeWhitelist: []string{"agent-ace"}}, false, "agent-ace"},
|
|
||||||
{"a person's login without the tools", protectionRule{Rule: "main", MergeWhitelist: []string{"jschoubben"}}, false, forgeToolsAccount},
|
|
||||||
{"a push allowed", protectionRule{Rule: "main", Push: true, MergeWhitelist: tools}, false, "push"},
|
|
||||||
{"an administrator's override", protectionRule{Rule: "main", MergeWhitelist: tools, AdminMayOverride: true}, false, "administrator"},
|
|
||||||
} {
|
|
||||||
f := judgedRule([]protectionRule{c.rule}, "main")
|
|
||||||
if f.Guarded != c.want || !strings.Contains(f.Why, c.why) {
|
|
||||||
t.Errorf("%s: %+v, want guarded %v saying %q", c.name, f, c.want, c.why)
|
|
||||||
}
|
|
||||||
if strings.Contains(f.Why, "requires no status") {
|
|
||||||
t.Errorf("%s: a trunk is still judged on a required status: %s", c.name, f.Why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An id the forge could not give keeps the id already recorded.
|
|
||||||
|
|
||||||
// **The operator's Approve of a reviewed group registers a moved module from its new repository** (novox/hq ADR 0420 §7,
|
|
||||||
// issue 1025). sudo is built from the catalogue; a merge into the repository of node-engine's module, announced by
|
|
||||||
// mesh-delivery with the ask it merged on, brings its build from there. Accepted with the operator's Approve of the
|
|
||||||
// merged head; refused with no kept merge, a warrant for other heads, a Decline, a person who is not the operator, and
|
|
||||||
// through a verb.
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A build's outcome is taken in the same way whoever hears it (novox/hq issue 176): recorded, and
|
|
||||||
// the module registered with its source as the seat and path when the request said so — never the
|
|
||||||
// URL. A definition naming an installation is recorded and not registered; a failure is recorded
|
|
||||||
// and said.
|
|
||||||
|
|
||||||
// novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the
|
|
||||||
// module keeps following the branch it followed — a new one, the default branch.
|
|
||||||
|
|
||||||
// novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered,
|
|
||||||
// so a push sends what the newer request built.
|
|
||||||
|
|
||||||
func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC)
|
|
||||||
if got, ok := link.BuildAskedAt(link.NewBuildID(at)); !ok || !got.Equal(at) {
|
|
||||||
t.Errorf("read back %v %v; want %v", got, ok, at)
|
|
||||||
}
|
|
||||||
if got, ok := link.BuildAskedAt("build-1791064317392539762"); !ok || got.Format(time.TimeOnly) != "21:51:57" {
|
|
||||||
t.Errorf("the incident's id reads as %v %v", got, ok)
|
|
||||||
}
|
|
||||||
for _, id := range []string{"b-1", "build-2", "build-", "build-x", ""} {
|
|
||||||
if _, ok := link.BuildAskedAt(id); ok {
|
|
||||||
t.Errorf("%q read as a request time", id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq issue 1204: builds says the source fingerprint the builder recorded, the one tool that reads it.
|
|
||||||
func TestBuildsSaysTheSourceFingerprint(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 19, 10, 40, 0, 0, time.UTC)
|
|
||||||
built := inventory.Build{ID: "build-1", Module: "mailu", Commit: "b4527d9091e972f28cb365623d9280045e7741a3", On: "ace",
|
|
||||||
Repository: "novox/mesh-catalog", At: at, ReuseKey: "r1:abc",
|
|
||||||
SourceFingerprint: "src1:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}
|
|
||||||
|
|
||||||
// 1204/AC1
|
|
||||||
var out bytes.Buffer
|
|
||||||
printBuild(&out, built)
|
|
||||||
lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n")
|
|
||||||
if last := lines[len(lines)-1]; last != " source "+built.SourceFingerprint {
|
|
||||||
t.Errorf("last line %q; want the source line after the reuse key:\n%s", last, out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(lines[len(lines)-2], "reuse key r1:abc") {
|
|
||||||
t.Errorf("the source line does not follow the reuse key:\n%s", out.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1204/AC2
|
|
||||||
without := built
|
|
||||||
without.SourceFingerprint = ""
|
|
||||||
var plain bytes.Buffer
|
|
||||||
printBuild(&plain, without)
|
|
||||||
if strings.Contains(plain.String(), "source") {
|
|
||||||
t.Errorf("a build with no fingerprint says a source line:\n%s", plain.String())
|
|
||||||
}
|
|
||||||
if want := strings.Join(lines[:len(lines)-1], "\n") + "\n"; plain.String() != want {
|
|
||||||
t.Errorf("the other lines changed:\n%s\nwant\n%s", plain.String(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,109 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// builderOnTheBus is the builder's and the indexer's buckets on a test's own bus, which the controller's connection for
|
|
||||||
// the moment (servingBus) reaches: what a take-in from a key reads (R4e, issue 1851).
|
|
||||||
type builderOnTheBus struct {
|
|
||||||
js *broker.JetStream
|
|
||||||
builds, modules, manifests jetstream.KeyValue
|
|
||||||
}
|
|
||||||
|
|
||||||
func aBuilderOnTheBus(t *testing.T) builderOnTheBus {
|
|
||||||
t.Helper()
|
|
||||||
js, err := broker.Dial(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
api, err := jetstream.New(js.Conn())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx := t.Context()
|
|
||||||
b := builderOnTheBus{js: js}
|
|
||||||
for _, c := range []struct {
|
|
||||||
into *jetstream.KeyValue
|
|
||||||
bucket string
|
|
||||||
history uint8
|
|
||||||
}{
|
|
||||||
{&b.builds, sdkdelivery.BuilderBuildsBucket, sdkdelivery.BuilderBuildsHistory},
|
|
||||||
{&b.modules, sdkdelivery.ModulesBucket, sdkdelivery.ModulesHistory},
|
|
||||||
{&b.manifests, sdkdelivery.ManifestsBucket, 1},
|
|
||||||
} {
|
|
||||||
kv, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: c.bucket, History: c.history})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*c.into = kv
|
|
||||||
}
|
|
||||||
servingBus.Store(js)
|
|
||||||
t.Cleanup(func() { servingBus.Store(nil) })
|
|
||||||
withTrunkList(t, true)
|
|
||||||
// The indexer on this bus is read, not the "not held" a mesh with no bus reads (aMeshOnTheStore).
|
|
||||||
wasIndexed := indexedModule
|
|
||||||
indexedModule = theIndexedModule
|
|
||||||
t.Cleanup(func() { indexedModule = wasIndexed })
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
// indexed writes the indexer's trunk value of module at commit, naming fp, and its manifest: a module of one archive.
|
|
||||||
func (b builderOnTheBus) indexed(t *testing.T, module, version, repository, path, commit, fp string) string {
|
|
||||||
t.Helper()
|
|
||||||
keys, trunk := aKeyFor(module, version, fp, repository, path, commit)
|
|
||||||
raw := keys.manifests[trunk.Manifest]
|
|
||||||
value, err := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: module, JSON: raw})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := b.manifests.Put(context.Background(), trunk.Manifest, value); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
trunk.FP = "src1:" + fp[len("src2:"):]
|
|
||||||
encoded, err := sdkdelivery.EncodeModule(trunk)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := b.modules.Put(context.Background(), module, encoded); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return trunk.Manifest
|
|
||||||
}
|
|
||||||
|
|
||||||
// built writes the builder's key of module at fp: done, its one archive named with a reference, or in another state.
|
|
||||||
func (b builderOnTheBus) built(t *testing.T, module, version, fp, manifest, state string) {
|
|
||||||
t.Helper()
|
|
||||||
keys, _ := aKeyFor(module, version, fp, "", "", "")
|
|
||||||
v := keys.keys[module].Build
|
|
||||||
v.Manifest = manifest
|
|
||||||
if state != sdkdelivery.BuildDone {
|
|
||||||
v = sdkdelivery.BuilderBuild{State: state, By: "ace"}
|
|
||||||
if state == sdkdelivery.BuildFailed {
|
|
||||||
v.Error = "digests differ: ace sha256:1, g14 sha256:2"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
value, err := sdkdelivery.EncodeBuilderBuild(v)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
key, err := sdkdelivery.BuilderBuildKey(module, fp)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := b.builds.Put(context.Background(), key, value); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// theIndexedModule is the look at the indexer as the controller makes it, kept before any test replaces it.
|
|
||||||
var theIndexedModule = indexedModule
|
|
||||||
@@ -1,410 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236).
|
|
||||||
//
|
|
||||||
// **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the
|
|
||||||
// controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything,
|
|
||||||
// and a version whose data format moved (2.10 → 2.11) cannot be undone by putting the old one back.
|
|
||||||
// So nothing sends a new bus build on its own: its policy is `record` whatever anyone says (the
|
|
||||||
// catalogue's DerivedUpgrade, and `upgrade` refuses a roll-out), a plan builds it and sends nothing, a
|
|
||||||
// cascade holds the machine (ADR 0221), and a push naming that machine is refused while a new bus build
|
|
||||||
// waits for it (busHeld). The one way is this verb: a person, with why, the streams snapshotted first,
|
|
||||||
// whether it can be reverted said before it starts, the step said as `bus-maintenance` while it runs,
|
|
||||||
// and every stream, durable consumer and a round trip checked after (H-bus) — or the step said failed,
|
|
||||||
// with its snapshot as the way back.
|
|
||||||
|
|
||||||
// busStepProbe is the registry's row for the step; its kinds.
|
|
||||||
const (
|
|
||||||
busStepProbe = "DB"
|
|
||||||
kindBusMaintenance = "bus-maintenance"
|
|
||||||
kindBusUpgradeFailed = "bus-upgrade-failed"
|
|
||||||
)
|
|
||||||
|
|
||||||
// busStepBound is how long after its start a bus upgrade must be followed by a healthy bus.
|
|
||||||
var busStepBound = 15 * time.Minute
|
|
||||||
|
|
||||||
// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is: the bus
|
|
||||||
// machine's backup holder backs the bus module up now, whose dump is the streams' snapshot (novox/hq ADR
|
|
||||||
// 0235). A variable so a test takes none. A person who took one by hand says where with --snapshot-taken,
|
|
||||||
// and then none is taken — for a bus whose module does not yet carry the snapshot program.
|
|
||||||
var takeBusSnapshot = snapshotTheBusNow
|
|
||||||
|
|
||||||
// busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per
|
|
||||||
// machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds
|
|
||||||
// no bus module.
|
|
||||||
type busPending struct {
|
|
||||||
module string
|
|
||||||
machines []string
|
|
||||||
from map[string]string
|
|
||||||
to string
|
|
||||||
// same are the commits whose build was made from the same source as the build the mesh holds, or
|
|
||||||
// made the same artifacts and manifest (novox/hq issue 280).
|
|
||||||
same map[string]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the
|
|
||||||
// two builds were made from the same source, or made the same artifacts from the same manifest — a
|
|
||||||
// rebuild of the same source for another module's merge changes nothing the machine runs, whatever
|
|
||||||
// image digest it made (novox/hq issue 280).
|
|
||||||
func (b busPending) moves(machine string) bool {
|
|
||||||
from, known := b.from[machine]
|
|
||||||
if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return !known || !b.same[from]
|
|
||||||
}
|
|
||||||
|
|
||||||
// pendingBus reads what a bus upgrade would do.
|
|
||||||
func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, error) {
|
|
||||||
var b busPending
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
for name, m := range shelf {
|
|
||||||
if catalogue.ProvidesBus(m) {
|
|
||||||
b.module = name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if b.module == "" {
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
current, err := inv.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
b.to = current[b.module].Commit
|
|
||||||
if b.machines, err = inv.Running(ctx, b.module); err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
b.from, b.same = map[string]string{}, map[string]bool{}
|
|
||||||
made, err := inv.BuildFingerprints(ctx, b.module)
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
for commit, refs := range made {
|
|
||||||
b.same[commit] = refs != "" && refs == made[b.to]
|
|
||||||
}
|
|
||||||
sources, err := inv.BuildSourceFingerprints(ctx, b.module)
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
for commit, src := range sources {
|
|
||||||
if src != "" && src == sources[b.to] {
|
|
||||||
b.same[commit] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, n := range b.machines {
|
|
||||||
sent, known, err := inv.SentBuilds(ctx, n)
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
if known {
|
|
||||||
if c, carried := sent[b.module]; carried {
|
|
||||||
b.from[n] = c
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// busHeld names the machines a push may not send because sending them would replace the bus: the
|
|
||||||
// planned step's, not a push's (ADR 0236). Said with the remedy.
|
|
||||||
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
|
|
||||||
b, err := pendingBus(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out := map[string]string{}
|
|
||||||
for _, n := range machines {
|
|
||||||
for _, holder := range b.machines {
|
|
||||||
if n == holder && b.moves(n) {
|
|
||||||
out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+
|
|
||||||
"`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)",
|
|
||||||
n, b.module, short(orNotKnown(b.from[n])), short(b.to))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// busCommand is `bus` — what a bus upgrade would do and how the last went — and `bus upgrade`.
|
|
||||||
func busCommand(ctx context.Context, args []string) error {
|
|
||||||
sub := ""
|
|
||||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
||||||
sub, args = args[0], args[1:]
|
|
||||||
}
|
|
||||||
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
|
||||||
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
|
||||||
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
|
||||||
irreversible := set.Bool("irreversible", false, "the new version cannot be undone by putting the old one back, "+
|
|
||||||
"and this is the person's explicit word that it runs anyway")
|
|
||||||
why := addHandActFlags(set)
|
|
||||||
if rest, err := parseAround(set, args); err != nil {
|
|
||||||
return err
|
|
||||||
} else if len(rest) > 0 {
|
|
||||||
return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>]]")
|
|
||||||
}
|
|
||||||
switch sub {
|
|
||||||
case "":
|
|
||||||
return busStatus(ctx)
|
|
||||||
case "upgrade":
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub)
|
|
||||||
}
|
|
||||||
// Everything refused before anything is done.
|
|
||||||
if err := why.require("bus upgrade"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if *reversible == *irreversible {
|
|
||||||
return errors.New("bus upgrade says, before it starts, whether the new version can be undone by putting the " +
|
|
||||||
"old one back: --reversible, or --irreversible as your explicit word that it runs anyway (to-be 45 §8). " +
|
|
||||||
"Nothing was done")
|
|
||||||
}
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
inv := open.inventory
|
|
||||||
b, err := pendingBus(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if b.module == "" {
|
|
||||||
return errors.New("the mesh holds no module that provides its bus: there is nothing to upgrade")
|
|
||||||
}
|
|
||||||
var moving []string
|
|
||||||
for _, n := range b.machines {
|
|
||||||
if b.moves(n) {
|
|
||||||
moving = append(moving, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(moving) == 0 {
|
|
||||||
fmt.Printf("every machine running %s runs the build the mesh holds (%s): nothing to upgrade\n", b.module, short(b.to))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
where := strings.TrimSpace(*snapshot)
|
|
||||||
if where == "" {
|
|
||||||
for _, n := range moving {
|
|
||||||
fmt.Printf("snapshotting the bus's streams on %s first (its backup holder, ADR 0235)…\n", n)
|
|
||||||
if where, err = takeBusSnapshot(ctx, b.module, n); err != nil {
|
|
||||||
return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w — a snapshot "+
|
|
||||||
"taken by hand is said with --snapshot-taken <where>", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
from := map[string]bool{}
|
|
||||||
for _, n := range moving {
|
|
||||||
from[orNotKnown(b.from[n])] = true
|
|
||||||
}
|
|
||||||
step, err := inv.StartBusStep(ctx, inventory.BusStep{Module: b.module, Machines: moving,
|
|
||||||
From: strings.Join(sortedKeys(from), ", "), To: b.to, Snapshot: where, Reversible: *reversible,
|
|
||||||
By: link.Caller(), Why: strings.TrimSpace(*why.why)})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
cause := "bus-upgrade"
|
|
||||||
if strings.TrimSpace(*why.cause) == "" {
|
|
||||||
why.cause = &cause
|
|
||||||
}
|
|
||||||
why.record(ctx, "bus upgrade", append([]string{b.module}, moving...))
|
|
||||||
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
|
|
||||||
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
|
|
||||||
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
|
|
||||||
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
|
|
||||||
if err != nil {
|
|
||||||
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
|
|
||||||
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
|
|
||||||
}
|
|
||||||
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
|
||||||
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
|
|
||||||
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// busStatus is `bus`: what an upgrade would do, and the last step.
|
|
||||||
func busStatus(ctx context.Context) error {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
b, err := pendingBus(ctx, open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if b.module == "" {
|
|
||||||
fmt.Println("the mesh holds no module that provides its bus")
|
|
||||||
} else {
|
|
||||||
fmt.Printf("the bus is %s, built %s, on %s; never rolled out — a planned step (`bus upgrade`)\n", b.module,
|
|
||||||
short(b.to), orNone(strings.Join(b.machines, ", ")))
|
|
||||||
for _, n := range b.machines {
|
|
||||||
state := "runs it"
|
|
||||||
if b.moves(n) {
|
|
||||||
state = "runs " + short(orNotKnown(b.from[n])) + ": `bus upgrade` replaces it"
|
|
||||||
}
|
|
||||||
fmt.Printf(" %-10s %s\n", n, state)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Println(" `bus upgrade` has the bus machine's backup holder snapshot the streams first (ADR 0235)")
|
|
||||||
s, found, err := open.inventory.LatestBusStep(ctx)
|
|
||||||
if err != nil || !found {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
state := "running since " + s.Started.Local().Format("2006-01-02 15:04")
|
|
||||||
if s.Ended != nil {
|
|
||||||
state = s.Outcome + " at " + s.Ended.Local().Format("2006-01-02 15:04")
|
|
||||||
}
|
|
||||||
fmt.Printf("last step %d: %s → %s on %s by %s (%s): %s; snapshot %s\n", s.ID, s.From, short(s.To),
|
|
||||||
strings.Join(s.Machines, ", "), orNone(s.By), s.Why, state, s.Snapshot)
|
|
||||||
if s.Found != "" {
|
|
||||||
fmt.Printf(" %s\n", s.Found)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeBusStep is DB: a bus upgrade running is said as `bus-maintenance`; the bus healthy again after
|
|
||||||
// the machines reported the new build ends it done; past its bound, unhealthy, it ends failed and is
|
|
||||||
// said — urgent, with its snapshot — while the bus is still not healthy.
|
|
||||||
func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
||||||
inv := d.open.inventory
|
|
||||||
s, found, err := inv.LatestBusStep(ctx)
|
|
||||||
if err != nil || !found {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if s.Ended != nil && s.Outcome != "failed" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
problems, err := busHealth(ctx, d)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
reports, err := inv.LastReports(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
applied := true
|
|
||||||
for _, r := range reports {
|
|
||||||
for _, n := range s.Machines {
|
|
||||||
if r.Node == n && (!r.Current || r.Outcome != inventory.OutcomeApplied || r.At == nil || r.At.Before(s.Started)) {
|
|
||||||
applied = false
|
|
||||||
problems = append(problems, n+" has not reported the new bus applied")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
id := s.Module
|
|
||||||
if s.Ended == nil {
|
|
||||||
switch {
|
|
||||||
case applied && len(problems) == 0:
|
|
||||||
return nil, inv.EndBusStep(ctx, s.ID, "done", "the bus answered healthy after the upgrade")
|
|
||||||
case time.Since(s.Started) > busStepBound:
|
|
||||||
found := strings.Join(problems, "; ")
|
|
||||||
if err := inv.EndBusStep(ctx, s.ID, "failed", found); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
s.Found = found
|
|
||||||
default:
|
|
||||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "maintenance",
|
|
||||||
Kind: kindBusMaintenance, Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("the bus is being upgraded (step %d, %s → %s on %s, by %s: %s); its snapshot is %s",
|
|
||||||
s.ID, s.From, short(s.To), strings.Join(s.Machines, ", "), orNone(s.By), s.Why, s.Snapshot),
|
|
||||||
Said: orNone(strings.Join(problems, "; "))}}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(problems) == 0 {
|
|
||||||
return nil, nil // failed, and healthy since: nothing wrong now
|
|
||||||
}
|
|
||||||
way := "put the old build back"
|
|
||||||
if !s.Reversible {
|
|
||||||
way = "restore the snapshot"
|
|
||||||
}
|
|
||||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "upgrade-failed",
|
|
||||||
Kind: kindBusUpgradeFailed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
|
||||||
Summary: fmt.Sprintf("the bus upgrade (step %d, %s → %s) did not end healthy within %s: %s — the way back is to %s (%s)",
|
|
||||||
s.ID, s.From, short(s.To), busStepBound, strings.Join(problems, "; "), way, s.Snapshot)}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func sortedKeys(set map[string]bool) []string {
|
|
||||||
out := make([]string, 0, len(set))
|
|
||||||
for k := range set {
|
|
||||||
out = append(out, k)
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// busSnapshotWithin is how long the bus machine's backup holder is given to take the bus's snapshot.
|
|
||||||
var busSnapshotWithin = 15 * time.Minute
|
|
||||||
|
|
||||||
// snapshotTheBusNow asks the bus machine's backup holder to back the bus module up now — its dump is
|
|
||||||
// the streams' snapshot (novox/hq ADR 0235) — and waits until it says a backup newer than the ask:
|
|
||||||
// where the snapshot is, as a person reads it. The serving controller's connection, or one of its own.
|
|
||||||
func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) {
|
|
||||||
var where string
|
|
||||||
err := onTheBus(func(conn *nats.Conn) error {
|
|
||||||
asked := time.Now()
|
|
||||||
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", node,
|
|
||||||
map[string]any{"module": module}, 30*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
|
||||||
}
|
|
||||||
if answer.Error != "" {
|
|
||||||
return fmt.Errorf("%s's backup holder would not take the bus's snapshot: %s", node, answer.Error)
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(busSnapshotWithin)
|
|
||||||
for {
|
|
||||||
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", node, map[string]any{}, 10*time.Second)
|
|
||||||
if err == nil && answer.Error == "" {
|
|
||||||
if measured, err := readHolder(answer.Result); err == nil {
|
|
||||||
for item, m := range measured[module] {
|
|
||||||
if m.LastBackup != nil && m.LastBackup.After(asked) && m.Error == "" {
|
|
||||||
where = fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, item,
|
|
||||||
m.LastBackup.UTC().Format(time.RFC3339))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
return fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s; the bus is "+
|
|
||||||
"not replaced", node, busSnapshotWithin)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return where, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// errBusWaits is a send refused because it would replace the bus outside its planned step.
|
|
||||||
var errBusWaits = errors.New("a new bus build waits for its planned step")
|
|
||||||
|
|
||||||
type busStepKey struct{}
|
|
||||||
|
|
||||||
// withBusStep marks a send as the bus's planned step: the one send that may replace the bus.
|
|
||||||
func withBusStep(ctx context.Context) context.Context {
|
|
||||||
return context.WithValue(ctx, busStepKey{}, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on }
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Every user's slow consumer is S9's, not the controller's alone** (novox/hq ADR 0357, issue 270). On
|
|
||||||
// 2026-10-07 the bus closed two clients on the laptop as slow consumers and nothing raised a condition:
|
|
||||||
// the controller heard only its own connection. Said by the bus module and read back, a user the bus
|
|
||||||
// closes is one condition naming the module and its machine, raised at the first look and cleared an hour
|
|
||||||
// after the last closure.
|
|
||||||
func TestAUserTheBusClosedAsASlowConsumerIsOneConditionNamingItsModuleAndMachine(t *testing.T) {
|
|
||||||
closed := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
|
||||||
now := closed.Add(20 * time.Second)
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
||||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
||||||
t.Cleanup(func() { k.Close(t.Context()) })
|
|
||||||
|
|
||||||
advisories := link.BusClosures([]link.BusClosure{
|
|
||||||
{User: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Hour),
|
|
||||||
Closed: closed.Add(-10 * time.Minute)},
|
|
||||||
{User: "laptop.claude-code", CID: 44, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Minute),
|
|
||||||
Closed: closed},
|
|
||||||
{User: "anchor.desk", CID: 7, Reason: "Maximum Message Payload Exceeded", Opened: closed.Add(-time.Hour),
|
|
||||||
Closed: closed},
|
|
||||||
}, []string{link.AdvisorySlowConsumer, link.AdvisorySlowConsumer, link.AdvisoryConnectionClosed}, closed.Add(-time.Hour))
|
|
||||||
look := func() {
|
|
||||||
t.Helper()
|
|
||||||
f := &signalFacts{now: now, host: "anchor", advisories: advisories}
|
|
||||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
look()
|
|
||||||
|
|
||||||
slow, found, err := conditions.ReadOne(t.Context(), store, "bus.laptop.claude-code.slow-consumer")
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("no slow-consumer condition for the user: %v %v", found, err)
|
|
||||||
}
|
|
||||||
if slow.Subject.Machine != "laptop" || slow.Observations != 2 || !slow.LastObserved.Equal(closed) {
|
|
||||||
t.Fatalf("the slow consumer is said on %q, %d closure(s), last %s: %+v", slow.Subject.Machine, slow.Observations,
|
|
||||||
slow.LastObserved, slow)
|
|
||||||
}
|
|
||||||
if !strings.Contains(slow.Summary, "claude-code on laptop") || strings.Contains(slow.Summary, "laptop.claude-code") {
|
|
||||||
t.Fatalf("the summary does not name the module in words, or carries the dotted user: %q", slow.Summary)
|
|
||||||
}
|
|
||||||
if !strings.Contains(slow.Headline, "Claude-code on laptop") {
|
|
||||||
t.Fatalf("the headline is %q", slow.Headline)
|
|
||||||
}
|
|
||||||
cut, found, err := conditions.ReadOne(t.Context(), store, "bus.anchor.desk.connection-closed")
|
|
||||||
if err != nil || !found || cut.Subject.Machine != "anchor" || !strings.Contains(cut.Summary, "maximum message payload exceeded") {
|
|
||||||
t.Fatalf("the user cut off for breaking the bus's rules: %v %v %+v", found, err, cut)
|
|
||||||
}
|
|
||||||
|
|
||||||
// An hour after the newest closure without another, both clear.
|
|
||||||
now = closed.Add(time.Hour + time.Minute)
|
|
||||||
look()
|
|
||||||
for _, key := range []string{"bus.laptop.claude-code.slow-consumer", "bus.anchor.desk.connection-closed"} {
|
|
||||||
if _, found, _ := k.Get(t.Context(), key); found {
|
|
||||||
t.Fatalf("%s is still open an hour after the last closure", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A read of the bus module's events that fails does not blind S9** (review of #326): the controller's own
|
|
||||||
// signals are still said, and the failed read is a condition of its own.
|
|
||||||
func TestABusModuleReadThatFailsKeepsTheControllersOwnSignals(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
|
||||||
f := &signalFacts{now: at.Add(time.Minute), host: "anchor",
|
|
||||||
advisories: []link.Advisory{{Kind: link.AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind",
|
|
||||||
First: at, Last: at, Count: 1}},
|
|
||||||
busFaultsErr: errors.New("nats: timeout")}
|
|
||||||
keys := map[string]bool{}
|
|
||||||
for _, o := range watchAdvisories(f) {
|
|
||||||
keys[o.Key()] = true
|
|
||||||
}
|
|
||||||
if !keys["bus.controller.slow-consumer"] || !keys["bus.bus-module.bus-faults-unread"] {
|
|
||||||
t.Fatalf("observed %v: the controller's own slow consumer and the unread events, both", keys)
|
|
||||||
}
|
|
||||||
for _, row := range signalsTable {
|
|
||||||
if row.Row == "S9" && row.needs(f) != nil {
|
|
||||||
t.Fatal("the S9 row went blind for a read of the bus module's events")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,195 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A send held for the bus's planned step (novox/hq issue 336).
|
|
||||||
//
|
|
||||||
// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned
|
|
||||||
// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every
|
|
||||||
// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only
|
|
||||||
// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move.
|
|
||||||
// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what
|
|
||||||
// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it
|
|
||||||
// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the
|
|
||||||
// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks'
|
|
||||||
// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB).
|
|
||||||
|
|
||||||
// kindBusStepWaiting is S17's kind: bus.<module>.step-waiting.
|
|
||||||
const kindBusStepWaiting = "bus-step-waiting"
|
|
||||||
|
|
||||||
// busFacts is a new bus build waiting for its planned step, and the sends held for it.
|
|
||||||
type busFacts struct {
|
|
||||||
module, to string
|
|
||||||
// from is the build each machine of the bus runs; machines those whose bus the step would replace.
|
|
||||||
from map[string]string
|
|
||||||
machines []string
|
|
||||||
waits []busWaitFacts
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaitFacts is one walk whose send was refused because it would replace the bus.
|
|
||||||
type busWaitFacts struct {
|
|
||||||
plan, repository, commit string
|
|
||||||
// modules are what its tier sends: what waits.
|
|
||||||
modules []string
|
|
||||||
// since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal.
|
|
||||||
since time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note
|
|
||||||
// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save.
|
|
||||||
var busRefusedFirst = &firstSeen{at: map[string]time.Time{}}
|
|
||||||
|
|
||||||
type firstSeen struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
at map[string]time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// mark keeps the first moment an id was seen.
|
|
||||||
func (s *firstSeen) mark(id string, at time.Time) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if _, seen := s.at[id]; !seen {
|
|
||||||
s.at[id] = at
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// of is when an id was first seen; zero when it was not.
|
|
||||||
func (s *firstSeen) of(id string) time.Time {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return s.at[id]
|
|
||||||
}
|
|
||||||
|
|
||||||
// keepOnly forgets every id not given: a walk no longer held is not held since then.
|
|
||||||
func (s *firstSeen) keepOnly(ids map[string]bool) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
for id := range s.at {
|
|
||||||
if !ids[id] {
|
|
||||||
delete(s.at, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step.
|
|
||||||
func refusedForTheBus(err error) bool {
|
|
||||||
return err != nil && strings.Contains(err.Error(), errBusWaits.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its
|
|
||||||
// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace
|
|
||||||
// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not.
|
|
||||||
func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts {
|
|
||||||
f := busFacts{module: b.module, to: b.to, from: b.from}
|
|
||||||
for _, n := range b.machines {
|
|
||||||
if b.moves(n) {
|
|
||||||
f.machines = append(f.machines, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(f.machines) == 0 {
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
for _, p := range plans {
|
|
||||||
if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
since := p.Updated
|
|
||||||
if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) {
|
|
||||||
since = seen
|
|
||||||
}
|
|
||||||
var modules []string
|
|
||||||
if p.Tier >= 0 && p.Tier < len(p.Tiers) {
|
|
||||||
modules = append(modules, p.Tiers[p.Tier]...)
|
|
||||||
} else {
|
|
||||||
modules = planModules(p)
|
|
||||||
}
|
|
||||||
sort.Strings(modules)
|
|
||||||
f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules,
|
|
||||||
since: since})
|
|
||||||
}
|
|
||||||
sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) })
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out.
|
|
||||||
func (b busFacts) heldByTheBus() map[string]bool {
|
|
||||||
out := map[string]bool{}
|
|
||||||
for _, w := range b.waits {
|
|
||||||
out[w.plan] = true
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with
|
|
||||||
// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true").
|
|
||||||
const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`"
|
|
||||||
|
|
||||||
// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator.
|
|
||||||
func watchBusWaits(f *signalFacts) []conditions.Observation {
|
|
||||||
b := f.bus
|
|
||||||
if len(b.waits) == 0 || len(b.machines) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
since := b.waits[0].since
|
|
||||||
var walks, what []string
|
|
||||||
for _, w := range b.waits {
|
|
||||||
walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit),
|
|
||||||
strings.Join(w.modules, ", ")))
|
|
||||||
what = append(what, deliveryWhat(w.modules, w.repository))
|
|
||||||
}
|
|
||||||
var from []string
|
|
||||||
for _, n := range b.machines {
|
|
||||||
from = append(from, short(orNotKnown(b.from[n])))
|
|
||||||
}
|
|
||||||
machines := namesWords(b.machines, 3)
|
|
||||||
in := f.now.Sub(since)
|
|
||||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting",
|
|
||||||
Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
|
||||||
Machine: b.machines[0], Also: b.machines[1:],
|
|
||||||
Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+
|
|
||||||
"the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "),
|
|
||||||
b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb,
|
|
||||||
since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")),
|
|
||||||
Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)),
|
|
||||||
Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax),
|
|
||||||
Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+
|
|
||||||
"installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines,
|
|
||||||
namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax),
|
|
||||||
Needs: "start the bus upgrade " + FromMeshMCPServer,
|
|
||||||
Resolved: "Resolved: the bus upgrade started, and sends go on"}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sortedUnique is a list sorted, each once.
|
|
||||||
func sortedUnique(xs []string) []string {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var out []string
|
|
||||||
for _, x := range xs {
|
|
||||||
if !seen[x] {
|
|
||||||
seen[x] = true
|
|
||||||
out = append(out, x)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// clipWords keeps a plain sentence within a bound, at a word.
|
|
||||||
func clipWords(s string, n int) string {
|
|
||||||
if len(s) <= n {
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
cut := strings.LastIndex(s[:n-1], " ")
|
|
||||||
if cut <= 0 {
|
|
||||||
cut = n - 1
|
|
||||||
}
|
|
||||||
return s[:cut] + "…"
|
|
||||||
}
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
|
||||||
b := aBusOnAnchor()
|
|
||||||
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
|
|
||||||
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
|
|
||||||
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
|
|
||||||
Updated: updated}
|
|
||||||
}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
bus busPending
|
|
||||||
plans []inventory.Plan
|
|
||||||
first map[string]time.Time
|
|
||||||
want []string
|
|
||||||
since time.Time
|
|
||||||
}{
|
|
||||||
{"a walk refused for the bus", b,
|
|
||||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
|
|
||||||
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
|
|
||||||
{"refused earlier than its last save, as this controller saw it", b,
|
|
||||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
|
|
||||||
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
|
|
||||||
{"a walk refused for another reason", b,
|
|
||||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
|
|
||||||
nil, time.Time{}},
|
|
||||||
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
|
|
||||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
|
||||||
{"the bus already runs the build held: its step started", func() busPending {
|
|
||||||
o := aBusOnAnchor()
|
|
||||||
o.from = map[string]string{"anchor": o.to}
|
|
||||||
return o
|
|
||||||
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
|
||||||
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
|
|
||||||
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
|
|
||||||
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
|
|
||||||
return []inventory.Plan{p}
|
|
||||||
}(), nil, nil, time.Time{}},
|
|
||||||
} {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
|
|
||||||
var ids []string
|
|
||||||
for _, w := range got.waits {
|
|
||||||
ids = append(ids, w.plan)
|
|
||||||
}
|
|
||||||
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
|
|
||||||
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
|
|
||||||
}
|
|
||||||
if len(c.want) > 0 {
|
|
||||||
if !got.waits[0].since.Equal(c.since) {
|
|
||||||
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
|
|
||||||
}
|
|
||||||
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
|
|
||||||
t.Errorf("behind %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a
|
|
||||||
// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that
|
|
||||||
// ends it — and the wait is not read as a walk running late.
|
|
||||||
|
|
||||||
// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held.
|
|
||||||
func aBusOnAnchor() busPending {
|
|
||||||
return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"},
|
|
||||||
to: "32307bd1bbbb", same: map[string]bool{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it.
|
|
||||||
func refusedNote(b busPending, tier int) string {
|
|
||||||
held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) +
|
|
||||||
"), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)"
|
|
||||||
return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again"
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises
|
|
||||||
// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to,
|
|
||||||
// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the
|
|
||||||
// condition clears once the bus's machine runs the new build.
|
|
||||||
func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
|
||||||
b := aBusOnAnchor()
|
|
||||||
held := func(entered time.Duration) *signalFacts {
|
|
||||||
f := calm(now)
|
|
||||||
f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1,
|
|
||||||
entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}}
|
|
||||||
f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"},
|
|
||||||
waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122",
|
|
||||||
modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
f := held(time.Minute)
|
|
||||||
got := watchBusWaits(f)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("a send held for the bus a minute ago raised %+v", got)
|
|
||||||
}
|
|
||||||
o := got[0]
|
|
||||||
if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator {
|
|
||||||
t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus",
|
|
||||||
`"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} {
|
|
||||||
if !strings.Contains(o.Summary, want) {
|
|
||||||
t.Errorf("its summary does not say %q: %s", want, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" {
|
|
||||||
t.Errorf("its words do not say what waits and what the operator does: %+v", o)
|
|
||||||
}
|
|
||||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
|
||||||
Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok {
|
|
||||||
t.Errorf("its words are not plain: %s", why)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Past S3's bound: still the bus's wait, never a stalled walk.
|
|
||||||
late := held(45 * time.Minute)
|
|
||||||
if s3 := watchPlans(late); len(s3) != 0 {
|
|
||||||
t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3)
|
|
||||||
}
|
|
||||||
// A walk held for something else past its bound is still stalled.
|
|
||||||
other := held(45 * time.Minute)
|
|
||||||
other.plans[0].bus = false
|
|
||||||
if s3 := watchPlans(other); len(s3) != 1 {
|
|
||||||
t.Fatalf("a walk held for something else past its bound raised %+v", s3)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Through the keeper: open at the first tick, cleared when the step started.
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{},
|
|
||||||
Now: func() time.Time { return now }})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
||||||
w.see(t.Context(), held(time.Minute))
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" {
|
|
||||||
t.Fatalf("after the first tick, open: %+v (%v)", open, err)
|
|
||||||
}
|
|
||||||
started := held(time.Minute)
|
|
||||||
started.bus = busFacts{module: "nats", to: b.to}
|
|
||||||
started.plans[0].bus = false
|
|
||||||
w.see(t.Context(), started)
|
|
||||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
||||||
t.Fatalf("the step started, and open: %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,258 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"crypto/rsa"
|
|
||||||
"crypto/x509"
|
|
||||||
"crypto/x509/pkix"
|
|
||||||
"encoding/pem"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math/big"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The bus's own certificate, made by the mesh rather than borrowed from an image.
|
|
||||||
//
|
|
||||||
// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq
|
|
||||||
// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's
|
|
||||||
// image, which worked while the broker was one that happened to carry it and stopped the day the
|
|
||||||
// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be
|
|
||||||
// raised. Substituting another image the bundle names does not help — none of them carry it
|
|
||||||
// either.
|
|
||||||
//
|
|
||||||
// So the program that needs a certificate makes one. It is the mesh's own binary, already on the
|
|
||||||
// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the
|
|
||||||
// image it writes into but a mounted directory.
|
|
||||||
//
|
|
||||||
// **Self-signed, and that is the design** — a host pins this server's exact certificate and
|
|
||||||
// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at
|
|
||||||
// this moment in a bootstrap there is no mesh to ask one of.
|
|
||||||
//
|
|
||||||
// Idempotent, because the step is applied again on every reconcile and a second certificate would
|
|
||||||
// be one the hosts that pinned the first no longer believe.
|
|
||||||
|
|
||||||
// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the
|
|
||||||
// loopback address the machine's own foundation dials.
|
|
||||||
var busCertificateNames = []string{"mesh-broker"}
|
|
||||||
|
|
||||||
const busCertificateLife = 10 * 365 * 24 * time.Hour
|
|
||||||
|
|
||||||
// busCertificate makes the bus's certificate in a directory, or says whether one is there.
|
|
||||||
//
|
|
||||||
// broker certificate --into /tls make it if it is not there
|
|
||||||
// broker certificate --check --into /tls exit non-zero unless a usable pair is
|
|
||||||
func busCertificate(args []string) error {
|
|
||||||
into, check := "", false
|
|
||||||
for i := 0; i < len(args); i++ {
|
|
||||||
switch args[i] {
|
|
||||||
case "--check":
|
|
||||||
check = true
|
|
||||||
case "--into":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return errors.New("--into needs a directory")
|
|
||||||
}
|
|
||||||
into = args[i+1]
|
|
||||||
i++
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("broker certificate [--check] --into <directory>: %q", args[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if into == "" {
|
|
||||||
return errors.New("broker certificate [--check] --into <directory>")
|
|
||||||
}
|
|
||||||
crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")
|
|
||||||
|
|
||||||
if usable, err := busCertificateUsable(crt, key); err != nil {
|
|
||||||
return err
|
|
||||||
} else if usable {
|
|
||||||
fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if check {
|
|
||||||
// Said as a failure, because that is what the caller asked: a bootstrap's verify runs
|
|
||||||
// this and a false answer is what makes the step run.
|
|
||||||
return fmt.Errorf("no usable certificate and key at %s", into)
|
|
||||||
}
|
|
||||||
return writeBusCertificate(crt, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busCertificateUsable says whether a certificate and its key are both there and parse.
|
|
||||||
//
|
|
||||||
// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted
|
|
||||||
// between the two files leaves behind, and a step that treated it as done would hand the server a
|
|
||||||
// certificate with no key and report success.
|
|
||||||
func busCertificateUsable(crt, key string) (bool, error) {
|
|
||||||
certPEM, err := os.ReadFile(crt)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
keyPEM, err := os.ReadFile(key)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if _, err := tlsPairParses(certPEM, keyPEM); err != nil {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) {
|
|
||||||
block, _ := pem.Decode(certPEM)
|
|
||||||
if block == nil || block.Type != "CERTIFICATE" {
|
|
||||||
return nil, errors.New("not a certificate")
|
|
||||||
}
|
|
||||||
certificate, err := x509.ParseCertificate(block.Bytes)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
keyBlock, _ := pem.Decode(keyPEM)
|
|
||||||
if keyBlock == nil {
|
|
||||||
return nil, errors.New("not a key")
|
|
||||||
}
|
|
||||||
if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil {
|
|
||||||
if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return certificate, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeBusCertificate(crt, key string) error {
|
|
||||||
private, err := rsa.GenerateKey(rand.Reader, 2048)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
template := &x509.Certificate{
|
|
||||||
SerialNumber: serial,
|
|
||||||
Subject: pkix.Name{CommonName: busCertificateNames[0]},
|
|
||||||
DNSNames: busCertificateNames,
|
|
||||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
|
||||||
NotBefore: time.Now().Add(-time.Hour),
|
|
||||||
NotAfter: time.Now().Add(busCertificateLife),
|
|
||||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
|
||||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
||||||
BasicConstraintsValid: true,
|
|
||||||
}
|
|
||||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
pkcs8, err := x509.MarshalPKCS8PrivateKey(private)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The key first, and only then the certificate**, so the pair a reader finds is never a
|
|
||||||
// certificate whose key has not been written yet — the one order in which an interruption
|
|
||||||
// leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable).
|
|
||||||
if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n",
|
|
||||||
busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// busAccounts writes the mesh's composed user list to a file.
|
|
||||||
//
|
|
||||||
// **For genesis, where no declaration can deliver it** (novox/hq 04-ISSUES/146). Everywhere else
|
|
||||||
// the list reaches the machine running the bus as a resource of the module that holds it — which
|
|
||||||
// requires that machine to be an enrolled node, and at genesis it is not: the first node cannot
|
|
||||||
// enrol because the account it would enrol with cannot be composed onto a bus it has no declaration
|
|
||||||
// for. The installer breaks that circle by placing the file itself, once, and the module takes the
|
|
||||||
// file over from its first push.
|
|
||||||
//
|
|
||||||
// The same composition, not a second one: this asks the store for the same records and renders them
|
|
||||||
// with the same composer the declaration uses. A genesis that hand-wrote an account would be a
|
|
||||||
// second statement of who may say what, able to disagree with the first.
|
|
||||||
//
|
|
||||||
// **It writes to standard output unless told a file**, and that is the point: the control plane
|
|
||||||
// composes and says what it composed, and whoever is raising the machine puts it where that
|
|
||||||
// machine's bus reads it. A control plane that wrote into the bus's own directory would have to
|
|
||||||
// know where that is and how to make the server re-read it — which is the module's knowledge, and
|
|
||||||
// the module is what takes this over on the first push.
|
|
||||||
//
|
|
||||||
// broker accounts > /var/lib/mesh-bus-conf/accounts.conf
|
|
||||||
func busAccounts(ctx context.Context, args []string) error {
|
|
||||||
into := ""
|
|
||||||
for i := 0; i < len(args); i++ {
|
|
||||||
switch args[i] {
|
|
||||||
case "--into":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return errors.New("--into needs a file")
|
|
||||||
}
|
|
||||||
into = args[i+1]
|
|
||||||
i++
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("broker accounts --into <file>: %q", args[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
|
|
||||||
records, err := open.inventory.BusRecords(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
users, err := broker.Users(records)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
kept, err := open.inventory.BusUsers(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
hashes := make(map[string]string, len(kept))
|
|
||||||
for name, u := range kept {
|
|
||||||
hashes[name] = u.PasswordHash
|
|
||||||
}
|
|
||||||
filled, missing := broker.WithPasswords(users, hashes)
|
|
||||||
if len(missing) > 0 {
|
|
||||||
// To standard error, always: the composed file may be going to standard output, and a
|
|
||||||
// remark in the middle of it is a configuration the server refuses to parse.
|
|
||||||
fmt.Fprintf(os.Stderr, "leaving out %d user(s) the mesh has minted no credential for: %s\n",
|
|
||||||
len(missing), strings.Join(missing, ", "))
|
|
||||||
}
|
|
||||||
if len(filled) == 0 {
|
|
||||||
return errors.New("not one user has a credential, so this list would refuse every " +
|
|
||||||
"connection in the mesh")
|
|
||||||
}
|
|
||||||
accounts, err := broker.ComposeAccounts(filled)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if into == "" {
|
|
||||||
fmt.Print(accounts)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(into, []byte(accounts), 0o600); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Printf("wrote %d user(s) to %s\n", len(filled), into)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,121 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/tls"
|
|
||||||
"crypto/x509"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for
|
|
||||||
// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is
|
|
||||||
// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once.
|
|
||||||
|
|
||||||
func TestTheBusCertificateLoadsAsAServersWould(t *testing.T) {
|
|
||||||
into := t.TempDir()
|
|
||||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
|
||||||
t.Fatalf("the bus could not be given a certificate: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The check a cheaper test would not make. The key was present and valid and the server could
|
|
||||||
// not start, once, because nothing loaded the pair the way a server loads it
|
|
||||||
// (novox/hq 04-ISSUES/014).
|
|
||||||
pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("a TLS server cannot load what was written: %v", err)
|
|
||||||
}
|
|
||||||
leaf := pair.Leaf
|
|
||||||
if leaf == nil {
|
|
||||||
if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := leaf.VerifyHostname("mesh-broker"); err != nil {
|
|
||||||
t.Errorf("the certificate is not for the name the bus is reached by: %v", err)
|
|
||||||
}
|
|
||||||
if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" {
|
|
||||||
t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The key is not readable by anything else on the machine; the certificate is public and is.
|
|
||||||
key, err := os.Stat(filepath.Join(into, "tls.key"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if key.Mode().Perm() != 0o600 {
|
|
||||||
t.Errorf("the key is %v", key.Mode().Perm())
|
|
||||||
}
|
|
||||||
crt, err := os.Stat(filepath.Join(into, "tls.crt"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if crt.Mode().Perm() != 0o644 {
|
|
||||||
t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Made once.** The step is applied again on every reconcile, and a second certificate is one the
|
|
||||||
// hosts that pinned the first no longer believe (novox/hq ADR 0004).
|
|
||||||
func TestTheBusCertificateIsMadeOnce(t *testing.T) {
|
|
||||||
into := t.TempDir()
|
|
||||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
first, err := os.ReadFile(filepath.Join(into, "tls.crt"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
again, err := os.ReadFile(filepath.Join(into, "tls.crt"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if string(first) != string(again) {
|
|
||||||
t.Fatal("running it twice replaced the certificate every host had pinned")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The verify half: false before, true after, which is what makes the bootstrap run the step at all.
|
|
||||||
func TestTheCheckIsFalseUntilThereIsAPair(t *testing.T) {
|
|
||||||
into := t.TempDir()
|
|
||||||
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
|
|
||||||
t.Fatal("an empty directory reported a usable certificate")
|
|
||||||
}
|
|
||||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := busCertificate([]string{"--check", "--into", into}); err != nil {
|
|
||||||
t.Fatalf("the certificate it just made does not satisfy its own check: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that
|
|
||||||
// called it done would hand the server a certificate with no key and report success.
|
|
||||||
func TestACertificateWithoutItsKeyIsNotUsable(t *testing.T) {
|
|
||||||
into := t.TempDir()
|
|
||||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.Remove(filepath.Join(into, "tls.key")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
|
|
||||||
t.Fatal("a certificate with no key passed the check")
|
|
||||||
}
|
|
||||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil {
|
|
||||||
t.Fatalf("it did not replace the unusable pair: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWhereToWriteIsRequired(t *testing.T) {
|
|
||||||
if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") {
|
|
||||||
t.Fatalf("it did not ask where to write: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,254 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
|
|
||||||
// D6 and D7).
|
|
||||||
//
|
|
||||||
// **What the controller asserts at its start and what the self-check expects to find are one
|
|
||||||
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
|
|
||||||
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
|
|
||||||
// the drift rather than the fault.
|
|
||||||
|
|
||||||
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
|
|
||||||
// can now hear a declaration.
|
|
||||||
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(nodes))
|
|
||||||
for _, n := range nodes {
|
|
||||||
names = append(names, n.Name)
|
|
||||||
}
|
|
||||||
if err := broker.Raise(r, names); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
|
|
||||||
// whenever the bus is up.
|
|
||||||
if files, ok := r.(interface{ EnsureDeclarationFiles(nodes []string) error }); ok {
|
|
||||||
if err := files.EnsureDeclarationFiles(names); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
|
||||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
|
||||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
|
||||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
|
||||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
|
||||||
// consumer nothing had created (2026-09-28).
|
|
||||||
holders, err := seatHolders(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
|
||||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
|
||||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
|
||||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
|
||||||
consumers, records, err := moduleConsumersAndRecords(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// And a grant of a restricted stream that composes nothing, said on every raise and send (novox/hq ADR 0376 §2).
|
|
||||||
sayWithheld(records, " ")
|
|
||||||
// And the work queues of seats that name their caller or their kind, with each holder's worker
|
|
||||||
// (novox/hq ADR 0259 §3): an ask queues until the router takes it, a channel's work until that kind
|
|
||||||
// takes it.
|
|
||||||
trafficStreams, trafficWorkers, err := seatTrafficObjects(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// And every module's own stream (novox/hq ADR 0363), from the catalogue, before the consumers on it.
|
|
||||||
modStreams, err := inv.DeclaredStreams(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
|
||||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
|
||||||
var failed []error
|
|
||||||
for _, s := range broker.ModuleStreams(modStreams) {
|
|
||||||
if err := r.EnsureStream(s); err != nil {
|
|
||||||
failed = append(failed, fmt.Errorf("the module stream %s: %w", s.Name, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, s := range trafficStreams {
|
|
||||||
if err := r.EnsureStream(s); err != nil {
|
|
||||||
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, c := range trafficWorkers {
|
|
||||||
if err := r.EnsureConsumer(c); err != nil {
|
|
||||||
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
restricted := map[string]bool{}
|
|
||||||
for _, s := range modStreams {
|
|
||||||
restricted[s.Name] = s.Restricted
|
|
||||||
}
|
|
||||||
for _, c := range consumers {
|
|
||||||
if err := broker.EnsureHeard(r, c.Consumer, restricted[c.Consumer.Stream]); err != nil {
|
|
||||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **And a consumer no longer granted loses what it reads through** (novox/hq ADR 0363 §6): every consumer
|
|
||||||
// on a module's stream the composition no longer asks for is removed, on this very send.
|
|
||||||
if p, prunes := r.(broker.Pruner); prunes && len(modStreams) > 0 {
|
|
||||||
var wanted []broker.Consumer
|
|
||||||
for _, c := range consumers {
|
|
||||||
wanted = append(wanted, c.Consumer)
|
|
||||||
}
|
|
||||||
removed, err := broker.PruneHeard(p, modStreams, wanted)
|
|
||||||
if err != nil {
|
|
||||||
failed = append(failed, err)
|
|
||||||
}
|
|
||||||
for _, gone := range removed {
|
|
||||||
fmt.Printf(" removed the consumer %s: its module is no longer granted that stream (novox/hq ADR 0363)\n", gone)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(failed) > 0 {
|
|
||||||
return nil, errors.Join(failed...)
|
|
||||||
}
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// What raises the condition a send says when the objects it implies could not be asserted, and its kind.
|
|
||||||
const (
|
|
||||||
sourceBusObjects = "bus-objects"
|
|
||||||
kindBusObjectsUnasserted = "bus-objects-unasserted"
|
|
||||||
)
|
|
||||||
|
|
||||||
// assertOnSend asserts, on the bus a send is about to use, every object assertBusObjects derives —
|
|
||||||
// **whenever a declaration is sent, not only when the controller starts** (novox/hq issue 208).
|
|
||||||
//
|
|
||||||
// A module assigned after the controller started was sent its declaration and found no consumer to
|
|
||||||
// bind (`consumer not found`, messenger on 2026-10-06), and a seat holder assigned after it found no
|
|
||||||
// worker: the objects a declaration implies were asserted at start and nowhere else, so they existed
|
|
||||||
// only for what was assigned before the last restart. The same derivation, not a second list of what
|
|
||||||
// a send needs: what start asserts, the self-check expects and a send asserts are one answer. Every
|
|
||||||
// part is idempotent, so asserting the whole of it again is the no-op a restart already relies on.
|
|
||||||
//
|
|
||||||
// **A failure is said and raised, and the send goes on.** The objects are the mesh's, not the
|
|
||||||
// machines' being sent: holding every machine back for one consumer that none of them may use would
|
|
||||||
// turn one fault into all of them, and the declarations are not what is wrong. It is never silent —
|
|
||||||
// said in the send's own output and raised as a condition, which the next send that asserts them
|
|
||||||
// clears — and the start-time raise still refuses to serve without them.
|
|
||||||
func assertOnSend(ctx context.Context, inv *inventory.Inventory, r broker.Raiser, indent string) error {
|
|
||||||
_, err := assertBusObjects(ctx, inv, r)
|
|
||||||
var observed []conditions.Observation
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("%sTHE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES: %v\n", indent, err)
|
|
||||||
fmt.Printf("%s a module may find no consumer to bind, or a holder no worker; sent anyway, raised as "+
|
|
||||||
"condition %s, and asserted again by the next send\n", indent, unassertedObservation(err).Key())
|
|
||||||
observed = append(observed, unassertedObservation(err))
|
|
||||||
}
|
|
||||||
// Observed when it failed, cleared when it did not: a send that asserted everything is the
|
|
||||||
// observation that the bus holds what it should.
|
|
||||||
if kerr := withKeeper(ctx, func(k *conditions.Keeper) error {
|
|
||||||
return k.Reconcile(ctx, sourceBusObjects, observed)
|
|
||||||
}); kerr != nil {
|
|
||||||
fmt.Printf("%sand whether the bus holds what this send implies could not be kept as a condition: %v\n",
|
|
||||||
indent, kerr)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// unassertedObservation is a send's failure to assert the bus's objects, as a condition.
|
|
||||||
func unassertedObservation(err error) conditions.Observation {
|
|
||||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "objects", Token: "unasserted",
|
|
||||||
Kind: kindBusObjectsUnasserted, Severity: conditions.Warning, Source: sourceBusObjects,
|
|
||||||
Summary: "the bus's streams and consumers could not be asserted when a declaration was sent: " +
|
|
||||||
"a module may find no consumer to bind, or a seat's holder no worker",
|
|
||||||
Said: err.Error()}
|
|
||||||
}
|
|
||||||
|
|
||||||
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
|
||||||
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
|
||||||
consumers, _, err := moduleConsumersAndRecords(ctx, inv)
|
|
||||||
return consumers, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// moduleConsumersAndRecords is moduleConsumers with the records they were composed from.
|
|
||||||
func moduleConsumersAndRecords(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, broker.Records, error) {
|
|
||||||
records, err := inv.BusRecords(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, broker.Records{}, err
|
|
||||||
}
|
|
||||||
users, err := broker.Users(records)
|
|
||||||
if err != nil {
|
|
||||||
return nil, broker.Records{}, err
|
|
||||||
}
|
|
||||||
return broker.ConsumersOf(users), records, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// seatTrafficObjects is the work queues and workers of seats that name their caller or their kind, from
|
|
||||||
// the records the user list is composed from.
|
|
||||||
func seatTrafficObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
|
||||||
records, err := inv.BusRecords(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
users, err := broker.Users(records)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
streams, workers := broker.SeatTrafficObjects(users)
|
|
||||||
// And the queue of every such seat the catalogue declares, held or not: work queues from registration,
|
|
||||||
// so what is submitted before a holder is assigned waits for it (the correctness review of 2026-10-08).
|
|
||||||
declared, err := inv.DeclaredTrafficSeats(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
have := map[string]bool{}
|
|
||||||
for _, s := range streams {
|
|
||||||
have[s.Name] = true
|
|
||||||
}
|
|
||||||
for _, s := range broker.TrafficQueues(declared) {
|
|
||||||
if !have[s.Name] {
|
|
||||||
streams = append(streams, s)
|
|
||||||
have[s.Name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return streams, workers, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
|
||||||
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
|
||||||
consumers, err := moduleConsumers(ctx, inv)
|
|
||||||
return len(consumers), err
|
|
||||||
}
|
|
||||||
|
|
||||||
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
|
|
||||||
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
|
||||||
var rec recordingRaiser
|
|
||||||
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
|
|
||||||
}
|
|
||||||
return rec.streams, rec.consumers, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordingRaiser keeps what it was asked to assert and asserts nothing.
|
|
||||||
type recordingRaiser struct {
|
|
||||||
streams []broker.Stream
|
|
||||||
consumers []broker.Consumer
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
|
|
||||||
r.streams = append(r.streams, s)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
|
|
||||||
r.consumers = append(r.consumers, c)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,207 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
|
||||||
// only when the controller starts.
|
|
||||||
|
|
||||||
// aCarriedConsumer is the runtime on laptop and, carried by it, a module that consumes an event: the
|
|
||||||
// shape of messenger on 2026-10-06, assigned after the controller started.
|
|
||||||
func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
|
|
||||||
t.Helper()
|
|
||||||
register(t, open, catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1",
|
|
||||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "messenger", Version: "1",
|
|
||||||
Consumes: []string{"billing.order.placed"}})
|
|
||||||
for _, m := range []string{catalogue.RuntimeModule, "messenger"} {
|
|
||||||
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
consumers, err := moduleConsumers(t.Context(), open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, c := range consumers {
|
|
||||||
if c.Module == "messenger" && c.Node == "laptop" {
|
|
||||||
return c.Consumer
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("messenger on laptop is derived no consumer: %+v", consumers)
|
|
||||||
return broker.Consumer{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// aLateHolder is a module holding a build seat (node-build-go) on anchor, assigned after the controller started, and
|
|
||||||
// the worker its seat's queue should have for it: a build seat is pulled by kind, so the worker is the one for the
|
|
||||||
// seat's first accepted kind (the build agent's seat, which had one worker for all, is gone with the builder).
|
|
||||||
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
|
|
||||||
t.Helper()
|
|
||||||
// With an account to read, as a holder has: a module with none is no bus user, so its seat's workers are not derived.
|
|
||||||
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
|
|
||||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/late-builder/broker"}},
|
|
||||||
Claims: []catalogue.Claim{{Name: "node-build-go", Scope: catalogue.ScopeNode}}})
|
|
||||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, workers, err := seatTrafficObjects(t.Context(), open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, s := range inventory.MeshSeats() {
|
|
||||||
if s.Name != "node-build-go" || len(s.Accepts) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, w := range workers {
|
|
||||||
if w.Name == broker.WorkerName(s.Name, s.Accepts[0]) {
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatal("the build seat's holder is derived no worker")
|
|
||||||
return broker.Consumer{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// asserted says whether a recording holds a consumer by stream and name.
|
|
||||||
func asserted(rec *recordingRaiser, want broker.Consumer) bool {
|
|
||||||
for _, c := range rec.consumers {
|
|
||||||
if c.Stream == want.Stream && c.Name == want.Name {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// What a send asserts includes what was assigned after the start's assertion: a carried module's
|
|
||||||
// consumer and a late holder's worker. The derivation is the start's own, so this holds without a bus.
|
|
||||||
func TestASendAssertsWhatWasAssignedAfterStart(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
var atStart recordingRaiser
|
|
||||||
if _, err := assertBusObjects(t.Context(), open.inventory, &atStart); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
consumer := aCarriedConsumer(t, open)
|
|
||||||
worker := aLateHolder(t, open)
|
|
||||||
if asserted(&atStart, consumer) || asserted(&atStart, worker) {
|
|
||||||
t.Fatal("the start asserted what was not yet assigned; the test proves nothing")
|
|
||||||
}
|
|
||||||
var onSend recordingRaiser
|
|
||||||
if err := assertOnSend(t.Context(), open.inventory, &onSend, ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !asserted(&onSend, consumer) {
|
|
||||||
t.Fatalf("messenger's consumer %s on %s is not asserted by the send: %+v", consumer.Name, consumer.Stream, onSend.consumers)
|
|
||||||
}
|
|
||||||
if !asserted(&onSend, worker) {
|
|
||||||
t.Fatalf("the late holder's worker %s on %s is not asserted by the send: %+v", worker.Name, worker.Stream, onSend.consumers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// failingRaiser refuses one consumer by name and records everything it was asked.
|
|
||||||
type failingRaiser struct {
|
|
||||||
recordingRaiser
|
|
||||||
refuse string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *failingRaiser) EnsureConsumer(c broker.Consumer) error {
|
|
||||||
f.consumers = append(f.consumers, c)
|
|
||||||
if c.Name == f.refuse {
|
|
||||||
return errors.New("nats: API error: code=503 description=insufficient resources")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// A send whose objects cannot be asserted says so in its output and raises a condition — and the next
|
|
||||||
// send that asserts them clears it. The consumers after the refused one are still asked for.
|
|
||||||
func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
consumer := aCarriedConsumer(t, open)
|
|
||||||
worker := aLateHolder(t, open)
|
|
||||||
failing := &failingRaiser{refuse: consumer.Name}
|
|
||||||
|
|
||||||
var sendErr error
|
|
||||||
out := stdoutOf(t, func() error {
|
|
||||||
sendErr = assertOnSend(t.Context(), open.inventory, failing, " ")
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if sendErr == nil || !strings.Contains(sendErr.Error(), "how messenger on laptop hears what it consumes") {
|
|
||||||
t.Fatalf("the failure is not answered: %v", sendErr)
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, "THE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES") ||
|
|
||||||
!strings.Contains(out, "insufficient resources") || !strings.Contains(out, "bus.objects.unasserted") {
|
|
||||||
t.Fatalf("the failure is not said in the send's output:\n%s", out)
|
|
||||||
}
|
|
||||||
if !asserted(&failing.recordingRaiser, worker) {
|
|
||||||
t.Fatal("the seat's worker was not asked for")
|
|
||||||
}
|
|
||||||
c, open1, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted")
|
|
||||||
if err != nil || !open1 {
|
|
||||||
t.Fatalf("no condition raised: %v", err)
|
|
||||||
}
|
|
||||||
if c.Kind != kindBusObjectsUnasserted || c.Source != sourceBusObjects ||
|
|
||||||
len(c.Evidence) == 0 || !strings.Contains(c.Evidence[0].Said, "insufficient resources") {
|
|
||||||
t.Fatalf("the condition does not say what failed: %+v", c)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The next send asserts them, and that observation clears it.
|
|
||||||
if err := assertOnSend(t.Context(), open.inventory, &recordingRaiser{}, ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, stillOpen, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); err != nil || stillOpen {
|
|
||||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
|
||||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
|
||||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
|
||||||
url := testbus.URL(t)
|
|
||||||
open := aMesh(t)
|
|
||||||
js, err := broker.Dial(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
|
||||||
_ = js.Context().DeleteStream(s)
|
|
||||||
}
|
|
||||||
// The controller starting, before either was assigned.
|
|
||||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
consumer := aCarriedConsumer(t, open)
|
|
||||||
worker := aLateHolder(t, open)
|
|
||||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
|
||||||
for _, c := range []broker.Consumer{consumer, worker} {
|
|
||||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
|
||||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
server := link.ConnectNats(js, nil, nil)
|
|
||||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, c := range []broker.Consumer{consumer, worker} {
|
|
||||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
|
||||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
|
||||||
t.Fatal("a send that asserted everything raised a condition")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,103 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
|
||||||
//
|
|
||||||
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
|
||||||
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
|
||||||
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
|
||||||
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
|
||||||
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
|
||||||
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
|
||||||
// its gate starts as before.
|
|
||||||
|
|
||||||
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
|
||||||
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
|
||||||
var busyGateSlack = 5 * time.Minute
|
|
||||||
|
|
||||||
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
|
|
||||||
// controller's clock from when it first read the wait.
|
|
||||||
const busyEngineBound = 15 * time.Minute
|
|
||||||
|
|
||||||
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
|
|
||||||
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
|
|
||||||
// machine nor a wrong word from it holds a walk longer.
|
|
||||||
func busyGateEnds(b *inventory.GateBusy) time.Time {
|
|
||||||
until := b.Seen.Add(busyEngineBound)
|
|
||||||
if !b.Until.IsZero() && b.Until.Before(until) {
|
|
||||||
until = b.Until
|
|
||||||
}
|
|
||||||
return until.Add(busyGateSlack)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
|
||||||
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
|
||||||
// gate's Since to its end. Pure but for the gate it is handed.
|
|
||||||
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
|
||||||
if g.Busy != nil && g.Busy.Ended != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for _, j := range pairs {
|
|
||||||
b, waits := busyOn(j, f)
|
|
||||||
if !waits {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if g.Busy == nil {
|
|
||||||
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
|
|
||||||
Seen: now}
|
|
||||||
}
|
|
||||||
if g.Busy.Seen.IsZero() {
|
|
||||||
g.Busy.Seen = now
|
|
||||||
}
|
|
||||||
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
|
||||||
if now.After(busyGateEnds(g.Busy)) {
|
|
||||||
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
|
||||||
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
|
||||||
end := now
|
|
||||||
g.Busy.Ended, g.Busy.PastBound = &end, true
|
|
||||||
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
|
||||||
g.Read(now, false, g.Busy.Words())
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if g.Busy == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
// Replaced: the judging starts at the report that said so.
|
|
||||||
end := now
|
|
||||||
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
|
||||||
end = r.At.UTC()
|
|
||||||
}
|
|
||||||
g.Busy.Ended = &end
|
|
||||||
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
|
||||||
// running because it said busy.
|
|
||||||
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
|
||||||
r, said := f.reports[j.node]
|
|
||||||
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
|
||||||
return inventory.BusyWait{}, false
|
|
||||||
}
|
|
||||||
return inventory.WaitingFor(r.Busy, j.module)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
|
||||||
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
|
||||||
func busyStill(j judged, f gateFacts) (string, bool) {
|
|
||||||
b, waits := busyOn(j, f)
|
|
||||||
if !waits {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
|
||||||
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
|
||||||
}
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
|
||||||
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
|
||||||
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
|
||||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
|
||||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
|
||||||
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
|
||||||
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
|
||||||
reported := sent.Add(20 * time.Second)
|
|
||||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
|
||||||
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
|
||||||
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
|
||||||
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
|
||||||
// As judgeMoves hands it: the report held against the gate's own send.
|
|
||||||
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
|
||||||
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
now := reported.Add(time.Minute)
|
|
||||||
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
|
||||||
t.Fatal("a gate whose module's container waits as busy judged it")
|
|
||||||
}
|
|
||||||
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
|
||||||
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
|
||||||
}
|
|
||||||
for _, said := range []string{gateLine(g), g.Last} {
|
|
||||||
if !strings.Contains(said, "waits for a busy container on novox") {
|
|
||||||
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(g.Last, "check of #280") {
|
|
||||||
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
|
||||||
now = reported.Add(10 * time.Minute)
|
|
||||||
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
|
||||||
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The machine reports it replaced: the judging starts at that report.
|
|
||||||
replacedAt := reported.Add(12 * time.Minute)
|
|
||||||
now = replacedAt.Add(5 * time.Second)
|
|
||||||
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
|
||||||
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
|
||||||
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
|
||||||
t.Fatal("the gate still waits after the container was replaced")
|
|
||||||
}
|
|
||||||
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
|
||||||
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
|
||||||
}
|
|
||||||
if strings.Contains(gateLine(g), "waits for a busy container") {
|
|
||||||
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
|
||||||
}
|
|
||||||
// And a later report listing it waiting again does not hold a gate that started.
|
|
||||||
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
|
||||||
t.Error("a gate that started waits again")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
|
||||||
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
|
||||||
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
|
||||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
|
||||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
|
||||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
|
||||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
|
||||||
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
|
||||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &sent, Current: true, Busy: waiting}}}
|
|
||||||
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
|
||||||
if busyWait(g, pairs, f, now) {
|
|
||||||
t.Fatal("a gate waited past the engine's bound and the slack")
|
|
||||||
}
|
|
||||||
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
|
||||||
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
|
||||||
}
|
|
||||||
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
|
||||||
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
|
||||||
// report not about the gate's send says nothing of it.
|
|
||||||
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
|
||||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
|
||||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
|
||||||
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
|
||||||
pairs := []judged{{module: "build-agent", node: "ace"}}
|
|
||||||
at := sent.Add(time.Minute)
|
|
||||||
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
|
||||||
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
|
||||||
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
|
||||||
}
|
|
||||||
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
|
||||||
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
|
||||||
if busyWait(g, pairs, elsewhere, at) {
|
|
||||||
t.Fatal("a wait reported about an earlier declaration held the gate")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
|
|
||||||
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
|
|
||||||
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
|
|
||||||
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
|
||||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
|
|
||||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
|
||||||
ahead := seen.Add(time.Hour)
|
|
||||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
|
||||||
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
|
|
||||||
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
|
|
||||||
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
|
|
||||||
}
|
|
||||||
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
|
|
||||||
t.Fatal("the gate stopped waiting within the bound on its own clock")
|
|
||||||
}
|
|
||||||
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
|
|
||||||
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
|
|
||||||
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
|
|
||||||
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
|
|
||||||
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
|
|
||||||
{"id": "agent-env", "type": "file"}}}}
|
|
||||||
lines := busyNotSent(modules, false)
|
|
||||||
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
|
|
||||||
!strings.Contains(lines[0], "replaced at once") {
|
|
||||||
t.Fatalf("the plan does not say the busy not sent: %v", lines)
|
|
||||||
}
|
|
||||||
if lines := busyNotSent(modules, true); len(lines) != 0 {
|
|
||||||
t.Fatalf("a busy sent is said as not sent: %v", lines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
|
||||||
handlers, behind, err := seatToolHandlers()
|
|
||||||
if err != nil || len(behind) != 0 {
|
|
||||||
t.Fatalf("%v %v", behind, err)
|
|
||||||
}
|
|
||||||
calls, ok := handlers["calls"]
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("calls is not served")
|
|
||||||
}
|
|
||||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), `"node"`) {
|
|
||||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), "not across a restart") {
|
|
||||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
|
||||||
}
|
|
||||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
|
||||||
t.Errorf("calls answered %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
|
|
||||||
// RequestTimeout) — the shortest wait of a caller the mesh ships.
|
|
||||||
const consoleWaits = 30 * time.Second
|
|
||||||
|
|
||||||
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
|
|
||||||
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
|
|
||||||
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
|
|
||||||
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
|
||||||
if link.AnswerWithin >= consoleWaits/2 {
|
|
||||||
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
|
|
||||||
}
|
|
||||||
if link.AnswerWithin >= broker.ResponseTTL {
|
|
||||||
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
|
||||||
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
|
||||||
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
verb string
|
|
||||||
args map[string]any
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
|
||||||
{"push", map[string]any{"why": "w"}, true},
|
|
||||||
{"command", map[string]any{"command": "builds"}, false},
|
|
||||||
{"status", map[string]any{}, false},
|
|
||||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
|
||||||
} {
|
|
||||||
argv, err := argvFor(c.verb, c.args)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
|
||||||
}
|
|
||||||
if got := answersFirst(argv); got != c.want {
|
|
||||||
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
|
||||||
@@ -1,643 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"path"
|
|
||||||
"regexp"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **The controller catches up on merges it did not hear** (novox/hq ADR 0457, issue 1336). A merge the forge announced while the
|
|
||||||
// controller was being replaced was never heard, and nothing asked for it again: the trunk held a merge no walk answered. At its
|
|
||||||
// start, and each time it hears a merge, the controller reads the last catchUpWindow merge commits of a trunk from the forge, and
|
|
||||||
// hears each one it holds no record of as if the forge had announced it then, on the warrant the router holds for it.
|
|
||||||
// It lasts until release 6 of design 47 deletes delivery plans and batches (ADR 0429 §7, §8).
|
|
||||||
|
|
||||||
// catchUpWindow is how many of a trunk's newest merge commits are read; an older merge is a person's to decide.
|
|
||||||
const catchUpWindow = 50
|
|
||||||
|
|
||||||
// kindMergeNotHeard is the condition of a merge the controller did not hear and holds no warrant to plan.
|
|
||||||
const kindMergeNotHeard = "merge-not-heard"
|
|
||||||
|
|
||||||
// trunkMerge is a merge commit of a trunk as the forge lists it.
|
|
||||||
type trunkMerge struct {
|
|
||||||
Commit string
|
|
||||||
// Head is the merge commit's second parent: the pull request's head that was merged.
|
|
||||||
Head string
|
|
||||||
Subject string
|
|
||||||
At time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// trunkCommit is a commit of a trunk as the forge lists it, newest first: a merge commit has a second parent.
|
|
||||||
type trunkCommit struct {
|
|
||||||
Commit, SecondParent, Subject string
|
|
||||||
At time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// trunkCommitsPage is one page (fifty) of a trunk's commits, newest first; empty past the last. A variable so a test needs no
|
|
||||||
// forge.
|
|
||||||
var trunkCommitsPage = func(ctx context.Context, owner, repo, branch string, page int) ([]trunkCommit, error) {
|
|
||||||
var found struct {
|
|
||||||
Result []struct {
|
|
||||||
SHA string `json:"sha"`
|
|
||||||
Parents []struct {
|
|
||||||
SHA string `json:"sha"`
|
|
||||||
} `json:"parents"`
|
|
||||||
Commit struct {
|
|
||||||
Message string `json:"message"`
|
|
||||||
Committer struct {
|
|
||||||
Date string `json:"date"`
|
|
||||||
} `json:"committer"`
|
|
||||||
} `json:"commit"`
|
|
||||||
} `json:"result"`
|
|
||||||
}
|
|
||||||
apiPath := fmt.Sprintf("/repos/%s/%s/commits?sha=%s&limit=50&page=%d&stat=false&verification=false&files=false",
|
|
||||||
owner, repo, url.QueryEscape(branch), page)
|
|
||||||
missing, err := askForgeApi(ctx, apiPath, &found)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if missing {
|
|
||||||
return nil, fmt.Errorf("the forge holds no branch %s of %s/%s", branch, owner, repo)
|
|
||||||
}
|
|
||||||
var out []trunkCommit
|
|
||||||
for _, c := range found.Result {
|
|
||||||
at, _ := time.Parse(time.RFC3339, c.Commit.Committer.Date)
|
|
||||||
tc := trunkCommit{Commit: c.SHA, Subject: firstLine(c.Commit.Message), At: at.UTC()}
|
|
||||||
if len(c.Parents) >= 2 {
|
|
||||||
tc.SecondParent = c.Parents[1].SHA
|
|
||||||
}
|
|
||||||
out = append(out, tc)
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readTrunkMerges is the last catchUpWindow merge commits of a repository's trunk, oldest first: pages are read until that many
|
|
||||||
// merges are held or the trunk ends.
|
|
||||||
func readTrunkMerges(ctx context.Context, owner, repo, branch string) ([]trunkMerge, error) {
|
|
||||||
var merges []trunkMerge
|
|
||||||
for page := 1; page <= 8 && len(merges) < catchUpWindow; page++ {
|
|
||||||
commits, err := trunkCommitsPage(ctx, owner, repo, branch, page)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(commits) == 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
for _, c := range commits {
|
|
||||||
// A merge of the trunk is a pull request's merge into it. A commit with a second parent that is none is a merge made
|
|
||||||
// inside some branch ("Merge remote-tracking branch 'origin/main' into fix/..."), which the trunk's history lists too and
|
|
||||||
// which no Approve ever covered: read as a gap it would raise a condition for nothing a person decides.
|
|
||||||
if c.SecondParent != "" && mergesIntoTrunk(c.Subject, branch) {
|
|
||||||
merges = append(merges, trunkMerge{Commit: c.Commit, Head: c.SecondParent, Subject: c.Subject, At: c.At})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return lastMerges(merges), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sinceTheRecord are the merges that can be gaps (novox/hq ADR 0463 §2): those made since the controller's record of merges begins on
|
|
||||||
// the mesh (first, the earliest merge it holds of any repository and branch, less mergeGrace as owedLate judges). A merge before it is
|
|
||||||
// history on every trunk, and a trunk with no merge kept of its own still has the merges made since as gaps. With no merge kept
|
|
||||||
// anywhere there is no record to have a gap in.
|
|
||||||
func sinceTheRecord(merges []trunkMerge, kept bool, first time.Time) []trunkMerge {
|
|
||||||
if !kept {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var out []trunkMerge
|
|
||||||
for _, m := range merges {
|
|
||||||
if !m.At.Before(first.Add(-mergeGrace)) {
|
|
||||||
out = append(out, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastMerges is the newest catchUpWindow of merges, given newest first, oldest first.
|
|
||||||
func lastMerges(newestFirst []trunkMerge) []trunkMerge {
|
|
||||||
if len(newestFirst) > catchUpWindow {
|
|
||||||
newestFirst = newestFirst[:catchUpWindow]
|
|
||||||
}
|
|
||||||
out := slices.Clone(newestFirst)
|
|
||||||
slices.Reverse(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
var mergeSubject = regexp.MustCompile(`^Merge pull request '(.*)' \(#(\d+)\) from (\S+) into (\S+)$`)
|
|
||||||
|
|
||||||
// pullOfSubject is the pull request a merge commit's subject names: its title and number.
|
|
||||||
func pullOfSubject(subject string) (title string, number int, ok bool) {
|
|
||||||
m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject))
|
|
||||||
if m == nil {
|
|
||||||
return "", 0, false
|
|
||||||
}
|
|
||||||
n, err := strconv.Atoi(m[2])
|
|
||||||
if err != nil || n <= 0 {
|
|
||||||
return "", 0, false
|
|
||||||
}
|
|
||||||
return m[1], n, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// mergesIntoTrunk says a commit's subject is a pull request's merge into this trunk.
|
|
||||||
func mergesIntoTrunk(subject, branch string) bool {
|
|
||||||
m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject))
|
|
||||||
return m != nil && m[4] == branch
|
|
||||||
}
|
|
||||||
|
|
||||||
// headBranchOfSubject is the branch a merge commit's subject says was merged.
|
|
||||||
func headBranchOfSubject(subject string) string {
|
|
||||||
if m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)); m != nil {
|
|
||||||
return m[3]
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// readMergeEvent is the merge as the forge would have announced it: the commit, the pull request, the files it changed and the
|
|
||||||
// directories among them that hold a module.json. A variable so a test needs no forge.
|
|
||||||
var readMergeEvent = func(ctx context.Context, owner, repo, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
|
||||||
m := link.SourceMoved{Owner: owner, Repo: repo, Base: branch, Head: headBranchOfSubject(tm.Subject), HeadSHA: tm.Head,
|
|
||||||
Commit: tm.Commit, MergedAt: tm.At.UTC().Format(time.RFC3339)}
|
|
||||||
if title, number, ok := pullOfSubject(tm.Subject); ok {
|
|
||||||
m.Number, m.Title = number, title
|
|
||||||
}
|
|
||||||
var found struct {
|
|
||||||
Result struct {
|
|
||||||
Files []struct {
|
|
||||||
Filename string `json:"filename"`
|
|
||||||
Status string `json:"status"`
|
|
||||||
} `json:"files"`
|
|
||||||
} `json:"result"`
|
|
||||||
}
|
|
||||||
missing, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(tm.Commit), &found)
|
|
||||||
if err != nil {
|
|
||||||
return m, err
|
|
||||||
}
|
|
||||||
if missing || len(found.Result.Files) == 0 {
|
|
||||||
return m, fmt.Errorf("the forge named no files for the merge %.8s of %s/%s", tm.Commit, owner, repo)
|
|
||||||
}
|
|
||||||
for _, f := range found.Result.Files {
|
|
||||||
m.Paths = append(m.Paths, f.Filename)
|
|
||||||
if f.Status == "removed" || f.Status == "deleted" {
|
|
||||||
m.Removed = append(m.Removed, f.Filename)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The forge lists a merge's files by the page; a list as long as a page may be a beginning.
|
|
||||||
m.PathsTruncated = len(m.Paths) >= 300
|
|
||||||
dirs := map[string]bool{}
|
|
||||||
for _, p := range m.Paths {
|
|
||||||
for d := path.Dir(p); ; d = path.Dir(d) {
|
|
||||||
dirs[d] = true
|
|
||||||
if d == "." || d == "/" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(dirs) > 64 {
|
|
||||||
return m, nil // too many to look at: the old rule stands, a directory is a module when its manifest changed
|
|
||||||
}
|
|
||||||
var holding []string
|
|
||||||
for d := range dirs {
|
|
||||||
file := "module.json"
|
|
||||||
if d != "." {
|
|
||||||
file = d + "/module.json"
|
|
||||||
}
|
|
||||||
_, exists, err := manifestCached(ctx, owner, repo, file, tm.Commit)
|
|
||||||
if err != nil {
|
|
||||||
return m, err
|
|
||||||
}
|
|
||||||
if exists {
|
|
||||||
holding = append(holding, d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(holding)
|
|
||||||
m.ModuleDirs, m.ModuleDirsSaid = holding, true
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mergeAskPrefix is what the id of a merge ask made by the retired module mesh-delivery begins with: the pull request's number
|
|
||||||
// and the head's first twelve digits (`merge-<number>-<head>-<try>`). The teller's ids name neither (see findMergeWarrant).
|
|
||||||
func mergeAskPrefix(number int, head string) string {
|
|
||||||
if len(head) > 12 {
|
|
||||||
head = head[:12]
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("merge-%d-%s-", number, head)
|
|
||||||
}
|
|
||||||
|
|
||||||
// askersOfMerges are the askers a merge's warrant is read under: the teller alone (ADR 0437 §6), since R4e stopped taking
|
|
||||||
// mesh-delivery's (ADR 0450 §8; issue 1318).
|
|
||||||
func askersOfMerges() []string { return []string{sdkdelivery.TellerModule} }
|
|
||||||
|
|
||||||
// caughtUpWarrant judges a chosen merge ask of the router as the warrant of a merge: the ask is the operator's, the warrant
|
|
||||||
// answers that very ask as it was published, and it is the operator's Approve proved by the channel it came through (ADR 0437
|
|
||||||
// §6, as movedOnApprove judges an announced one). The heads the Approve was bound to are checked by the caller, where it can
|
|
||||||
// name them.
|
|
||||||
func caughtUpWarrant(asker string, ask *asks.Ask, w *asks.Warrant) (string, error) {
|
|
||||||
if err := askerAccepted(asker); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case ask == nil || w == nil:
|
|
||||||
return "", errors.New("the router holds no closed record of the ask")
|
|
||||||
case ask.Who != asks.Operator:
|
|
||||||
return "", fmt.Errorf("the ask %s is for %q to answer, not the operator", ask.ID, ask.Who)
|
|
||||||
}
|
|
||||||
option, err := w.For(asker, *ask)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if option.ID != optionApprove || w.Option != optionApprove {
|
|
||||||
return "", fmt.Errorf("the operator chose %q on the ask %s, not an Approve", w.Option, w.Ask)
|
|
||||||
}
|
|
||||||
if w.By == nil || w.By.Who != asks.Operator || w.By.Verified == "" {
|
|
||||||
return "", fmt.Errorf("the ask %s was not answered by the operator on a channel that proves who answered", w.Ask)
|
|
||||||
}
|
|
||||||
return w.Says() + " (ask " + w.Ask + ")", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// routerAskIDs are the ids of the asks an asker has in the router's record. A variable so a test needs no bus.
|
|
||||||
var routerAskIDs = func(ctx context.Context, inv catalogueReader, asker string) ([]string, error) {
|
|
||||||
serving := servingBus.Load()
|
|
||||||
if serving == nil {
|
|
||||||
return nil, errors.New("only the serving controller reads the router's record")
|
|
||||||
}
|
|
||||||
bucket, err := asksRecords(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if bucket == "" {
|
|
||||||
return nil, errors.New("no module declares the operator channel's records")
|
|
||||||
}
|
|
||||||
api, err := jetstream.New(serving.Conn())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
kv, err := api.KeyValue(ctx, bucket)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return askIDsOf(ctx, kv, asker)
|
|
||||||
}
|
|
||||||
|
|
||||||
// askIDsOf are the ids of the asks an asker has in the router's bucket.
|
|
||||||
func askIDsOf(ctx context.Context, kv jetstream.KeyValue, asker string) ([]string, error) {
|
|
||||||
lister, err := kv.ListKeysFiltered(ctx, asker+".>")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var ids []string
|
|
||||||
for key := range lister.Keys() {
|
|
||||||
ids = append(ids, strings.TrimPrefix(key, asker+"."))
|
|
||||||
}
|
|
||||||
// The lister stops its own watch when it has sent the last key, so a stop of ours finds it stopped: nats answers that with
|
|
||||||
// "invalid subscription", which is no failure of the read (issue 1336). Any other error stays one.
|
|
||||||
if err := lister.Stop(); err != nil && !errors.Is(err, nats.ErrBadSubscription) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return ids, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// closedRecord is a router record of an ask that ended: it never changes, so it is read once.
|
|
||||||
type closedRecord struct {
|
|
||||||
ask *asks.Ask
|
|
||||||
w *asks.Warrant
|
|
||||||
}
|
|
||||||
|
|
||||||
var closedRecords sync.Map // asker + "." + id -> closedRecord
|
|
||||||
|
|
||||||
// closedRecordOf is the router's record of an ask, kept once it is closed. A nil ask is an ask still open, ended without a
|
|
||||||
// choice, or not there.
|
|
||||||
func closedRecordOf(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
|
||||||
if v, ok := closedRecords.Load(asker + "." + id); ok {
|
|
||||||
r := v.(closedRecord)
|
|
||||||
return r.ask, r.w, nil
|
|
||||||
}
|
|
||||||
ask, w, err := routerAskOf(ctx, inv, asker, id)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if ask != nil && w != nil {
|
|
||||||
closedRecords.Store(asker+"."+id, closedRecord{ask, w})
|
|
||||||
}
|
|
||||||
return ask, w, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryIDOf is the id the gate gives a delivery, and so the key of its head in the heads an Approve binds: the repository as
|
|
||||||
// owner/name, lower-cased, `@` and the first twelve characters of the head.
|
|
||||||
func deliveryIDOf(owner, repo, head string) string {
|
|
||||||
if len(head) > 12 {
|
|
||||||
head = head[:12]
|
|
||||||
}
|
|
||||||
return strings.ToLower(owner+"/"+repo) + "@" + head
|
|
||||||
}
|
|
||||||
|
|
||||||
// coversHead says an ask the operator approved is for this merge: its Approve is bound to the heads of the pull request alone
|
|
||||||
// (mergeAct over {delivery id: head}, which is how a pull request not in a group is asked), or, for a group's ask, whose other
|
|
||||||
// members' heads the router's record does not carry in full, the ask as published names this pull request at this head in its
|
|
||||||
// details ("<repo>#<number> ... at <head>:"), and the digest the warrant answers covers those words.
|
|
||||||
func coversHead(ask *asks.Ask, owner, repo string, number int, head string) (how string, ok bool) {
|
|
||||||
option, found := ask.Option(optionApprove)
|
|
||||||
if !found {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
heads := map[string]string{deliveryIDOf(owner, repo, head): head}
|
|
||||||
if option.Performs(mergeAct(ask.ID, optionApprove, heads)) == nil {
|
|
||||||
return "bound to the head of this pull request", true
|
|
||||||
}
|
|
||||||
short := head
|
|
||||||
if len(short) > 12 {
|
|
||||||
short = short[:12]
|
|
||||||
}
|
|
||||||
line := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(repo) + `#` + strconv.Itoa(number) + ` .* at ` + regexp.QuoteMeta(short) + `:`)
|
|
||||||
if line.MatchString(ask.Details) {
|
|
||||||
return "a group's ask naming this pull request at this head", true
|
|
||||||
}
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
// findMergeWarrant finds the warrant of a merge, with no read the controller does not make today: the router's chosen merge
|
|
||||||
// ask for it. Under the teller (releases 3e on) the id is `merge-<group>-<heads hash>-<tail>`, which names neither the pull
|
|
||||||
// request nor the head, so the teller's closed asks are read and the one the operator approved whose heads cover the merge's
|
|
||||||
// second parent is the warrant (coversHead). Under mesh-delivery, which made the earlier merges, the id names the pull request
|
|
||||||
// and the head and is found by them. Why it holds; or found false with why not, for a person to read; or an error when the
|
|
||||||
// router could not be read, which is no answer. A variable so a test needs no bus.
|
|
||||||
var findMergeWarrant = func(ctx context.Context, inv catalogueReader, owner, repo string, number int, head string) (why string, found bool, reason string, err error) {
|
|
||||||
reason = "the router holds no chosen merge ask of " + strings.Join(askersOfMerges(), " or ") + " that covers the head " + head[:min(12, len(head))]
|
|
||||||
for _, asker := range askersOfMerges() {
|
|
||||||
ids, err := routerAskIDs(ctx, inv, asker)
|
|
||||||
if err != nil {
|
|
||||||
return "", false, "", err
|
|
||||||
}
|
|
||||||
for _, id := range ids {
|
|
||||||
named := number > 0 && strings.HasPrefix(id, mergeAskPrefix(number, head))
|
|
||||||
if asker == sdkdelivery.TellerModule && !strings.HasPrefix(id, sdkdelivery.AskKindMerge+"-") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ask, w, err := closedRecordOf(ctx, inv, asker, id)
|
|
||||||
if err != nil {
|
|
||||||
return "", false, "", err
|
|
||||||
}
|
|
||||||
if ask == nil || w == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
warrant, err := caughtUpWarrant(asker, ask, w)
|
|
||||||
if err != nil {
|
|
||||||
if named {
|
|
||||||
reason = fmt.Sprintf("%s's ask %s holds no operator's Approve: %v", asker, id, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
how, covers := coversHead(ask, owner, repo, number, head)
|
|
||||||
if !covers && !named {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !covers {
|
|
||||||
how = "the ask's id names this pull request and head"
|
|
||||||
}
|
|
||||||
return warrant + ", " + how, true, "", nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", false, reason, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// caughtTrunk is one trunk the catalogue builds from.
|
|
||||||
type caughtTrunk struct{ Owner, Repo, Branch string }
|
|
||||||
|
|
||||||
func (t caughtTrunk) repository() string { return t.Owner + "/" + t.Repo }
|
|
||||||
|
|
||||||
// trunksOfCatalogue are the trunks of the repositories on the mesh's forge the catalogue builds a module from.
|
|
||||||
func trunksOfCatalogue(forms *sourceForms, entries []inventory.Entry) []caughtTrunk {
|
|
||||||
var out []caughtTrunk
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Provided || e.Source.Repository == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
owner, name, onForge := forms.onTheForge(e.Source.Repository, e.Source.Seat)
|
|
||||||
if !onForge {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
branch := followedBranch(e.Source.Ref)
|
|
||||||
if branch == "" {
|
|
||||||
branch = "main"
|
|
||||||
}
|
|
||||||
t := caughtTrunk{owner, name, branch}
|
|
||||||
if !slices.Contains(out, t) {
|
|
||||||
out = append(out, t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// catchUpBeside runs a catch-up beside the hearing of the merge that asked for it, so the bus's acknowledgement of that merge
|
|
||||||
// waits for no forge. A variable so a test runs it where it stands.
|
|
||||||
var catchUpBeside = func(run func()) { go run() }
|
|
||||||
|
|
||||||
// catchingUp makes one catch-up at a time.
|
|
||||||
var catchingUp sync.Mutex
|
|
||||||
|
|
||||||
// settledMerges are the merges this process found nothing to plan for: heard, and no record kept, because nothing reads them.
|
|
||||||
// They are not read from the forge again.
|
|
||||||
var settledMerges sync.Map
|
|
||||||
|
|
||||||
// catchUp hears the merges of the catalogue's trunks (only one, when only names it) that the controller holds no record of.
|
|
||||||
// It says whether a trunk was left for later: the forge, the router or the store did not answer.
|
|
||||||
func (f following) catchUp(ctx context.Context, only *caughtTrunk) (later bool) {
|
|
||||||
catchingUp.Lock()
|
|
||||||
defer catchingUp.Unlock()
|
|
||||||
inv := f.open.inventory
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: the catalogue cannot be read, so no merge is caught up now: %v\n", err)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
kept, first, err := inv.EarliestKept(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: what the controller kept cannot be read, so no merge is caught up now: %v\n", err)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if kept {
|
|
||||||
fmt.Printf("catch-up: the controller's record of merges begins %s; a merge made before it, less %s, is history on every trunk\n",
|
|
||||||
first.UTC().Format(time.RFC3339), mergeGrace)
|
|
||||||
} else {
|
|
||||||
fmt.Println("catch-up: the controller holds no merge of any trunk yet, so none can have been missed")
|
|
||||||
}
|
|
||||||
trunks := trunksOfCatalogue(newSourceForms(ctx, inv), entries)
|
|
||||||
if only != nil {
|
|
||||||
trunks = slices.DeleteFunc(trunks, func(t caughtTrunk) bool {
|
|
||||||
return !strings.EqualFold(t.repository(), only.repository()) || t.Branch != only.Branch
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for _, t := range trunks {
|
|
||||||
if f.catchUpTrunk(ctx, t, kept, first) {
|
|
||||||
later = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return later
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f following) catchUpTrunk(ctx context.Context, t caughtTrunk, kept bool, first time.Time) (later bool) {
|
|
||||||
inv := f.open.inventory
|
|
||||||
repository := t.repository()
|
|
||||||
merges, err := readTrunkMerges(ctx, t.Owner, t.Repo, t.Branch)
|
|
||||||
if err != nil {
|
|
||||||
// Not nothing to catch up: left for the next start or the next heard merge (ADR 0457 clause 3).
|
|
||||||
fmt.Printf("catch-up: the forge did not answer for the last merges of %s %s, so they are not caught up now; the next start "+
|
|
||||||
"or the next merge heard there does it: %v\n", repository, t.Branch, err)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// **A merge made before the controller's record of merges begins is history, never a gap** (novox/hq ADR 0463 §2).
|
|
||||||
read := len(merges)
|
|
||||||
merges = sinceTheRecord(merges, kept, first)
|
|
||||||
fmt.Printf("catch-up: %s %s: %d of the last %d pull request merges were made since the record began\n", repository, t.Branch,
|
|
||||||
len(merges), read)
|
|
||||||
unwarranted := map[string]bool{}
|
|
||||||
for _, tm := range merges {
|
|
||||||
id := fmt.Sprintf("%s_%.8s", repository, tm.Commit)
|
|
||||||
if _, done := settledMerges.Load(repository + "@" + tm.Commit); done {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err != nil {
|
|
||||||
fmt.Printf("catch-up: the record of %s cannot be read, so %s is not caught up now: %v\n", repository, tm.Commit[:8], err)
|
|
||||||
return true
|
|
||||||
} else if known {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
m, err := readMergeEvent(ctx, t.Owner, t.Repo, t.Branch, tm)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: the merge %.8s of %s cannot be read from the forge, so it is not caught up now: %v\n", tm.Commit, repository, err)
|
|
||||||
later = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
read, err := readForPlanning(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
facts, err := factsOfMerge(ctx, inv, m, time.Now().UTC(), entries, read)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: %s %.8s cannot be judged now: %v\n", repository, tm.Commit, err)
|
|
||||||
later = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// The very test hearMerge makes: a merge it would plan, or take as a first build, is a gap; any other it would say
|
|
||||||
// nothing reads, and is no gap for a person to decide.
|
|
||||||
if !facts.plans() && !facts.addsModuleToBuilds {
|
|
||||||
settledMerges.Store(repository+"@"+tm.Commit, true)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
why, found, reason, err := findMergeWarrant(ctx, inv, t.Owner, t.Repo, m.Number, tm.Head)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: the router cannot be read, so %s %.8s is not caught up now: %v\n", repository, tm.Commit, err)
|
|
||||||
later = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
unwarranted[id] = true
|
|
||||||
f.raiseMergeNotHeard(ctx, repository, t.Branch, tm, m, reason)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
now := time.Now().UTC()
|
|
||||||
said := fmt.Sprintf("caught up: merged %s, not heard, planned %s", tm.At.Format(time.RFC3339), now.Format(time.RFC3339))
|
|
||||||
fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard; %s; on the warrant %s\n", repository, t.Branch, tm.Commit, said, why)
|
|
||||||
m.Title = strings.TrimSpace(m.Title + " (" + said + ")")
|
|
||||||
if err := f.hearMerge(ctx, m, now); err != nil {
|
|
||||||
fmt.Printf("catch-up: %s %.8s could not be heard now: %v\n", repository, tm.Commit, err)
|
|
||||||
later = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err == nil && !known {
|
|
||||||
settledMerges.Store(repository+"@"+tm.Commit, true) // heard, and nothing reads it
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Cleared only when every gap was judged: one left for later may still be open.
|
|
||||||
if !later {
|
|
||||||
f.clearMergesHeard(ctx, repository, unwarranted)
|
|
||||||
}
|
|
||||||
return later
|
|
||||||
}
|
|
||||||
|
|
||||||
// raiseMergeNotHeard keeps open, and observes again, the condition of a merge with no warrant to plan it. One for each
|
|
||||||
// repository and commit, however many catch-ups find it.
|
|
||||||
func (f following) raiseMergeNotHeard(ctx context.Context, repository, branch string, tm trunkMerge, m link.SourceMoved, reason string) {
|
|
||||||
fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard, and is not planned: %s\n", repository, branch, tm.Commit, reason)
|
|
||||||
if conditionsFrom == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, tm.Commit), Kind: kindMergeNotHeard,
|
|
||||||
Severity: conditions.Warning, Source: "catch-up", Resolver: conditions.ResolverOperator,
|
|
||||||
Summary: fmt.Sprintf("the merge %.8s of %s into %s was not heard by the controller, and no operator's Approve of it is on "+
|
|
||||||
"record to plan it on: a person decides", tm.Commit, repository, branch),
|
|
||||||
Said: fmt.Sprintf("%s %.8s (pull request %d, head %.8s, merged %s): %s", repository, tm.Commit, m.Number, tm.Head,
|
|
||||||
tm.At.Format(time.RFC3339), reason),
|
|
||||||
Headline: fmt.Sprintf("A merge into %s was never delivered", repository),
|
|
||||||
Explanation: fmt.Sprintf("A merge into %s was made while the controller did not hear it, and no Approve of yours for it is on "+
|
|
||||||
"record that the controller can plan it on. It stays open while the merge is among the latest %d merges of that repository. "+
|
|
||||||
"The details say what was looked for.", repository, catchUpWindow),
|
|
||||||
Needs: "decide whether that merge should be delivered, and say so if it should.",
|
|
||||||
Resolved: "the merge is planned, or is no longer among the latest merges of its repository"}
|
|
||||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
|
||||||
fmt.Printf("catch-up: the condition of %s %.8s could not be kept: %v\n", repository, tm.Commit, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// clearMergesHeard clears the open conditions of a repository's merges not heard that this catch-up did not find again: the
|
|
||||||
// gap was planned, or left the window.
|
|
||||||
func (f following) clearMergesHeard(ctx context.Context, repository string, still map[string]bool) {
|
|
||||||
if conditionsFrom == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
open, err := conditionsFrom.Open(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: the open conditions cannot be read, so none is cleared now: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, c := range open {
|
|
||||||
if c.Kind != kindMergeNotHeard || c.Subject.Scope != conditions.ScopeMerge ||
|
|
||||||
!strings.HasPrefix(c.Subject.ID, repository+"_") || still[c.Subject.ID] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err := conditionsFrom.Clear(ctx, c.Key, "the merge is planned, or left the last merges of its trunk"); err != nil {
|
|
||||||
fmt.Printf("catch-up: the condition %s could not be cleared: %v\n", c.Key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// catchUpAtStart catches up every trunk once the controller serves, and again every minute while a trunk was left for later.
|
|
||||||
func catchUpAtStart(ctx context.Context, open *stores) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(30 * time.Second):
|
|
||||||
}
|
|
||||||
for try := 0; try < 20; try++ {
|
|
||||||
if !(following{open: open}).catchUp(ctx, nil) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(time.Minute):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Println("catch-up: a trunk is still left for later after twenty tries; the next merge heard there does it")
|
|
||||||
}
|
|
||||||
@@ -1,352 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The checks of ADR 0457 (issue 1336): what the controller does about merges it did not hear.
|
|
||||||
|
|
||||||
// aTrunk is a forge's trunk as the catch-up reads it: its merges oldest first, the module each moves, and which of them the
|
|
||||||
// router holds an operator's Approve for.
|
|
||||||
type aTrunk struct {
|
|
||||||
merges []trunkMerge
|
|
||||||
module map[string]string
|
|
||||||
approved map[string]bool
|
|
||||||
read []string // the trunks asked of the forge, as owner/repo
|
|
||||||
events int // merges read in full from the forge
|
|
||||||
silent bool // the forge does not answer
|
|
||||||
}
|
|
||||||
|
|
||||||
func aTrunkOf(t *testing.T, repo string, modules ...string) *aTrunk {
|
|
||||||
t.Helper()
|
|
||||||
tr := &aTrunk{module: map[string]string{}, approved: map[string]bool{}}
|
|
||||||
for i, m := range modules {
|
|
||||||
c := fmt.Sprintf("%s-c%02d", repo, i+1)
|
|
||||||
tr.merges = append(tr.merges, trunkMerge{Commit: c, Head: fmt.Sprintf("%shead%02d", repo, i+1),
|
|
||||||
Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from pr/%s into main", i+1, 100+i, c),
|
|
||||||
At: t0.Add(time.Duration(i) * time.Minute)})
|
|
||||||
tr.module[c] = m
|
|
||||||
tr.approved[c] = true
|
|
||||||
}
|
|
||||||
was, wasEvent, wasWarrant := trunkCommitsPage, readMergeEvent, findMergeWarrant
|
|
||||||
trunkCommitsPage = func(_ context.Context, owner, name, _ string, page int) ([]trunkCommit, error) {
|
|
||||||
tr.read = append(tr.read, owner+"/"+name)
|
|
||||||
if tr.silent {
|
|
||||||
return nil, errors.New("the forge did not answer")
|
|
||||||
}
|
|
||||||
if page > 1 || owner+"/"+name != "novox/"+repo {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
var newest []trunkCommit
|
|
||||||
for i := len(tr.merges) - 1; i >= 0; i-- {
|
|
||||||
m := tr.merges[i]
|
|
||||||
newest = append(newest, trunkCommit{Commit: m.Commit, SecondParent: m.Head, Subject: m.Subject, At: m.At})
|
|
||||||
}
|
|
||||||
return newest, nil
|
|
||||||
}
|
|
||||||
readMergeEvent = func(_ context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
|
||||||
tr.events++
|
|
||||||
m := catalogueMerge(tm.Commit, tr.module[tm.Commit], tm.At)
|
|
||||||
m.Owner, m.Repo, m.Base, m.HeadSHA = owner, name, branch, tm.Head
|
|
||||||
_, m.Number, _ = pullOfSubject(tm.Subject)
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
findMergeWarrant = func(_ context.Context, _ catalogueReader, _, _ string, number int, head string) (string, bool, string, error) {
|
|
||||||
for _, m := range tr.merges {
|
|
||||||
if strings.Contains(m.Subject, fmt.Sprintf("(#%d)", number)) && m.Head == head && tr.approved[m.Commit] {
|
|
||||||
return "the operator chose Approve (ask merge-" + fmt.Sprint(number) + ")", true, "", nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", false, "the router holds no chosen merge ask for the pull request and head", nil
|
|
||||||
}
|
|
||||||
wasBeside := catchUpBeside
|
|
||||||
catchUpBeside = func(run func()) { run() }
|
|
||||||
t.Cleanup(func() {
|
|
||||||
trunkCommitsPage, readMergeEvent, findMergeWarrant, catchUpBeside = was, wasEvent, wasWarrant, wasBeside
|
|
||||||
settledMerges.Clear()
|
|
||||||
})
|
|
||||||
return tr
|
|
||||||
}
|
|
||||||
|
|
||||||
// aKeptMergeBefore is a merge the controller heard before the merges of a test: where its record begins, so what a trunk holds from
|
|
||||||
// then on can be a gap (novox/hq ADR 0463 §2).
|
|
||||||
func aKeptMergeBefore(t *testing.T, open *stores) {
|
|
||||||
t.Helper()
|
|
||||||
hear(t, open, catalogueMerge("mesh-catalog-seed", "app", t0.Add(-time.Hour)), t0.Add(-time.Hour))
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeKnown(t *testing.T, open *stores, commit string) bool {
|
|
||||||
t.Helper()
|
|
||||||
_, known, err := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", commit)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return known
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 1: the trunk holds merges 1, 2 and 3 and the controller holds records of 1 and 3 only; at its start it plans 2, said
|
|
||||||
// as caught up; a second start plans nothing new.
|
|
||||||
func TestAGapBeforeANewerHeardMergeIsPlannedAtTheControllersStart(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
|
|
||||||
hear(t, open, catalogueMerge("mesh-catalog-c01", "app", t0), t0.Add(time.Second))
|
|
||||||
hear(t, open, catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)), t0.Add(2*time.Minute+time.Second))
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c02") {
|
|
||||||
t.Fatal("the merge to be caught up is known before the catch-up")
|
|
||||||
}
|
|
||||||
if (following{open: open}).catchUp(t.Context(), nil) {
|
|
||||||
t.Fatal("the catch-up says a trunk was left for later")
|
|
||||||
}
|
|
||||||
if !mergeKnown(t, open, "mesh-catalog-c02") {
|
|
||||||
t.Fatal("the merge the controller did not hear was not planned")
|
|
||||||
}
|
|
||||||
kept, _, _ := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", "mesh-catalog-c02")
|
|
||||||
if said := announcedOf(kept).Title; !strings.Contains(said, "caught up: merged ") || !strings.Contains(said, "not heard, planned ") {
|
|
||||||
t.Fatalf("the merge does not say it was caught up: %q", said)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(10*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) == 0 {
|
|
||||||
t.Fatal("no walk answers the caught-up merge")
|
|
||||||
}
|
|
||||||
if _, in := ws[0].Modules["notes"]; !in {
|
|
||||||
t.Fatalf("the walk does not build notes, which the caught-up merge moved: %v", ws[0].Modules)
|
|
||||||
}
|
|
||||||
before := tr.events
|
|
||||||
if (following{open: open}).catchUp(t.Context(), nil) || tr.events != before {
|
|
||||||
t.Fatalf("a second start read %d merge(s) in full again", tr.events-before)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 1: a heard merge of a repository whose trunk holds earlier merges the controller has no record of plans them all, the
|
|
||||||
// earlier first, and the forge is read for that repository's trunk only.
|
|
||||||
func TestAHeardMergeCatchesUpTheEarlierOnesOfItsOwnTrunkOnly(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
|
|
||||||
var order []string
|
|
||||||
inner := readMergeEvent
|
|
||||||
readMergeEvent = func(ctx context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
|
||||||
order = append(order, tm.Commit)
|
|
||||||
return inner(ctx, owner, name, branch, tm)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { readMergeEvent = inner })
|
|
||||||
tr.read = nil
|
|
||||||
heard := catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute))
|
|
||||||
if err := (following{open: open}).SourceMoved(t.Context(), heard); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, c := range []string{"mesh-catalog-c01", "mesh-catalog-c02", "mesh-catalog-c03"} {
|
|
||||||
if !mergeKnown(t, open, c) {
|
|
||||||
t.Fatalf("%s is not planned: the heard merge and the earlier ones of its trunk are all wanted", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(order) != 2 || order[0] != "mesh-catalog-c01" || order[1] != "mesh-catalog-c02" {
|
|
||||||
t.Fatalf("the earlier merges were heard in the order %v, not the trunk's", order)
|
|
||||||
}
|
|
||||||
if len(tr.read) == 0 {
|
|
||||||
t.Fatal("the forge was not read for the trunk of the merge heard")
|
|
||||||
}
|
|
||||||
for _, r := range tr.read {
|
|
||||||
if r != "novox/mesh-catalog" {
|
|
||||||
t.Fatalf("the forge was read for %s, not only for the trunk of the merge heard: %v", r, tr.read)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 1: a trunk of fifty-one merges whose oldest is unheard plans nothing: the oldest is outside the window.
|
|
||||||
func TestAMergeOlderThanTheLastFiftyIsNotRead(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
modules := make([]string, 51)
|
|
||||||
for i := range modules {
|
|
||||||
modules[i] = "app"
|
|
||||||
}
|
|
||||||
aTrunkOf(t, "mesh-catalog", modules...)
|
|
||||||
// Fifty are known; the oldest is not.
|
|
||||||
for i := 2; i <= 51; i++ {
|
|
||||||
c := fmt.Sprintf("mesh-catalog-c%02d", i)
|
|
||||||
hear(t, open, catalogueMerge(c, "app", t0.Add(time.Duration(i)*time.Minute)), t0.Add(time.Duration(i)*time.Minute+time.Second))
|
|
||||||
}
|
|
||||||
// Paged as the forge pages: fifty a page, and a trunk of another repository is empty (a short page, or none).
|
|
||||||
pages := trunkCommitsPage
|
|
||||||
trunkCommitsPage = func(ctx context.Context, owner, name, branch string, page int) ([]trunkCommit, error) {
|
|
||||||
all, err := pages(ctx, owner, name, branch, 1)
|
|
||||||
if err != nil || page < 1 || (page-1)*50 >= len(all) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return all[(page-1)*50 : min(page*50, len(all))], nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = pages })
|
|
||||||
(following{open: open}).catchUp(t.Context(), nil)
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("the oldest of fifty-one merges was read and planned, outside the window")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 2: a gap with no chosen Approve in the router is not planned and raises one condition naming its repository and
|
|
||||||
// commit; a second start leaves that one condition open, observed twice; planning the gap clears it.
|
|
||||||
func TestAGapWithNoApproveRaisesOneConditionKeptOpenAndClearedWhenPlanned(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
aKeptMergeBefore(t, open)
|
|
||||||
keeper, _ := withConditionsInMemory(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "notes")
|
|
||||||
tr.approved["mesh-catalog-c01"] = false
|
|
||||||
catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
|
|
||||||
catch()
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge with no Approve on record was planned")
|
|
||||||
}
|
|
||||||
raised := func() []conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
all, err := keeper.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var out []conditions.Condition
|
|
||||||
for _, c := range all {
|
|
||||||
if c.Kind == kindMergeNotHeard {
|
|
||||||
out = append(out, c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
one := raised()
|
|
||||||
if len(one) != 1 || !strings.Contains(one[0].Subject.ID, "novox/mesh-catalog") || !strings.Contains(one[0].Subject.ID, "mesh-cata") {
|
|
||||||
t.Fatalf("the gap raised %+v, not one condition naming its repository and commit", one)
|
|
||||||
}
|
|
||||||
catch()
|
|
||||||
if again := raised(); len(again) != 1 || again[0].Observations < 2 {
|
|
||||||
t.Fatalf("a second catch-up left %+v: one condition, observed twice, was wanted", again)
|
|
||||||
}
|
|
||||||
tr.approved["mesh-catalog-c01"] = true
|
|
||||||
catch()
|
|
||||||
if !mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("the gap was not planned once its Approve was on record")
|
|
||||||
}
|
|
||||||
if left := raised(); len(left) != 0 {
|
|
||||||
t.Fatalf("planning the gap left its condition open: %+v", left)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 3: a forge that does not answer leaves the catch-up undone and says so, and the next start does it.
|
|
||||||
func TestAForgeThatDoesNotAnswerLeavesTheCatchUpForTheNextStart(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
aKeptMergeBefore(t, open)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "notes")
|
|
||||||
tr.silent = true
|
|
||||||
if later := (following{open: open}).catchUp(t.Context(), nil); !later {
|
|
||||||
t.Fatal("a forge that did not answer counted as nothing to catch up")
|
|
||||||
}
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge was planned without the forge's word")
|
|
||||||
}
|
|
||||||
tr.silent = false
|
|
||||||
if later := (following{open: open}).catchUp(t.Context(), nil); later || !mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatalf("the next start did not catch up (left for later: %v)", later)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Finding 2 of the review: a gap that adds a module is a gap, judged as hearMerge judges it. With the operator's Approve on record it
|
|
||||||
// is heard (hearMerge asks the new module in its walk); without, it raises the condition and is not lost in silence.
|
|
||||||
func TestAGapThatAddsAModuleIsHeardOnItsWarrantAndRaisedWithout(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
aKeptMergeBefore(t, open)
|
|
||||||
keeper, _ := withConditionsInMemory(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "newmodule")
|
|
||||||
tr.approved["mesh-catalog-c01"] = false
|
|
||||||
catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
|
|
||||||
catch()
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge adding a module was heard with no Approve on record")
|
|
||||||
}
|
|
||||||
all, err := keeper.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var raised int
|
|
||||||
for _, c := range all {
|
|
||||||
if c.Kind == kindMergeNotHeard {
|
|
||||||
raised++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if raised != 1 {
|
|
||||||
t.Fatalf("a merge adding a module with no Approve raised %d condition(s), not one: it was lost in silence", raised)
|
|
||||||
}
|
|
||||||
tr.approved["mesh-catalog-c01"] = true
|
|
||||||
catch()
|
|
||||||
if !mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge adding a module was not heard once its Approve was on record")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A trunk with merges from before the controller began keeping merges, and merges since: only the later ones are gaps.
|
|
||||||
func TestAMergeFromBeforeTheRecordBeganIsHistoryAndNoCondition(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
keeper, _ := withConditionsInMemory(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "notes", "notes", "notes")
|
|
||||||
tr.approved["mesh-catalog-c01"], tr.approved["mesh-catalog-c02"], tr.approved["mesh-catalog-c03"] = false, false, false
|
|
||||||
// The record begins at the second merge: the first is two days older.
|
|
||||||
tr.merges[0].At = t0.Add(-48 * time.Hour)
|
|
||||||
hear(t, open, catalogueMerge("mesh-catalog-c02", "notes", t0.Add(time.Minute)), t0.Add(time.Minute))
|
|
||||||
delete(tr.approved, "mesh-catalog-c02")
|
|
||||||
(following{open: open}).catchUp(t.Context(), nil)
|
|
||||||
// The first merge (two days before the record) was not even read from the forge: only the third, which is unknown, was.
|
|
||||||
if tr.events != 1 {
|
|
||||||
t.Fatalf("%d merge(s) were read from the forge: the one from before the record is history and is not read", tr.events)
|
|
||||||
}
|
|
||||||
all, err := keeper.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var raised int
|
|
||||||
for _, c := range all {
|
|
||||||
if c.Kind == kindMergeNotHeard {
|
|
||||||
raised++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if raised != 1 {
|
|
||||||
t.Fatalf("%d condition(s) raised: only the merge since the record began (the third) is a gap", raised)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq ADR 0463 §2: the record begins at the earliest merge kept on ANY trunk. A trunk the controller kept no merge of has the
|
|
||||||
// merges made since as gaps (mesh-delivery #1 on novox), and the merges made before as history.
|
|
||||||
func TestATrunkWithNoKeptMergeHasItsLaterMergesAsGapsAndItsEarlierOnesAsHistory(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
// The record begins on another trunk, at t0 less an hour.
|
|
||||||
hear(t, open, repoMerge("one", "c-one", t0.Add(-time.Hour)), t0.Add(-time.Hour))
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "notes", "notes")
|
|
||||||
tr.merges[0].At = t0.Add(-72 * time.Hour) // before the record: history
|
|
||||||
(following{open: open}).catchUp(t.Context(), nil)
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge from before the record began was heard")
|
|
||||||
}
|
|
||||||
if !mergeKnown(t, open, "mesh-catalog-c02") {
|
|
||||||
t.Fatal("a merge since the record began, on a trunk with no merge kept, was not caught up")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,318 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The pieces of the catch-up that need no store (novox/hq ADR 0457, issue 1336).
|
|
||||||
|
|
||||||
func TestAMergeCommitsSubjectNamesItsPullRequest(t *testing.T) {
|
|
||||||
title, number, ok := pullOfSubject("Merge pull request 'Fix keycloak's sign-in (issue 1332)' (#378) from fix/1332-sign-in into main")
|
|
||||||
if !ok || number != 378 || title != "Fix keycloak's sign-in (issue 1332)" {
|
|
||||||
t.Fatalf("read %q %d %v", title, number, ok)
|
|
||||||
}
|
|
||||||
if b := headBranchOfSubject("Merge pull request 'x' (#378) from fix/1332-sign-in into main"); b != "fix/1332-sign-in" {
|
|
||||||
t.Fatalf("the head branch read as %q", b)
|
|
||||||
}
|
|
||||||
if _, _, ok := pullOfSubject("Merge branch 'main' of somewhere"); ok {
|
|
||||||
t.Fatal("a merge that names no pull request read as one")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOnlyTheLastFiftyMergesOfATrunkAreRead(t *testing.T) {
|
|
||||||
pages := map[int][]trunkCommit{}
|
|
||||||
var all []trunkCommit // newest first: 51 merges, a plain commit between each pair
|
|
||||||
for i := 51; i >= 1; i-- {
|
|
||||||
all = append(all, trunkCommit{Commit: fmt.Sprintf("merge-%02d", i), SecondParent: fmt.Sprintf("head-%02d", i),
|
|
||||||
Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from fix/x into main", i, i)},
|
|
||||||
trunkCommit{Commit: fmt.Sprintf("plain-%02d", i)})
|
|
||||||
}
|
|
||||||
for i := 0; i < len(all); i += 50 {
|
|
||||||
pages[i/50+1] = all[i:min(i+50, len(all))]
|
|
||||||
}
|
|
||||||
was := trunkCommitsPage
|
|
||||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { return pages[page], nil }
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = was })
|
|
||||||
got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got) != catchUpWindow || got[0].Commit != "merge-02" || got[len(got)-1].Commit != "merge-51" {
|
|
||||||
t.Fatalf("read %d merges from %s to %s: the oldest of 51 is outside the window", len(got), got[0].Commit, got[len(got)-1].Commit)
|
|
||||||
}
|
|
||||||
if got[0].Head != "head-02" {
|
|
||||||
t.Fatalf("a merge's head is its second parent, not %q", got[0].Head)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) {
|
|
||||||
was := trunkCommitsPage
|
|
||||||
trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) {
|
|
||||||
return nil, errors.New("the forge did not answer")
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = was })
|
|
||||||
if got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main"); err == nil {
|
|
||||||
t.Fatalf("a forge that did not answer read as %d merges: nothing to catch up", len(got))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) {
|
|
||||||
ok := approvedMerge(t)
|
|
||||||
if _, err := caughtUpWarrant(sdkdelivery.TellerModule, &ok.ask, &ok.warrant); err != nil {
|
|
||||||
t.Errorf("the operator's Approve under the teller was refused: %v", err)
|
|
||||||
}
|
|
||||||
// 1318 (ADR 0450 §8): mesh-delivery's warrant of before the gate is no longer taken.
|
|
||||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &ok.warrant); err == nil {
|
|
||||||
t.Error("a warrant under mesh-delivery was accepted")
|
|
||||||
}
|
|
||||||
if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil {
|
|
||||||
t.Error("a warrant of an asker the controller takes none from was accepted")
|
|
||||||
}
|
|
||||||
declined := ok.warrant
|
|
||||||
declined.Option, declined.Label = "decline", "Decline"
|
|
||||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &declined); err == nil {
|
|
||||||
t.Error("a Decline was accepted as a warrant")
|
|
||||||
}
|
|
||||||
unproved := ok.warrant
|
|
||||||
by := *unproved.By
|
|
||||||
by.Verified = ""
|
|
||||||
unproved.By = &by
|
|
||||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &unproved); err == nil {
|
|
||||||
t.Error("an answer whose channel proves nobody was accepted")
|
|
||||||
}
|
|
||||||
if _, err := caughtUpWarrant(theDelivery, nil, nil); err == nil {
|
|
||||||
t.Error("a router that holds nothing was accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// aTellersAsk is the ask the delivery gate composes for a pull request, as the router holds it once the operator chose Approve
|
|
||||||
// (the gate's composeMergeAsk: its id from sdkdelivery.AskingID of the group and the heads, its Approve bound to mergeAct over the
|
|
||||||
// heads, the pull request and its head's first twelve digits in the details), with the warrant the router wrote.
|
|
||||||
func aTellersAsk(t *testing.T, group string, members ...sdkdelivery.Head) (asks.Ask, asks.Warrant) {
|
|
||||||
t.Helper()
|
|
||||||
key, err := sdkdelivery.MergeAskKey(group, members)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
id := sdkdelivery.AskingID(key, 1)
|
|
||||||
heads := map[string]string{}
|
|
||||||
var details []string
|
|
||||||
for _, m := range members {
|
|
||||||
owner, repo, _ := strings.Cut(m.Repository, "/")
|
|
||||||
heads[deliveryIDOf(owner, repo, m.Head)] = m.Head
|
|
||||||
details = append(details, fmt.Sprintf("%s#%d %q at %s: issues: none named; review: approved; reaches: no delivery plan yet", repo, m.Number, "title", m.Head[:12]))
|
|
||||||
}
|
|
||||||
bind := func(option string) string {
|
|
||||||
d, err := asks.ActDigest(mergeAct(id, option, heads))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
at := time.Date(2026, 10, 20, 12, 2, 1, 0, time.UTC)
|
|
||||||
ask := asks.Ask{ID: id, Headline: "Merge into main", Explanation: "passed its checks.", Who: asks.Operator, Expires: at.Add(24 * time.Hour),
|
|
||||||
OnExpiry: "nothing is merged; the ask is made again", Details: strings.Join(details, "\n"),
|
|
||||||
Options: []asks.Option{
|
|
||||||
{ID: optionApprove, Label: "Approve", Does: "merges", Level: asks.Approve, Binds: bind(optionApprove)},
|
|
||||||
{ID: "decline", Label: "Decline", Does: "nothing is merged", Level: asks.Approve, Binds: bind("decline")}}}
|
|
||||||
w := asks.Warrant{Ask: id, Asker: sdkdelivery.TellerModule, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve",
|
|
||||||
Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"},
|
|
||||||
At: at.Add(time.Minute), AskDigest: ask.Digest()}
|
|
||||||
return ask, w
|
|
||||||
}
|
|
||||||
|
|
||||||
// withRouter hands findMergeWarrant these asks as the router's records, by asker.
|
|
||||||
func withRouter(t *testing.T, records map[string]map[string]struct {
|
|
||||||
ask asks.Ask
|
|
||||||
w asks.Warrant
|
|
||||||
}) {
|
|
||||||
t.Helper()
|
|
||||||
wasIDs, wasAsk := routerAskIDs, routerAskOf
|
|
||||||
closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true })
|
|
||||||
routerAskIDs = func(_ context.Context, _ catalogueReader, asker string) ([]string, error) {
|
|
||||||
var ids []string
|
|
||||||
for id := range records[asker] {
|
|
||||||
ids = append(ids, id)
|
|
||||||
}
|
|
||||||
return ids, nil
|
|
||||||
}
|
|
||||||
routerAskOf = func(_ context.Context, _ catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
|
||||||
r, ok := records[asker][id]
|
|
||||||
if !ok {
|
|
||||||
return nil, nil, nil
|
|
||||||
}
|
|
||||||
return &r.ask, &r.w, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() {
|
|
||||||
routerAskIDs, routerAskOf = wasIDs, wasAsk
|
|
||||||
closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
type aRouterRecord = struct {
|
|
||||||
ask asks.Ask
|
|
||||||
w asks.Warrant
|
|
||||||
}
|
|
||||||
|
|
||||||
// The teller's asks, as the router held them on 2026-10-20: the id is the router's own key, rebuilt here from the group and the
|
|
||||||
// head the gate composed it for (mesh-controller#405 at 8a330afd, mesh-catalog#381 at 1c702422, mesh-host#124 at 52600ec7). It names
|
|
||||||
// neither the pull request nor the head, so the warrant is found by the heads the Approve is bound to.
|
|
||||||
func TestATellersWarrantIsFoundByTheHeadItsApproveIsBoundTo(t *testing.T) {
|
|
||||||
real := []struct {
|
|
||||||
group, key, owner, repo string
|
|
||||||
number int
|
|
||||||
head string
|
|
||||||
}{
|
|
||||||
{"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-ba59b03220f8-989afe0a3bcf", "novox", "mesh-controller", 405, "8a330afd44ba852873339cf1210e90989dca46e1"},
|
|
||||||
{"fix/1351-remove-todays-delivery-module", "merge-fix_1351_remove_todays_delivery_module-64a353-d93b97b9697b", "novox", "mesh-catalog", 381, "1c7024224dd77c5b603299378883e0993b2e2fdd"},
|
|
||||||
{"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-6a59ba48da14-424e053ebc9c", "novox", "mesh-host", 124, "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"},
|
|
||||||
}
|
|
||||||
teller := map[string]aRouterRecord{}
|
|
||||||
for _, r := range real {
|
|
||||||
ask, w := aTellersAsk(t, r.group, sdkdelivery.Head{Repository: r.owner + "/" + r.repo, Number: r.number, Head: r.head})
|
|
||||||
if ask.ID != r.key {
|
|
||||||
t.Fatalf("the id of %s#%d is %q, and the router held it as %q: the fixture is not the router's", r.repo, r.number, ask.ID, r.key)
|
|
||||||
}
|
|
||||||
teller[ask.ID] = aRouterRecord{ask, w}
|
|
||||||
}
|
|
||||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: teller})
|
|
||||||
for _, r := range real {
|
|
||||||
why, found, reason, err := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, r.head)
|
|
||||||
if err != nil || !found || !strings.Contains(why, r.key) || !strings.Contains(why, "bound to the head of this pull request") {
|
|
||||||
t.Errorf("%s#%d: the warrant was not found: %q %v %q %v", r.repo, r.number, why, found, reason, err)
|
|
||||||
}
|
|
||||||
// Another head of the same pull request is another merge: the Approve was not given for it.
|
|
||||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, "ffffffffffffffffffffffffffffffffffffffff"); found {
|
|
||||||
t.Errorf("%s#%d: another head found the warrant of this one", r.repo, r.number)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// One pull request's Approve is none for another repository's merge at the same head digits.
|
|
||||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-tools", 405, real[0].head); found {
|
|
||||||
t.Error("a merge of another repository found mesh-controller#405's warrant")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A group's ask binds several heads, and the router's record carries only twelve digits of the other members': the pull request is
|
|
||||||
// found in the ask as it was published, which the warrant's digest covers.
|
|
||||||
func TestAGroupsWarrantIsFoundByTheAsksWordsForThePullRequestAndItsHead(t *testing.T) {
|
|
||||||
a := sdkdelivery.Head{Repository: "novox/mesh-host", Number: 124, Head: "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"}
|
|
||||||
b := sdkdelivery.Head{Repository: "novox/mesh-controller", Number: 405, Head: "8a330afd44ba852873339cf1210e90989dca46e1"}
|
|
||||||
ask, w := aTellersAsk(t, "feat/r4b-sdk-users", a, b)
|
|
||||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, w}}})
|
|
||||||
for _, h := range []sdkdelivery.Head{a, b} {
|
|
||||||
owner, repo, _ := strings.Cut(h.Repository, "/")
|
|
||||||
why, found, reason, err := findMergeWarrant(t.Context(), nil, owner, repo, h.Number, h.Head)
|
|
||||||
if err != nil || !found || !strings.Contains(why, "a group's ask naming this pull request at this head") {
|
|
||||||
t.Errorf("%s#%d: not found in the group's ask: %q %v %q %v", repo, h.Number, why, found, reason, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 381, a.Head); found {
|
|
||||||
t.Error("a pull request the group's ask does not name found its warrant")
|
|
||||||
}
|
|
||||||
// A Decline is no warrant.
|
|
||||||
declined := w
|
|
||||||
declined.Option, declined.Label = "decline", "Decline"
|
|
||||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, declined}}})
|
|
||||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-host", 124, a.Head); found {
|
|
||||||
t.Error("a Decline found a warrant")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A trunk shorter than a page, or with none, is read without a fault: the pages end when the forge's do.
|
|
||||||
func TestAShortTrunkIsReadWholeAndAnEmptyOneIsNoMerges(t *testing.T) {
|
|
||||||
was := trunkCommitsPage
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = was })
|
|
||||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) {
|
|
||||||
if page > 1 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return []trunkCommit{{Commit: "m2", SecondParent: "h2", Subject: "Merge pull request 'two' (#2) from fix/b into main"},
|
|
||||||
{Commit: "p"}, {Commit: "m1", SecondParent: "h1", Subject: "Merge pull request 'one' (#1) from fix/a into main"}}, nil
|
|
||||||
}
|
|
||||||
got, err := readTrunkMerges(t.Context(), "novox", "x", "main")
|
|
||||||
if err != nil || len(got) != 2 || got[0].Commit != "m1" || got[1].Commit != "m2" {
|
|
||||||
t.Fatalf("a short trunk read as %+v %v", got, err)
|
|
||||||
}
|
|
||||||
trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { return nil, nil }
|
|
||||||
if got, err := readTrunkMerges(t.Context(), "novox", "x", "main"); err != nil || len(got) != 0 {
|
|
||||||
t.Fatalf("an empty trunk read as %+v %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ADR 0253: what the operator reads of a merge not heard is plain: no hash, no identifier, nothing to do "by hand".
|
|
||||||
func TestTheConditionOfAMergeNotHeardIsInPlainWords(t *testing.T) {
|
|
||||||
keeper, _ := withConditionsInMemory(t)
|
|
||||||
tm := trunkMerge{Commit: "c0880c4d1f2e3a4b5c6d7e8f9a0b1c2d3e4f5a6b", Head: "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372"}
|
|
||||||
(following{}).raiseMergeNotHeard(t.Context(), "novox/mesh-catalog", "main", tm, link.SourceMoved{Number: 378},
|
|
||||||
"the router holds no chosen merge ask of mesh-delivery-teller or mesh-delivery that covers the head b9104454f9a5")
|
|
||||||
open, err := keeper.Open(t.Context())
|
|
||||||
if err != nil || len(open) != 1 {
|
|
||||||
t.Fatalf("opened %d conditions, %v", len(open), err)
|
|
||||||
}
|
|
||||||
c := open[0]
|
|
||||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs,
|
|
||||||
Resolved: c.Resolved}); !ok {
|
|
||||||
t.Fatalf("the condition's words are not plain: %s\n%q\n%q\n%q", why, c.Headline, c.Explanation, c.Needs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The trunk's history lists the merges made inside a pull request's branch too. On novox (2026-10-20) 15 of the 15 non-pull-request
|
|
||||||
// "gaps" of mesh-catalog and mesh-controller were such merges; only a pull request's merge into this trunk is a merge to catch up.
|
|
||||||
func TestAMergeMadeInsideABranchIsNoMergeOfTheTrunk(t *testing.T) {
|
|
||||||
was := trunkCommitsPage
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = was })
|
|
||||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) {
|
|
||||||
if page > 1 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return []trunkCommit{
|
|
||||||
{Commit: "c0880c4d", SecondParent: "b9104454", Subject: "Merge pull request 'Read the grant file' (#378) from fix/1332 into main"},
|
|
||||||
{Commit: "6394a62a", SecondParent: "x1", Subject: "Merge remote-tracking branch 'origin/main' into fix/1081-withdraw"},
|
|
||||||
{Commit: "896f1d28", SecondParent: "x2", Subject: "Merge feat/agents-write-git-ssh into feat/agents-write-git-ssh-f (issue 955 F)"},
|
|
||||||
{Commit: "4d03755c", SecondParent: "x3", Subject: "Bring the branch up to main after #354 (issue 1047)"},
|
|
||||||
{Commit: "other", SecondParent: "x4", Subject: "Merge pull request 'Elsewhere' (#5) from fix/y into nox-mesh"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main")
|
|
||||||
if err != nil || len(got) != 1 || got[0].Commit != "c0880c4d" {
|
|
||||||
t.Fatalf("read %+v %v: only keycloak's pull request merge into main is a merge to catch up", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq ADR 0463 §2: the record of merges begins at the earliest merge the controller holds on any trunk. A merge before it is
|
|
||||||
// history on every trunk; a trunk with no merge kept of its own still has the merges made since as gaps (mesh-delivery #1 on novox).
|
|
||||||
func TestOnlyTheMergesSinceTheRecordBeganAreGapsOnEveryTrunk(t *testing.T) {
|
|
||||||
first := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) // the first merge it heard, on some trunk of the mesh
|
|
||||||
trunk := func(prefix string) []trunkMerge {
|
|
||||||
return []trunkMerge{
|
|
||||||
{Commit: prefix + "-old-1", At: first.Add(-30 * 24 * time.Hour)},
|
|
||||||
{Commit: prefix + "-old-2", At: first.Add(-2 * time.Hour)},
|
|
||||||
{Commit: prefix + "-within-grace", At: first.Add(-mergeGrace / 2)},
|
|
||||||
{Commit: prefix + "-since", At: first.Add(24 * time.Hour)},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Two trunks, one of which the controller has kept no merge of: both read by the one starting point.
|
|
||||||
for _, prefix := range []string{"kept-trunk", "trunk-with-no-kept-merge"} {
|
|
||||||
var commits []string
|
|
||||||
for _, m := range sinceTheRecord(trunk(prefix), true, first) {
|
|
||||||
commits = append(commits, m.Commit)
|
|
||||||
}
|
|
||||||
if want := prefix + "-within-grace " + prefix + "-since"; strings.Join(commits, " ") != want {
|
|
||||||
t.Errorf("%s: the gaps are %v, want %q: earlier ones are history on every trunk, later ones are gaps", prefix, commits, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if none := sinceTheRecord(trunk("x"), false, time.Time{}); len(none) != 0 {
|
|
||||||
t.Errorf("with no merge kept anywhere there are %d gap(s), but no record to have one in", len(none))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,135 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/chain"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
|
|
||||||
// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them.
|
|
||||||
|
|
||||||
// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
|
|
||||||
func chainHolds(ctx context.Context) error {
|
|
||||||
_, err := theLease.epoch(ctx)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// openTheChain opens the chain's buckets on the bus: the serving controller's connection, or one dialled for a command.
|
|
||||||
func openTheChain(ctx context.Context) (*chain.Chain, error) {
|
|
||||||
js, err := aBus()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Not closed: a borrowed connection closes nothing, and a dialled one lasts as long as the command.
|
|
||||||
return chain.OnTheBus(ctx, js.Conn(), chainHolds)
|
|
||||||
}
|
|
||||||
|
|
||||||
// attachTheChain gives the inventory of the serving controller the chain on its connection.
|
|
||||||
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
|
|
||||||
bus, ok := server.Bus().(link.OverNATS)
|
|
||||||
if !ok {
|
|
||||||
return errors.New("the chain lives on the bus, and this control plane is not on it")
|
|
||||||
}
|
|
||||||
c, err := chain.OnTheBus(ctx, bus.Conn, chainHolds)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
inv.UseChain(c)
|
|
||||||
inv.RequireLayout()
|
|
||||||
// And the builds laid out under their modules, once, before they are read (issue 1121) — **before this returns**, so
|
|
||||||
// nothing the start goes on to read meets a chain not laid out yet: on 2026-10-18 the layout ran beside the start, a
|
|
||||||
// read of the start was refused, and the controller exited and was started again every five seconds. The layout of
|
|
||||||
// keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly.
|
|
||||||
if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil {
|
|
||||||
fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
|
|
||||||
go layBuildsOutSoon(ctx, inv)
|
|
||||||
}
|
|
||||||
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
|
||||||
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
|
||||||
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
|
||||||
go copySentBuildsSoon(ctx, inv)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not
|
|
||||||
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
|
||||||
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
|
||||||
|
|
||||||
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
|
|
||||||
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
|
|
||||||
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
|
|
||||||
// that names a commit and empties each column it has copied.
|
|
||||||
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
|
|
||||||
failed := false
|
|
||||||
for {
|
|
||||||
_, err := inv.CopySentBuilds(ctx)
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
|
|
||||||
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
|
||||||
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
|
|
||||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
|
||||||
o.Summary += err.Error()
|
|
||||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
|
||||||
failed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if keeper := conditionsFrom; keeper != nil && failed {
|
|
||||||
_, _ = keeper.Clear(ctx, o.Key(), "copied")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(chainMoveRetry):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// chainMoveRetry is how long the copy waits before it tries again.
|
|
||||||
const chainMoveRetry = 30 * time.Second
|
|
||||||
|
|
||||||
// kindBuildsNotLaidOut is the condition of the layout of the builds under their modules that has not succeeded: until it
|
|
||||||
// has, no build is read, and a send that needs one is refused with the reason.
|
|
||||||
const kindBuildsNotLaidOut = "builds-not-laid-out"
|
|
||||||
|
|
||||||
// layBuildsOutSoon lays the builds out under their modules as soon as this controller holds the lease (the chain refuses a
|
|
||||||
// write before), and tries again every half minute while it fails, loudly: each failure is printed, and the first raises an
|
|
||||||
// urgent condition until it has worked. It is idempotent (LayBuildsOut).
|
|
||||||
func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
|
|
||||||
failed := false
|
|
||||||
for {
|
|
||||||
err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) })
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindBuildsNotLaidOut, Token: kindBuildsNotLaidOut,
|
|
||||||
Kind: kindBuildsNotLaidOut, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
|
||||||
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
|
|
||||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
|
||||||
o.Summary += err.Error()
|
|
||||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
|
||||||
failed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if keeper := conditionsFrom; keeper != nil && failed {
|
|
||||||
_, _ = keeper.Clear(ctx, o.Key(), "laid out")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(chainMoveRetry):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,232 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/chain"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/lease"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there,
|
|
||||||
// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live
|
|
||||||
// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's
|
|
||||||
// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send
|
|
||||||
// is recorded as the controller records one it made, and the gate's facts are the store's.
|
|
||||||
func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var sent [][]string
|
|
||||||
wasSend := sendRollout
|
|
||||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
||||||
sent = append(sent, append([]string(nil), names...))
|
|
||||||
current, err := open.inventory.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, node := range names {
|
|
||||||
digest := fmt.Sprintf("d-%s-%d", node, len(sent))
|
|
||||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest,
|
|
||||||
map[string]string{"notes": current["notes"].Commit}); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { sendRollout = wasSend })
|
|
||||||
wasGather := gatherGateFacts
|
|
||||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
|
||||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
|
||||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
|
||||||
reports, err := open.inventory.LastReports(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return f, err
|
|
||||||
}
|
|
||||||
for _, r := range reports {
|
|
||||||
f.reports[r.Node] = r
|
|
||||||
}
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
|
||||||
|
|
||||||
// A merge of notes is heard, and its window closes: the batch is cut into the walk.
|
|
||||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
|
||||||
cutAt(t, open, t0.Add(time.Second+90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
|
||||||
}
|
|
||||||
rollout := ws[0].ID
|
|
||||||
|
|
||||||
// The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the
|
|
||||||
// outcome in as it does.
|
|
||||||
asking, err := inv.PlanByID(ctx, rollout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ {
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
buildID := asking.Modules["notes"].Build
|
|
||||||
if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) {
|
|
||||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The walk sends the tier to its first machine.
|
|
||||||
for try := 0; try < 3 && len(sent) == 0; try++ {
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
}
|
|
||||||
if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) {
|
|
||||||
t.Fatalf("sent %v", sent)
|
|
||||||
}
|
|
||||||
|
|
||||||
// One read of the assignment.
|
|
||||||
on, err := inv.AssignmentsOn(ctx, "anchor")
|
|
||||||
a, ok := assignmentOf(on, "notes")
|
|
||||||
if err != nil || !ok {
|
|
||||||
t.Fatalf("the assignments of anchor: %+v %v", on, err)
|
|
||||||
}
|
|
||||||
// Beside it a module pulled in by the assignment, which no send of this walk put anywhere.
|
|
||||||
if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") {
|
|
||||||
t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard)
|
|
||||||
}
|
|
||||||
if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout ||
|
|
||||||
a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() {
|
|
||||||
t.Fatalf("the assignment names %+v, not commit c-notes, build %s, rollout %s", a, buildID, rollout)
|
|
||||||
}
|
|
||||||
// One read of the rollout.
|
|
||||||
p, err := inv.PlanByID(ctx, rollout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}}
|
|
||||||
if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 ||
|
|
||||||
p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil {
|
|
||||||
t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange)
|
|
||||||
}
|
|
||||||
// And one read of the build names the same commit.
|
|
||||||
built, found, err := inv.BuildByID(ctx, buildID)
|
|
||||||
if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" {
|
|
||||||
t.Fatalf("the build: %+v %v %v", built, found, err)
|
|
||||||
}
|
|
||||||
// The verbs say the links.
|
|
||||||
if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") ||
|
|
||||||
!strings.Contains(lines[1], buildID) {
|
|
||||||
t.Fatalf("`plans` says %v", lines)
|
|
||||||
}
|
|
||||||
if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) {
|
|
||||||
t.Fatalf("`node show` says %v", lines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks
|
|
||||||
// nothing (issue 1095).
|
|
||||||
func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
|
||||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
node := nodeID(t, open, "anchor")
|
|
||||||
inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}})
|
|
||||||
if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Sent again by hand, with no rollout: the assignment still names the rollout that put the build there.
|
|
||||||
if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
on, _ := inv.AssignmentsOn(ctx, "anchor")
|
|
||||||
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
|
|
||||||
t.Fatalf("the assignment names %+v", on)
|
|
||||||
}
|
|
||||||
// A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit.
|
|
||||||
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() {
|
|
||||||
t.Fatalf("a send of unknown builds left %+v", on)
|
|
||||||
}
|
|
||||||
// A module sent a commit no build is kept of is named by its commit alone.
|
|
||||||
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" ||
|
|
||||||
runOf(on, "notes").Rollout != "" {
|
|
||||||
t.Fatalf("the assignment after a push of c2 by hand names %+v", on)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it.
|
|
||||||
func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) {
|
|
||||||
for _, a := range on {
|
|
||||||
if a.Module == module {
|
|
||||||
return a, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return chain.Assignment{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func runOf(on []chain.Assignment, module string) chain.Assignment {
|
|
||||||
a, _ := assignmentOf(on, module)
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
|
|
||||||
// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered
|
|
||||||
// through the `assign` verb for a module whose build is in flight, one read of `<node>.<module>` says state pending with the
|
|
||||||
// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied,
|
|
||||||
// and the table of pending assignments is read by nobody. No key is written by hand.
|
|
||||||
func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) {
|
|
||||||
open := aCatalogueMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
asksWithPaths(t)
|
|
||||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
|
||||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
|
||||||
t.Fatalf("an assignment while its build runs was refused: %v", err)
|
|
||||||
}
|
|
||||||
waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
|
||||||
if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() ||
|
|
||||||
waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" ||
|
|
||||||
!strings.HasPrefix(waiting.Pending.ID, "p-") {
|
|
||||||
t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err)
|
|
||||||
}
|
|
||||||
if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") {
|
|
||||||
t.Fatalf("a module only waiting is assigned: %v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
|
||||||
if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil ||
|
|
||||||
made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID {
|
|
||||||
t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err)
|
|
||||||
}
|
|
||||||
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
|
||||||
t.Fatal("the build registered sensors and it was not assigned")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,121 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/chain"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a
|
|
||||||
// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of
|
|
||||||
// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the
|
|
||||||
// build before and after.
|
|
||||||
|
|
||||||
// rolloutModules is what a plan sends of each module: the commit and the build of it.
|
|
||||||
func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule {
|
|
||||||
out := make(map[string]inventory.RolloutModule, len(p.Modules))
|
|
||||||
for m, s := range p.Modules {
|
|
||||||
if s != nil && s.Commit != "" {
|
|
||||||
out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout.
|
|
||||||
func sendingAs(ctx context.Context, p *inventory.Plan) context.Context {
|
|
||||||
return inventory.WithRollout(ctx, p.ID, rolloutModules(p))
|
|
||||||
}
|
|
||||||
|
|
||||||
// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit,
|
|
||||||
// else the commit the machine was last sent for it. Read before the send, which writes the new ones.
|
|
||||||
func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref {
|
|
||||||
was := map[string]chain.Ref{}
|
|
||||||
if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known {
|
|
||||||
for m, commit := range sent {
|
|
||||||
was[m] = chain.Ref{Commit: commit}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ran, err := inv.AssignmentRuns(ctx, node); err == nil {
|
|
||||||
for m, r := range ran {
|
|
||||||
was[m] = r
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return was
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and
|
|
||||||
// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the
|
|
||||||
// module is replaced, so a send made again says it once.
|
|
||||||
func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) {
|
|
||||||
for _, m := range modules {
|
|
||||||
s := p.Modules[m]
|
|
||||||
if s == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}}
|
|
||||||
if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 {
|
|
||||||
change.Before = p.Changed[at].Before
|
|
||||||
p.Changed[at] = change
|
|
||||||
} else {
|
|
||||||
p.Changed = append(p.Changed, change)
|
|
||||||
}
|
|
||||||
if s.Build != "" && !slices.Contains(p.Builds, s.Build) {
|
|
||||||
p.Builds = append(p.Builds, s.Build)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) {
|
|
||||||
sent := at.UTC()
|
|
||||||
p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none.
|
|
||||||
func refSaid(r chain.Ref) string {
|
|
||||||
switch {
|
|
||||||
case r.Build == "" && r.Commit == "":
|
|
||||||
return "nothing"
|
|
||||||
case r.Build == "":
|
|
||||||
return "commit " + short(r.Commit)
|
|
||||||
}
|
|
||||||
return r.Build + " (" + short(r.Commit) + ")"
|
|
||||||
}
|
|
||||||
|
|
||||||
// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after.
|
|
||||||
func changedLines(p inventory.Plan) []string {
|
|
||||||
if len(p.Changed) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := []string{" changed:"}
|
|
||||||
for _, c := range p.Changed {
|
|
||||||
out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the
|
|
||||||
// rollout that put it there and since when.
|
|
||||||
func runLines(assigned []chain.Assignment) []string {
|
|
||||||
var out []string
|
|
||||||
for _, a := range assigned {
|
|
||||||
if a.Build == "" && a.Commit == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit}))
|
|
||||||
if a.Rollout != "" {
|
|
||||||
line += ", sent by " + a.Rollout
|
|
||||||
}
|
|
||||||
if !a.Since.IsZero() {
|
|
||||||
line += " at " + a.Since.Local().Format("2006-01-02 15:04")
|
|
||||||
}
|
|
||||||
out = append(out, line)
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return append([]string{" what they run, as the chain names it:"}, out...)
|
|
||||||
}
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/chain"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one
|
|
||||||
// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the
|
|
||||||
// rollout carried once each, and one step for a machine and a tier.
|
|
||||||
func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
|
||||||
p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{
|
|
||||||
"notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}}
|
|
||||||
was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}}
|
|
||||||
|
|
||||||
noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at)
|
|
||||||
noteChanges(&p, "laptop", []string{"notes"}, nil, at)
|
|
||||||
// The send noted again, after what was sent had become what the machine runs: the first before stands.
|
|
||||||
noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute))
|
|
||||||
|
|
||||||
want := []chain.Change{
|
|
||||||
{Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}},
|
|
||||||
{Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}},
|
|
||||||
{Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}},
|
|
||||||
}
|
|
||||||
if len(p.Changed) != len(want) {
|
|
||||||
t.Fatalf("changed %+v", p.Changed)
|
|
||||||
}
|
|
||||||
for k := range want {
|
|
||||||
if p.Changed[k] != want[k] {
|
|
||||||
t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" {
|
|
||||||
t.Errorf("builds %v", p.Builds)
|
|
||||||
}
|
|
||||||
if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" {
|
|
||||||
t.Errorf("steps %+v", p.Steps)
|
|
||||||
}
|
|
||||||
lines := changedLines(p)
|
|
||||||
if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") ||
|
|
||||||
!strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") {
|
|
||||||
t.Errorf("changed lines %q", lines)
|
|
||||||
}
|
|
||||||
if got := changedLines(inventory.Plan{}); got != nil {
|
|
||||||
t.Errorf("a plan that changed nothing says %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
|
||||||
lines := runLines([]chain.Assignment{
|
|
||||||
{Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at},
|
|
||||||
{Node: "anchor", Module: "web", Commit: "c3"},
|
|
||||||
{Node: "anchor", Module: "never-sent"},
|
|
||||||
})
|
|
||||||
if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") ||
|
|
||||||
!strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") {
|
|
||||||
t.Fatalf("lines %q", lines)
|
|
||||||
}
|
|
||||||
if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil {
|
|
||||||
t.Fatal("a module never sent is said to run something")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
|
|
||||||
// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
|
|
||||||
// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
|
|
||||||
// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
|
|
||||||
func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
|
||||||
cutAt(t, open, t0.Add(91*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
|
||||||
}
|
|
||||||
rollout := ws[0].ID
|
|
||||||
asking, err := inv.PlanByID(ctx, rollout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ {
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
buildID := asking.Modules["notes"].Build
|
|
||||||
if buildID == "" || len(*asked) != 1 {
|
|
||||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// What the live path made is in the chain.
|
|
||||||
if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" {
|
|
||||||
t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err)
|
|
||||||
}
|
|
||||||
if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout {
|
|
||||||
t.Fatalf("the rollout is not read from the chain: %+v %v", p, err)
|
|
||||||
}
|
|
||||||
if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
} else if _, ok := assignmentOf(on, "notes"); !ok {
|
|
||||||
t.Fatalf("the assignment is not read from the chain: %+v", on)
|
|
||||||
}
|
|
||||||
|
|
||||||
// And the catalog of the store holds none of them.
|
|
||||||
have := map[string]bool{}
|
|
||||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
|
||||||
have[strings.Trim(row[0], `"`)] = true
|
|
||||||
}
|
|
||||||
for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} {
|
|
||||||
if have[table] {
|
|
||||||
t.Errorf("the table %s is in the controller's store", table)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, table := range []string{"build_gate", "assignment_generation"} {
|
|
||||||
if !have[table] {
|
|
||||||
t.Errorf("the table %s, which stays, is gone", table)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
|
|
||||||
t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"path"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
|
|
||||||
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
|
|
||||||
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
|
|
||||||
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
|
|
||||||
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
|
|
||||||
// planner, so the two can be compared.
|
|
||||||
|
|
||||||
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
|
|
||||||
type policyOf func(module string) (inventory.Upgrade, bool)
|
|
||||||
|
|
||||||
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
|
|
||||||
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
|
|
||||||
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
|
|
||||||
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
|
|
||||||
byName := map[string]inventory.Entry{}
|
|
||||||
for _, e := range entries {
|
|
||||||
byName[e.Manifest.Module] = e
|
|
||||||
}
|
|
||||||
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
|
|
||||||
holder := ""
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
|
|
||||||
holder = e.On[0]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var granting []string
|
|
||||||
machines := map[string]*link.MachinePlan{}
|
|
||||||
machine := func(name string) *link.MachinePlan {
|
|
||||||
if machines[name] == nil {
|
|
||||||
machines[name] = &link.MachinePlan{Machine: name}
|
|
||||||
}
|
|
||||||
return machines[name]
|
|
||||||
}
|
|
||||||
for _, tier := range r.Plan.Tiers {
|
|
||||||
for _, name := range tier {
|
|
||||||
e, held := byName[name]
|
|
||||||
if !held {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
u, known := policy(name)
|
|
||||||
waits := known && !u.RollOut
|
|
||||||
for _, on := range e.On {
|
|
||||||
if waits {
|
|
||||||
machine(on).Waits = append(machine(on).Waits, name)
|
|
||||||
} else {
|
|
||||||
machine(on).Receives = append(machine(on).Receives, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
|
|
||||||
!slices.Contains(granting, name) {
|
|
||||||
granting = append(granting, name)
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
|
||||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
|
||||||
// until a person takes the step, so merging it is a promise to take it.
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+
|
|
||||||
"nothing else is sent to %s either — unless the new build turns out the same as the one running there "+
|
|
||||||
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
|
|
||||||
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
|
|
||||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
|
||||||
case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData):
|
|
||||||
// Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377).
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+
|
|
||||||
"its move recreates none of its containers and moves none of its data, which the walk judges and then sends",
|
|
||||||
name, orNone(u.From)))
|
|
||||||
case waits && len(e.On) > 0:
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
|
||||||
orNone(u.From)))
|
|
||||||
}
|
|
||||||
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
|
|
||||||
var offers []string
|
|
||||||
for _, o := range e.Manifest.Provides {
|
|
||||||
offers = append(offers, o.Name)
|
|
||||||
}
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
|
|
||||||
name, strings.Join(offers, ", ")))
|
|
||||||
}
|
|
||||||
if e.Manifest.Data != nil && len(e.On) > 0 {
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(granting) > 0 {
|
|
||||||
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
|
|
||||||
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
|
|
||||||
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
|
|
||||||
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
|
|
||||||
}
|
|
||||||
var names []string
|
|
||||||
for name := range machines {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
for _, name := range names {
|
|
||||||
p.Machines = append(p.Machines, *machines[name])
|
|
||||||
}
|
|
||||||
p.Summary = summaryOf(p)
|
|
||||||
p.Class = changeClassOf(p, entries)
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
|
|
||||||
const classNone = "none"
|
|
||||||
|
|
||||||
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
|
|
||||||
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
|
|
||||||
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
|
|
||||||
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
|
|
||||||
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
|
|
||||||
// resolver claim its manifest makes.
|
|
||||||
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
|
|
||||||
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
|
|
||||||
return classNone
|
|
||||||
}
|
|
||||||
modules := append(append([]string{}, p.Moved...), p.Dependents...)
|
|
||||||
for _, dir := range p.New {
|
|
||||||
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
|
|
||||||
modules = append(modules, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return classOfModules(modules, entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
|
||||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
|
||||||
const grantsStepWord = "grants step"
|
|
||||||
|
|
||||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
|
||||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
|
||||||
|
|
||||||
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
|
|
||||||
func summaryOf(p link.ChangePlan) string {
|
|
||||||
if len(p.Moved) == 0 && len(p.New) == 0 {
|
|
||||||
return "builds nothing: the change touches no module of the mesh's graph"
|
|
||||||
}
|
|
||||||
var parts []string
|
|
||||||
what := strings.Join(p.Moved, ", ")
|
|
||||||
if len(p.Dependents) > 0 {
|
|
||||||
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
|
|
||||||
}
|
|
||||||
if len(p.New) > 0 {
|
|
||||||
if what != "" {
|
|
||||||
what += ", "
|
|
||||||
}
|
|
||||||
what += "new: " + strings.Join(p.New, ", ")
|
|
||||||
}
|
|
||||||
var to []string
|
|
||||||
for _, m := range p.Machines {
|
|
||||||
if len(m.Receives) > 0 {
|
|
||||||
to = append(to, m.Machine)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(to) > 0 {
|
|
||||||
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
|
|
||||||
} else {
|
|
||||||
parts = append(parts, "builds "+what+", sent nowhere")
|
|
||||||
}
|
|
||||||
bus := "no bus step"
|
|
||||||
for _, s := range p.Steps {
|
|
||||||
switch {
|
|
||||||
case strings.HasPrefix(s, busUpgradeNeeded):
|
|
||||||
bus = busUpgradeNeeded
|
|
||||||
case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step":
|
|
||||||
bus = "a bus step"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
parts = append(parts, bus)
|
|
||||||
waiting := 0
|
|
||||||
for _, m := range p.Machines {
|
|
||||||
waiting += len(m.Waits)
|
|
||||||
}
|
|
||||||
if waiting > 0 {
|
|
||||||
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
|
|
||||||
}
|
|
||||||
return strings.Join(parts, "; ")
|
|
||||||
}
|
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
|
|
||||||
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
|
|
||||||
// person, a provider whose consumers follow it.
|
|
||||||
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
|
||||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
|
||||||
entry := func(name, path string, on ...string) inventory.Entry {
|
|
||||||
e := fromRepo(name, catalogue_, path)
|
|
||||||
e.Source.BuiltFrom = "old"
|
|
||||||
e.On = on
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
nats := entry("nats", "modules/nats", "anchor")
|
|
||||||
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
||||||
gitea := entry("gitea", "modules/gitea", "anchor")
|
|
||||||
held := entry("photos", "modules/photos", "anchor", "laptop")
|
|
||||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
|
||||||
tools.On = []string{"anchor", "laptop"}
|
|
||||||
entries := []inventory.Entry{nats, gitea, held, tools}
|
|
||||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
|
||||||
policy := func(module string) (inventory.Upgrade, bool) {
|
|
||||||
if module == "photos" {
|
|
||||||
return inventory.Upgrade{RollOut: false, From: "a person"}, true
|
|
||||||
}
|
|
||||||
return inventory.Upgrade{RollOut: true}, true
|
|
||||||
}
|
|
||||||
plan := func(paths ...string) link.ChangePlan {
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
|
||||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
|
||||||
}
|
|
||||||
|
|
||||||
p := plan("modules/gitea/index.ts")
|
|
||||||
if p.Summary != "builds gitea → anchor; no bus step" {
|
|
||||||
t.Errorf("one module's change reads %q", p.Summary)
|
|
||||||
}
|
|
||||||
|
|
||||||
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
|
|
||||||
if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
|
||||||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
|
|
||||||
t.Errorf("the bus and a held module read %q", p.Summary)
|
|
||||||
}
|
|
||||||
got := map[string]string{}
|
|
||||||
for _, m := range p.Machines {
|
|
||||||
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
|
|
||||||
}
|
|
||||||
// The bus first, what stands on it after: the build plan's order, per machine.
|
|
||||||
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
|
|
||||||
t.Errorf("the deploy plan reads %v", got)
|
|
||||||
}
|
|
||||||
text := strings.Join(p.Steps, "\n")
|
|
||||||
// Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a
|
|
||||||
// person runs the bus's step, and the verb that does.
|
|
||||||
for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor",
|
|
||||||
"mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person",
|
|
||||||
"nats provides mesh-bus"} {
|
|
||||||
if !strings.Contains(text, want) {
|
|
||||||
t.Errorf("the steps do not say %q:\n%s", want, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
p = plan("merge-check.sh")
|
|
||||||
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
|
|
||||||
t.Errorf("a root file's plan reads %+v", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,337 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
|
|
||||||
//
|
|
||||||
// **The same functions registration runs, and nothing the command line adds** (ADR 0035): the strict
|
|
||||||
// parse with every per-manifest problem, then the rules no single manifest can be judged against,
|
|
||||||
// over exactly the manifests given. Somebody describing their own application in their own
|
|
||||||
// repository runs this before pushing and finds out there, rather than when a running mesh refuses
|
|
||||||
// the registration or, later, when a machine applies something that resolved and should not have.
|
|
||||||
//
|
|
||||||
// **What it cannot know without a store, it says.** The mesh's own seat set is the store's (ADR
|
|
||||||
// 0122); this binary carries a compiled copy that the store overrides when loaded, so a claim on a
|
|
||||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
|
||||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
|
||||||
// refused what it could not see would teach people to ignore it.
|
|
||||||
//
|
|
||||||
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
|
||||||
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
|
||||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
|
||||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
|
||||||
// provider's machine when a real machine's name first meets the module's.
|
|
||||||
// SomeManifestsVar, set by the merge gate, says the manifests given are only some of their repository's.
|
|
||||||
const SomeManifestsVar = "MESH_MODULE_CHECK_SOME"
|
|
||||||
|
|
||||||
func moduleCheck(paths []string, out io.Writer) error {
|
|
||||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
|
||||||
}
|
|
||||||
|
|
||||||
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|
||||||
if len(paths) == 0 {
|
|
||||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
|
||||||
"manifest of a repository together so the rules between them are checked too")
|
|
||||||
}
|
|
||||||
shelf := catalogue.Shelf{}
|
|
||||||
faulted := map[string]bool{}
|
|
||||||
failed := 0
|
|
||||||
// The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read
|
|
||||||
// once per repository: a manifest naming a key is judged against the list its own tree holds.
|
|
||||||
lists := map[string]catalogue.PredefinedList{}
|
|
||||||
listOf := func(path string) (catalogue.PredefinedList, error) {
|
|
||||||
root := catalogue.RepositoryRoot(filepath.Dir(path))
|
|
||||||
if root == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if list, read := lists[root]; read {
|
|
||||||
return list, nil
|
|
||||||
}
|
|
||||||
list, err := catalogue.ReadPredefined(root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
lists[root] = list
|
|
||||||
return list, nil
|
|
||||||
}
|
|
||||||
for _, path := range paths {
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
|
||||||
failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
list, err := listOf(path)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
|
||||||
failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
m, err := catalogue.ParseManifestWith(raw, list)
|
|
||||||
if err == nil {
|
|
||||||
// Each template's text read from the tree beside the manifest, and judged as inline content is
|
|
||||||
// (novox/hq ADR 0349 §2): a missing template is refused by name.
|
|
||||||
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
|
||||||
failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if first, twice := shelf[m.Module]; twice {
|
|
||||||
_ = first
|
|
||||||
fmt.Fprintf(out, "%s: %s was already given; two manifests name one module\n", path, m.Module)
|
|
||||||
failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
|
|
||||||
// catalogue-wide test, and at registration, which refuses in the same words.
|
|
||||||
if wrong := catalogue.TrustProblems(m); len(wrong) > 0 {
|
|
||||||
for _, p := range wrong {
|
|
||||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
|
||||||
}
|
|
||||||
failed += len(wrong)
|
|
||||||
faulted[m.Module] = true
|
|
||||||
}
|
|
||||||
// A module that declares a setting names the keys of its mergeable files (novox/hq ADR 0326 §3,
|
|
||||||
// issue 574): judged where a definition is written, as at registration, never where a stored
|
|
||||||
// build is read again.
|
|
||||||
if wrong := catalogue.WrittenSettingsProblems(m); len(wrong) > 0 {
|
|
||||||
for _, p := range wrong {
|
|
||||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
|
||||||
}
|
|
||||||
failed += len(wrong)
|
|
||||||
faulted[m.Module] = true
|
|
||||||
}
|
|
||||||
// A module lives whole in one repository (novox/hq ADR 0420 §1, issue 1019): when the check is told the
|
|
||||||
// repository it runs for (MESH_CHECK_REPOSITORY, set by the build's check), an artifact building from
|
|
||||||
// another is refused, as the merge gate refuses it. A run by hand names no repository and judges none.
|
|
||||||
if repo := os.Getenv("MESH_CHECK_REPOSITORY"); repo != "" && m.Build != nil {
|
|
||||||
for _, a := range m.Build.Artifacts {
|
|
||||||
if a.Context != nil && a.Context.Repository != "" && !sameRepository(a.Context.Repository,
|
|
||||||
link.SourceMoved{Owner: ownerOf(repo), Repo: repoOf(repo)}) {
|
|
||||||
fmt.Fprintf(out, "%s: %s builds artifact %s from %s: a module lives whole in one repository "+
|
|
||||||
"(novox/hq ADR 0420 §1)\n", path, m.Module, a.Name, a.Context.Repository)
|
|
||||||
failed++
|
|
||||||
faulted[m.Module] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
|
||||||
for _, p := range named {
|
|
||||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
|
||||||
}
|
|
||||||
failed += len(named)
|
|
||||||
faulted[m.Module] = true
|
|
||||||
}
|
|
||||||
shelf[m.Module] = m
|
|
||||||
}
|
|
||||||
|
|
||||||
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
|
||||||
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
|
||||||
// has already been said and would be said again here in a worse form.
|
|
||||||
//
|
|
||||||
// **Over some of a repository's manifests, a seat none of them declares is a note** (novox/hq issue 364), as
|
|
||||||
// this command's own word says above: the merge gate passes only the manifests a change touches, and says so
|
|
||||||
// with SomeManifestsVar, so a module that uses or claims a seat another module declares (the operator
|
|
||||||
// channel's, a channel bench) was refused there for a manifest it was not given. Given every manifest — the
|
|
||||||
// catalogue's own check, and registration — it stays a refusal.
|
|
||||||
some := os.Getenv(SomeManifestsVar) != ""
|
|
||||||
problems := catalogue.CatalogueProblems(shelf)
|
|
||||||
sort.Strings(problems)
|
|
||||||
for _, p := range problems {
|
|
||||||
if some && catalogue.IsUndeclaredSeat(p) {
|
|
||||||
fmt.Fprintf(out, "note: %s among the manifests given; registration judges it against the whole catalogue, "+
|
|
||||||
"and passing the declaring module's manifest too judges it here\n", p)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fmt.Fprintln(out, p)
|
|
||||||
failed++
|
|
||||||
}
|
|
||||||
|
|
||||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
|
||||||
// only the provider's manifest states.
|
|
||||||
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
|
||||||
sort.Strings(identities)
|
|
||||||
for _, p := range identities {
|
|
||||||
fmt.Fprintln(out, p)
|
|
||||||
}
|
|
||||||
failed += len(identities)
|
|
||||||
|
|
||||||
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
|
|
||||||
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
|
|
||||||
data := catalogue.DataProblems(shelf)
|
|
||||||
sort.Strings(data)
|
|
||||||
for _, p := range data {
|
|
||||||
fmt.Fprintln(out, p)
|
|
||||||
}
|
|
||||||
failed += len(data)
|
|
||||||
|
|
||||||
var names []string
|
|
||||||
for name := range shelf {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
|
|
||||||
// **Every long-running resource says how it is ready** (novox/hq ADR 0240 rule 8): warned until the
|
|
||||||
// date, refused from it. The count is the catalogue's: its merge check keeps the number and lets a
|
|
||||||
// change lower it, never raise it.
|
|
||||||
undeclared := 0
|
|
||||||
required := !checkNow().Before(catalogue.HealthRequiredFrom)
|
|
||||||
for _, name := range names {
|
|
||||||
missing := catalogue.Undeclared(shelf[name])
|
|
||||||
undeclared += len(missing)
|
|
||||||
if len(missing) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if required {
|
|
||||||
for _, id := range missing {
|
|
||||||
fmt.Fprintf(out, "%s: %s stays up and does not say how it is ready: a long-running resource declares "+
|
|
||||||
"health since %s (novox/hq ADR 0240 rule 8)\n", name, id, catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
|
||||||
}
|
|
||||||
failed += len(missing)
|
|
||||||
faulted[name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A file that asks for a setting and does not say whether it is trusted counts as trusted** (novox/hq issue
|
|
||||||
// 339): listed and counted, never refused, so an author can opt out a file nothing trusts.
|
|
||||||
unsaid := 0
|
|
||||||
for _, name := range names {
|
|
||||||
unsaid += len(catalogue.UnsaidTrust(shelf[name]))
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
m := shelf[name]
|
|
||||||
if faulted[name] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, "%s: ok", name)
|
|
||||||
if n := len(m.Tools); n > 0 {
|
|
||||||
fmt.Fprintf(out, ", %d tool(s)", n)
|
|
||||||
}
|
|
||||||
if len(m.Invokes) > 0 {
|
|
||||||
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
|
||||||
}
|
|
||||||
// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
|
|
||||||
if len(m.State) > 0 {
|
|
||||||
kept := make([]string, 0, len(m.State))
|
|
||||||
for _, s := range m.State {
|
|
||||||
kept = append(kept, s.Name)
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
|
||||||
}
|
|
||||||
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
|
||||||
if len(m.Logs) > 0 {
|
|
||||||
kept := make([]string, 0, len(m.Logs))
|
|
||||||
for _, l := range m.Logs {
|
|
||||||
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
|
||||||
}
|
|
||||||
if len(m.Reads) > 0 {
|
|
||||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
|
||||||
}
|
|
||||||
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
|
|
||||||
if items := m.DataItems(); len(items) > 0 {
|
|
||||||
kept := make([]string, 0, len(items))
|
|
||||||
for _, it := range items {
|
|
||||||
kept = append(kept, it.ID+" ("+it.Class+")")
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
|
||||||
}
|
|
||||||
// How what it runs is ready (ADR 0240): each declared check, and what is judged by liveness alone.
|
|
||||||
var checks []string
|
|
||||||
for _, r := range m.Resources {
|
|
||||||
if h, has, _ := catalogue.ReadHealth(r); has {
|
|
||||||
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// And its own runtime's, where it runs as its own account (novox/hq ADR 0366).
|
|
||||||
if own, has := catalogue.OwnRuntimeHealth(m); has {
|
|
||||||
if h, _, _ := catalogue.ReadHealth(own); h.Kind != "" {
|
|
||||||
checks = append(checks, fmt.Sprintf("%v by %s", own["id"], catalogue.HealthWords(h)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(checks) > 0 {
|
|
||||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
|
||||||
}
|
|
||||||
if missing := catalogue.UnsaidTrust(m); len(missing) > 0 {
|
|
||||||
fmt.Fprintf(out, "; WARNING: %s ask(s) for a setting and do(es) not say whether it is trusted, so it counts as "+
|
|
||||||
"trusted: set at the terminal alone; say %q false where nothing trusts it (novox/hq issue 339)",
|
|
||||||
strings.Join(missing, ", "), catalogue.TrustedField)
|
|
||||||
}
|
|
||||||
if missing := catalogue.Undeclared(m); len(missing) > 0 {
|
|
||||||
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
|
|
||||||
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
|
||||||
}
|
|
||||||
fmt.Fprintln(out)
|
|
||||||
}
|
|
||||||
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
|
|
||||||
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
|
|
||||||
fmt.Fprintf(out, "%s %d\n", UnsaidTrustLine, unsaid)
|
|
||||||
if failed > 0 {
|
|
||||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
|
||||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
|
||||||
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
|
||||||
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UndeclaredHealthLine starts the line `module check` says the count of long-running resources without
|
|
||||||
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
|
|
||||||
const UndeclaredHealthLine = "long-running resources without health:"
|
|
||||||
|
|
||||||
// UnsaidTrustLine starts the line `module check` says the count of files that ask for a setting and do not say
|
|
||||||
// whether it is trusted, and so count as trusted (novox/hq issue 339).
|
|
||||||
const UnsaidTrustLine = "files asking for a setting without saying whether it is trusted:"
|
|
||||||
|
|
||||||
// checkNow is the clock `module check` judges the date by; a test sets it.
|
|
||||||
var checkNow = time.Now
|
|
||||||
|
|
||||||
func joinInvokes(invokes []string) string {
|
|
||||||
if len(invokes) == 1 && invokes[0] == "*" {
|
|
||||||
return "every tool"
|
|
||||||
}
|
|
||||||
s := ""
|
|
||||||
for i, t := range invokes {
|
|
||||||
if i > 0 {
|
|
||||||
s += ", "
|
|
||||||
}
|
|
||||||
s += t
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// manifestsUnder lists every module.json below a directory, for `module check <dir>`.
|
|
||||||
func manifestsUnder(dir string) ([]string, error) {
|
|
||||||
var found []string
|
|
||||||
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git" || d.Name() == "dist") {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
if !d.IsDir() && d.Name() == "module.json" {
|
|
||||||
found = append(found, path)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
sort.Strings(found)
|
|
||||||
return found, err
|
|
||||||
}
|
|
||||||
@@ -1,161 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/beside"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
|
||||||
// with a known fault is named, and one without passes, with no store opened.
|
|
||||||
func TestModuleCheckNamesAFaultAndNeedsNoMesh(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
good := filepath.Join(dir, "good.json")
|
|
||||||
bad := filepath.Join(dir, "bad.json")
|
|
||||||
os.WriteFile(good, []byte(`{"module":"shop","version":"1","tools":["price"],"invokes":["mesh-catalog.catalog_modules"]}`), 0o600)
|
|
||||||
os.WriteFile(bad, []byte(`{"module":"till","version":"1","invokes":["shop"]}`), 0o600)
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{good}, &out); err != nil {
|
|
||||||
t.Fatalf("a sound manifest was refused: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "shop: ok, 1 tool(s), invokes mesh-catalog.catalog_modules") {
|
|
||||||
t.Fatalf("the report does not say what it checked:\n%s", out.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
out.Reset()
|
|
||||||
err := moduleCheck([]string{good, bad}, &out)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("a manifest invoking a module and no tool passed")
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), `till invokes "shop", which does not name a tool`) {
|
|
||||||
t.Fatalf("the fault is not named in the manifest's words:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The rules between manifests run over what was given together: a seat two modules declare is
|
|
||||||
// refused, which no single-manifest check can see.
|
|
||||||
func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
a := filepath.Join(dir, "a.json")
|
|
||||||
b := filepath.Join(dir, "b.json")
|
|
||||||
os.WriteFile(a, []byte(`{"module":"a","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
|
||||||
os.WriteFile(b, []byte(`{"module":"b","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{a, b}, &out); err == nil {
|
|
||||||
t.Fatalf("two declarations of one seat passed:\n%s", out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "a seat name means one protocol") {
|
|
||||||
t.Fatalf("the cross-manifest rule was not the one named:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
|
||||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
|
||||||
root := beside.Catalogue(t)
|
|
||||||
paths, err := manifestsUnder(root)
|
|
||||||
if err != nil || len(paths) == 0 {
|
|
||||||
t.Fatalf("no manifests under %s: %v", root, err)
|
|
||||||
}
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck(paths, &out); err != nil {
|
|
||||||
t.Fatalf("the catalogue does not pass its own check: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
|
|
||||||
// novox/hq ADR 0155: the check moves to registration once the catalogue passes it. Both
|
|
||||||
// ways in — `module add` and a build's result — go through this, and a name declared on
|
|
||||||
// purpose passes with its reason.
|
|
||||||
named := catalogue.Manifest{Module: "idp", Resources: []map[string]any{
|
|
||||||
{"id": "server", "type": "container", "image": "x@sha256:aa",
|
|
||||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}},
|
|
||||||
}}
|
|
||||||
err := namesNoInstallation(named)
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "login.mesh-one.be") ||
|
|
||||||
!strings.Contains(err.Error(), catalogue.NamesOnPurpose) {
|
|
||||||
t.Fatalf("a definition naming an installation is refused with the name and the way out; got %v", err)
|
|
||||||
}
|
|
||||||
meant := catalogue.Manifest{Module: "site", Resources: []map[string]any{
|
|
||||||
{"id": "server", "type": "container", "image": "registry.mesh-one.be/org/site@sha256:cc",
|
|
||||||
catalogue.NamesOnPurpose: map[string]any{
|
|
||||||
"registry.mesh-one.be": "built outside the mesh until its repository is a build source here"}},
|
|
||||||
}}
|
|
||||||
if err := namesNoInstallation(meant); err != nil {
|
|
||||||
t.Fatalf("a name declared on purpose passes; got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged
|
|
||||||
// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it —
|
|
||||||
// refused it. Held here, so a verb added to the table without the manifest fails this repository's
|
|
||||||
// own suite rather than its next pull request's gate.
|
|
||||||
func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
|
|
||||||
var out strings.Builder
|
|
||||||
if err := moduleCheck([]string{"../../module.json"}, &out); err != nil {
|
|
||||||
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module that uses a seat another module declares (novox/hq issue 364): refused over the whole catalogue when the
|
|
||||||
// declarer is missing, a note when the gate says it gives only the manifests a change touches.
|
|
||||||
func TestASeatAnotherModuleDeclaresIsANoteOverSomeManifests(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
user := filepath.Join(dir, "user.json")
|
|
||||||
os.WriteFile(user, []byte(`{"module":"asker","version":"1","uses":["operator-channel"]}`), 0o600)
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{user}, &out); err == nil {
|
|
||||||
t.Fatalf("a use of a seat nothing given declares passed the whole-catalogue check:\n%s", out.String())
|
|
||||||
}
|
|
||||||
t.Setenv(SomeManifestsVar, "1")
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{user}, &out); err != nil {
|
|
||||||
t.Fatalf("over some manifests the use was refused:\n%s", out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "note: asker uses the seat \"operator-channel\"") {
|
|
||||||
t.Fatalf("the note was not said:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A module lives whole in one repository** (novox/hq ADR 0420 §1, issue 1019): `module check`, told the
|
|
||||||
// repository it runs for, refuses an artifact whose build context names another; run by hand, it does not know.
|
|
||||||
func TestModuleCheckRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
write := func(name, context string) string {
|
|
||||||
path := filepath.Join(dir, name+".json")
|
|
||||||
raw := `{"module":"` + name + `","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"` + context + `}]}}`
|
|
||||||
if err := os.WriteFile(path, []byte(raw), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
other := write("other", `,"context":{"seat":"git","repository":"novox/mesh-controller","ref":"main"}`)
|
|
||||||
own := write("own", `,"context":{"seat":"git","repository":"novox/mesh-catalog","ref":"main"}`)
|
|
||||||
none := write("none", ``)
|
|
||||||
|
|
||||||
t.Setenv("MESH_CHECK_REPOSITORY", "novox/mesh-catalog")
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{other}, &out); err == nil {
|
|
||||||
t.Fatalf("a context naming another repository passed:\n%s", out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "other builds artifact server from novox/mesh-controller: a module lives whole in one repository (novox/hq ADR 0420 §1)") {
|
|
||||||
t.Errorf("the refusal is not in the words of the gate:\n%s", out.String())
|
|
||||||
}
|
|
||||||
for _, path := range []string{own, none} {
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
|
||||||
t.Errorf("%s was refused: %v\n%s", path, err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Setenv("MESH_CHECK_REPOSITORY", "")
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{other}, &out); err != nil {
|
|
||||||
t.Errorf("a local run, with no repository named, refused it: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,191 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"regexp"
|
|
||||||
"runtime/debug"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What a pull request reaches of the mesh's graph (novox/hq to-be 45 §9, issue 280). The controller no longer checks a
|
|
||||||
// pull request or says a verdict on it (R4g, ADR 0503 §3, ADR 0501 §5): the checker judges the heads, and mesh-delivery
|
|
||||||
// asks the operator's Approve on its verdict. What stays here is the mapping of a pull request's changed paths onto the
|
|
||||||
// mesh's module graph by the planner's own answer (reachOfMerge, touchedBy), which the delivery verbs (`delivery order`,
|
|
||||||
// the change plan) and the head's declared config (tellPullDeclares) read: **a changed file touches exactly the modules
|
|
||||||
// whose build reads it** — a module's own directory (the whole repository for one built from its root), or a repository
|
|
||||||
// its recipe packages — and a directory the change adds a module.json in is a new module.
|
|
||||||
|
|
||||||
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
|
|
||||||
// each beside it — and whose owner is the mesh's own.
|
|
||||||
var coreModules = map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host",
|
|
||||||
"node-tools": "mesh-tools", "nats": "mesh-catalog"}
|
|
||||||
|
|
||||||
// checkScope is what a pull request reaches of the mesh's graph, as the planner reckons it.
|
|
||||||
type checkScope struct {
|
|
||||||
// Modules are the modules a merge of the change would move itself — built from the repository into
|
|
||||||
// the pull request's base and reading a changed file, or packaging the repository's source — and
|
|
||||||
// Dependents those the plan would build after them; New the directories it adds a module in.
|
|
||||||
Modules []string
|
|
||||||
Dependents []string
|
|
||||||
New []string
|
|
||||||
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
|
|
||||||
Manifests []string
|
|
||||||
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
|
|
||||||
// module's build reads.
|
|
||||||
Width, Tiers int
|
|
||||||
Unread []string
|
|
||||||
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
|
|
||||||
// core's, or the change adds a module to it.
|
|
||||||
Mesh bool
|
|
||||||
// From is a module built from the repository, for how the mesh clones it; nil when none is.
|
|
||||||
From *inventory.Entry
|
|
||||||
// Reach is the planner's whole answer, which the change plan is made from.
|
|
||||||
Reach mergeReach
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s checkScope) gated() bool { return len(s.Modules)+len(s.New) > 0 }
|
|
||||||
|
|
||||||
// pullScope is what a pull request reaches: **the planner's own answer** (reachOfMerge), asked as if the
|
|
||||||
// head were merged into the base — never a mapping of its own, so a change to what a merge touches
|
|
||||||
// changes what is checked with it (novox/hq ADR 0238).
|
|
||||||
func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
|
||||||
edges []inventory.Edge) checkScope {
|
|
||||||
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL, Commit: p.Commit,
|
|
||||||
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
|
|
||||||
ModuleDirsSaid: p.ModuleDirsSaid}
|
|
||||||
r := reachOfMerge(m, entries, read, edges)
|
|
||||||
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
|
|
||||||
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
|
|
||||||
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
|
|
||||||
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
|
|
||||||
}
|
|
||||||
for _, d := range r.Added {
|
|
||||||
s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile))
|
|
||||||
}
|
|
||||||
sort.Strings(s.Manifests)
|
|
||||||
s.Manifests = slices.Compact(s.Manifests)
|
|
||||||
|
|
||||||
// Whose repository it is, for how it is cloned and whether its own check is the mesh's to run.
|
|
||||||
owners := map[string]bool{}
|
|
||||||
for i, e := range entries {
|
|
||||||
if e.Provided {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, core := coreModules[e.Manifest.Module]; core {
|
|
||||||
if owner := sourceOwner(e.Source.Repository); owner != "" {
|
|
||||||
owners[owner] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if sameRepository(e.Source.Repository, m) && s.From == nil {
|
|
||||||
s.From = &entries[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s.Mesh = s.From != nil || owners[strings.ToLower(p.Owner)] || s.gated()
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceOwner is the owner of a recorded repository, a path on the git seat or a URL: novox/mesh-host → novox.
|
|
||||||
func sourceOwner(repository string) string {
|
|
||||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
|
||||||
if len(parts) < 2 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return strings.ToLower(parts[len(parts)-2])
|
|
||||||
}
|
|
||||||
|
|
||||||
// PullUpdated hears a pull request's new head. It asks no check and says no verdict (pullUpdated).
|
|
||||||
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|
||||||
return f.pullUpdated(ctx, p, os.Stdout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// pullUpdated is PullUpdated saying what it did to w. It asks nothing of anybody and says no verdict (R4g; ADR 0503 §3, §5,
|
|
||||||
// ADR 0527): the checker judges every repository whose trunk requires `mesh/check`, hq included. All it still does is tell
|
|
||||||
// mesh-config what the head declares (ADR 0502), whatever judges it.
|
|
||||||
func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Writer) error {
|
|
||||||
inv := f.open.inventory
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
read, err := readForPlanning(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
edges, err := inv.Dependencies(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
scope := pullScope(p, entries, read, edges)
|
|
||||||
this := pullAtOf(p)
|
|
||||||
if this.Truncated {
|
|
||||||
this.Paths, this.Truncated = truncatedPaths(entries, p.Owner, p.Repo, scope.Modules), false
|
|
||||||
}
|
|
||||||
if scope.gated() || holdsModules(entries, p.Owner, p.Repo) {
|
|
||||||
tellPullDeclares(ctx, w, []pullAt{this})
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
|
||||||
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
|
||||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
|
||||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
|
||||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
|
||||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the
|
|
||||||
// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not
|
|
||||||
// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the
|
|
||||||
// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is
|
|
||||||
// judged against the SDK it moves to (internal/link/conformance_test.go).
|
|
||||||
func besideRefs(dir, running string) map[string]string {
|
|
||||||
switch dir {
|
|
||||||
case "mesh-catalog":
|
|
||||||
return map[string]string{dir: "main"}
|
|
||||||
case "mesh-controller":
|
|
||||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main",
|
|
||||||
"mesh-sdk": sdkPinned()}
|
|
||||||
}
|
|
||||||
return map[string]string{dir: running}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sdkModule is the Go module of the SDK the controller is built against.
|
|
||||||
const sdkModule = "git.novox.be/novox/mesh-sdk/go"
|
|
||||||
|
|
||||||
// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970.
|
|
||||||
var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`)
|
|
||||||
|
|
||||||
// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its
|
|
||||||
// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/).
|
|
||||||
// "main" only for a binary that records no SDK version — a local replace — which a running controller is
|
|
||||||
// not (novox/hq issue 449).
|
|
||||||
func sdkPinned() string {
|
|
||||||
info, ok := debug.ReadBuildInfo()
|
|
||||||
if !ok {
|
|
||||||
return "main"
|
|
||||||
}
|
|
||||||
for _, dep := range info.Deps {
|
|
||||||
if dep.Path != sdkModule {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if dep.Replace != nil {
|
|
||||||
dep = dep.Replace
|
|
||||||
}
|
|
||||||
if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil {
|
|
||||||
return m[1]
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(dep.Version, "v") {
|
|
||||||
return "go/" + dep.Version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "main"
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkEvents is where the serving controller says a check's verdict; nil in a command.
|
|
||||||
var checkEvents link.Bus
|
|
||||||
@@ -1,152 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **The mesh's module graph decides what a pull request's check runs**, not the repository (novox/hq
|
|
||||||
// ADR 0237 as amended): a change is mapped onto the graph by the rule a merge is (issue 278), the gate
|
|
||||||
// runs when it touches a module — a new one included — and a repository that is the mesh's and touches
|
|
||||||
// none still has its own merge-check.sh run.
|
|
||||||
func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) {
|
|
||||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
|
||||||
const controller = "http://forge.internal:20000/novox/mesh-controller.git"
|
|
||||||
const host = "http://forge.internal:20000/novox/mesh-host.git"
|
|
||||||
photos := fromRepo("photos", "http://forge.internal:20000/novox/photos.git", "")
|
|
||||||
photos.Source.Ref = "nox-mesh"
|
|
||||||
snake := fromRepo("snake", "jschoubben/snake", "")
|
|
||||||
snake.Source.Seat = "git"
|
|
||||||
entries := []inventory.Entry{
|
|
||||||
fromRepo("gitea", catalogue, "modules/gitea"),
|
|
||||||
fromRepo("keycloak", catalogue, "modules/keycloak"),
|
|
||||||
fromRepo("nats", catalogue, "modules/nats"),
|
|
||||||
fromRepo("mesh-controller", controller, ""),
|
|
||||||
fromRepo("mesh-host", host, ""),
|
|
||||||
photos, snake,
|
|
||||||
}
|
|
||||||
pull := func(owner, repo, base string, paths []string, dirs ...string) link.PullUpdated {
|
|
||||||
return link.PullUpdated{Owner: owner, Repo: repo, Base: base, Commit: "abc", Paths: paths,
|
|
||||||
ModuleDirs: dirs, ModuleDirsSaid: true}
|
|
||||||
}
|
|
||||||
for _, c := range []struct {
|
|
||||||
what string
|
|
||||||
p link.PullUpdated
|
|
||||||
modules, new, manifest string
|
|
||||||
mesh bool
|
|
||||||
judge string
|
|
||||||
}{
|
|
||||||
{"one module's own files", pull("novox", "mesh-catalog", "main", []string{"modules/gitea/index.ts"}, "modules/gitea"),
|
|
||||||
"gitea", "", "modules/gitea/module.json", true, ""},
|
|
||||||
{"a file no module's build reads touches no module (issue 280)",
|
|
||||||
pull("novox", "mesh-catalog", "main", []string{"merge-check.sh", "README.md"}), "", "", "", true, ""},
|
|
||||||
{"files the announcer could not list: everything built from it",
|
|
||||||
link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", PathsTruncated: true,
|
|
||||||
Paths: []string{"README.md"}}, "gitea,keycloak,nats", "",
|
|
||||||
"modules/gitea/module.json,modules/keycloak/module.json,modules/nats/module.json", true, ""},
|
|
||||||
{"a new module, said by the head", pull("novox", "mesh-catalog", "main",
|
|
||||||
[]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, "modules/newmod"),
|
|
||||||
"", "modules/newmod", "modules/newmod/module.json", true, ""},
|
|
||||||
{"the controller judges itself", pull("novox", "mesh-controller", "main", []string{"cmd/x.go"}),
|
|
||||||
"mesh-controller", "", "module.json", true, "self"},
|
|
||||||
{"the node-engine is judged by the controller the mesh runs, as any module is (issue 1391)",
|
|
||||||
pull("novox", "mesh-host", "main", []string{"validate/v.go"}), "mesh-host", "", "module.json", true, ""},
|
|
||||||
{"the core's owner, no module: the mesh's, its own check alone", pull("novox", "hq", "main", []string{"README.md"}),
|
|
||||||
"", "", "", true, ""},
|
|
||||||
{"a branch nothing is built from: the mesh's repository, no module", pull("novox", "photos", "master",
|
|
||||||
[]string{"server/x.js"}), "", "", "", true, ""},
|
|
||||||
{"the branch a module is built from", pull("novox", "photos", "nox-mesh", []string{"server/x.js"}),
|
|
||||||
"photos", "", "module.json", true, ""},
|
|
||||||
{"a repository on the forge's seat", pull("jschoubben", "snake", "main", []string{"index.html"}),
|
|
||||||
"snake", "", "module.json", true, ""},
|
|
||||||
{"a repository of nobody's, touching nothing", pull("someone", "dotfiles", "main", []string{"x"}),
|
|
||||||
"", "", "", false, ""},
|
|
||||||
{"a repository adding a module at its root", pull("someone", "newapp", "main", []string{"module.json", "x.js"}),
|
|
||||||
"", ".", "module.json", true, ""},
|
|
||||||
} {
|
|
||||||
s := pullScope(c.p, entries, nil, nil)
|
|
||||||
got := []string{strings.Join(s.Modules, ","), strings.Join(s.New, ","), strings.Join(s.Manifests, ",")}
|
|
||||||
want := []string{c.modules, c.new, c.manifest}
|
|
||||||
for i, what := range []string{"modules", "new", "manifests"} {
|
|
||||||
if got[i] != want[i] {
|
|
||||||
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if s.Mesh != c.mesh {
|
|
||||||
t.Errorf("%s: the mesh's %v, wanted %v", c.what, s.Mesh, c.mesh)
|
|
||||||
}
|
|
||||||
if s.gated() != (c.modules != "" || c.new != "") {
|
|
||||||
t.Errorf("%s: gated %v", c.what, s.gated())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module whose build packages another repository's source is touched by a change to it.
|
|
||||||
func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
|
|
||||||
entries := []inventory.Entry{fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")}
|
|
||||||
read := map[string][]inventory.ReadRepository{"node-tools": {{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}}
|
|
||||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "abc", Paths: []string{"go/x.go"}}, entries, read, nil)
|
|
||||||
if strings.Join(s.Modules, ",") != "node-tools" || len(s.Manifests) != 0 {
|
|
||||||
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
|
|
||||||
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
|
|
||||||
// module in both, and what stands on a touched module is named as its dependents in both.
|
|
||||||
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
|
|
||||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
|
||||||
nats := fromRepo("nats", catalogue, "modules/nats")
|
|
||||||
nats.Source.BuiltFrom = "old"
|
|
||||||
gitea := fromRepo("gitea", catalogue, "modules/gitea")
|
|
||||||
gitea.Source.BuiltFrom = "old"
|
|
||||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
|
||||||
entries := []inventory.Entry{nats, gitea, tools}
|
|
||||||
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
|
|
||||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
|
||||||
read := map[string][]inventory.ReadRepository{}
|
|
||||||
|
|
||||||
for _, c := range []struct {
|
|
||||||
what string
|
|
||||||
paths []string
|
|
||||||
moved, dependents string
|
|
||||||
width int
|
|
||||||
unread string
|
|
||||||
}{
|
|
||||||
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
|
|
||||||
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
|
|
||||||
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
|
|
||||||
} {
|
|
||||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
|
||||||
s := pullScope(p, entries, read, edges)
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
|
||||||
r := reachOfMerge(m, entries, read, edges)
|
|
||||||
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
|
|
||||||
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
|
|
||||||
}
|
|
||||||
if got := strings.Join(s.Dependents, ","); got != c.dependents {
|
|
||||||
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
|
|
||||||
}
|
|
||||||
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
|
|
||||||
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
|
|
||||||
}
|
|
||||||
if got := strings.Join(s.Unread, ","); got != c.unread {
|
|
||||||
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A repository whose build another module's recipe packages: that module is reached through the
|
|
||||||
// planner's `read`, and what stands on it after it.
|
|
||||||
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
|
|
||||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
|
|
||||||
Paths: []string{"src/index.ts"}}, entries, read, edges)
|
|
||||||
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
|
|
||||||
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
|
|
||||||
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
|
|
||||||
@@ -1,285 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/artifacts"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Letting the artifact store go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
|
||||||
//
|
|
||||||
// **Run where the records change.** A build is the moment new bytes landed in the store and the
|
|
||||||
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
|
||||||
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
|
||||||
//
|
|
||||||
// **And run when a person asks** (novox/hq ADR 0251): the `collect` verb is the same sweep, with
|
|
||||||
// bounds of its own. One implementation, so what a person runs is exactly what a build runs.
|
|
||||||
//
|
|
||||||
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
|
||||||
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
|
||||||
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
|
||||||
// having failed.
|
|
||||||
|
|
||||||
// sweepBounds is how much one sweep may ask of the store.
|
|
||||||
type sweepBounds struct {
|
|
||||||
// most is how many artifacts it lets go of at most.
|
|
||||||
most int
|
|
||||||
// budget is the longest it keeps its caller waiting.
|
|
||||||
budget time.Duration
|
|
||||||
// platforms is whether an index is let go of with its own platform manifests (novox/hq ADR 0257).
|
|
||||||
// Only a sweep a person confirmed, after its dry run listed what stays and names a platform: the
|
|
||||||
// records cannot see an unrecorded index, or a copy in progress, naming the same platform, and the
|
|
||||||
// store's tools can.
|
|
||||||
platforms bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// afterBuild are the bounds of the sweep inside somebody's build.
|
|
||||||
//
|
|
||||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has never
|
|
||||||
// collected has the whole history to get through, and a person waiting on `build` should not pay for
|
|
||||||
// it. What is left over is simply offered again next time — builds are frequent, and the point is
|
|
||||||
// that the store stops growing, not that it empties tonight.
|
|
||||||
var afterBuild = sweepBounds{most: mostPerSweep, budget: sweepBudget}
|
|
||||||
|
|
||||||
// sweepResult is what one sweep did.
|
|
||||||
type sweepResult struct {
|
|
||||||
// HoldersWritten is how many kept archives the store now holds by a manifest it did not before.
|
|
||||||
HoldersWritten int
|
|
||||||
// Missing is how many kept archives the store does not have at all.
|
|
||||||
Missing int
|
|
||||||
// Held is whether every kept archive could be asked to be held; false stops the sweep before it
|
|
||||||
// lets anything go.
|
|
||||||
Held bool
|
|
||||||
// LetGo is what the store let go of, and the record now says collected.
|
|
||||||
LetGo []string
|
|
||||||
// Skipped is how many references the sweep will not address (novox/hq issue 226).
|
|
||||||
Skipped int
|
|
||||||
// Indexes is how many eligible indexes a sweep a person did not confirm left for one that is: an
|
|
||||||
// index goes only with its platform manifests (novox/hq ADR 0257 §4).
|
|
||||||
Indexes int
|
|
||||||
// Left is how many eligible references were not asked about this time.
|
|
||||||
Left int
|
|
||||||
// Bounded is whether it stopped at its bounds rather than at a refusal.
|
|
||||||
Bounded bool
|
|
||||||
// Stopped says why the sweep stopped before the end; empty when it did not.
|
|
||||||
Stopped string
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweep holds every kept archive, then asks the store to let go of each reference in order, records
|
|
||||||
// what it let go of, and says what it did.
|
|
||||||
func sweep(ctx context.Context, inv *inventory.Inventory, store artifacts.Store, references, kept []string,
|
|
||||||
bounds sweepBounds) sweepResult {
|
|
||||||
within, stop := context.WithTimeout(ctx, bounds.budget)
|
|
||||||
defer stop()
|
|
||||||
var r sweepResult
|
|
||||||
|
|
||||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
|
||||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
|
||||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
|
||||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
|
||||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
|
||||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
|
||||||
// **An index goes with its platform manifests only when a person confirmed it** (novox/hq ADR
|
|
||||||
// 0257), and a kept index keeps its own: which platform manifests the kept indexes name is read
|
|
||||||
// from the store for every repository the sweep will touch, before the first delete. A single
|
|
||||||
// read that fails stops the sweep before it deletes anything, as a kept archive that cannot be
|
|
||||||
// held does. The sweep after a build leaves an eligible index for such a collect, below.
|
|
||||||
store.Spare = nil
|
|
||||||
if bounds.platforms {
|
|
||||||
if inv == nil {
|
|
||||||
r.Stopped = "no records to read which platform manifests a kept index names, so nothing was let go"
|
|
||||||
r.Left = len(references)
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
images, err := inv.KeptImages(ctx)
|
|
||||||
if err == nil {
|
|
||||||
store.Spare, err = store.SpareKeptIndexes(within, images, references)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
r.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err)
|
|
||||||
r.Left = len(references)
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
wrote, missing, err := holdKept(within, store, kept)
|
|
||||||
r.HoldersWritten, r.Missing = wrote, missing
|
|
||||||
if err != nil {
|
|
||||||
r.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: %v", err)
|
|
||||||
r.Left = len(references)
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
r.Held = true
|
|
||||||
|
|
||||||
for i, reference := range references {
|
|
||||||
if i >= bounds.most || within.Err() != nil {
|
|
||||||
r.Left = len(references) - i
|
|
||||||
r.Bounded = true
|
|
||||||
if i >= bounds.most {
|
|
||||||
r.Stopped = fmt.Sprintf("asked about %d, the most one sweep asks", bounds.most)
|
|
||||||
} else {
|
|
||||||
r.Stopped = fmt.Sprintf("ran out of its %s", bounds.budget)
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if !bounds.platforms {
|
|
||||||
// **An index waits for a confirmed collect** (novox/hq ADR 0257 §4). Let go of alone,
|
|
||||||
// its platform manifests would stay in the store and the record would say collected, so
|
|
||||||
// no later sweep would offer it again and they would stay for ever. Left eligible, the
|
|
||||||
// next collect a person confirms takes it with them.
|
|
||||||
index, err := store.IsIndex(within, reference)
|
|
||||||
if err != nil && !errors.Is(err, artifacts.Gone) && !errors.Is(err, artifacts.ErrNotOurs) {
|
|
||||||
r.Stopped = fmt.Sprintf("the artifact store could not say whether %s is an index, so nothing more was asked of it: %v", reference, err)
|
|
||||||
r.Left = len(references) - i
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if index {
|
|
||||||
r.Indexes++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
err := store.LetGo(within, reference)
|
|
||||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
|
||||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
|
||||||
// again for ever.
|
|
||||||
r.LetGo = append(r.LetGo, reference)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
|
||||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
|
||||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
|
||||||
// reason to stop, because the store was never asked. One of these at the front of
|
|
||||||
// the oldest-first order ended every sweep until this.
|
|
||||||
r.Skipped++
|
|
||||||
if r.Skipped == 1 {
|
|
||||||
fmt.Fprintf(os.Stderr, "the sweep will not address %s and went on: %v\n", reference, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
|
||||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
|
||||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
|
||||||
// front of whoever was building something.
|
|
||||||
r.Stopped = fmt.Sprintf("the artifact store kept %s, so nothing more was asked of it: %v", reference, err)
|
|
||||||
r.Left = len(references) - i
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(r.LetGo) > 0 {
|
|
||||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
|
||||||
// the sweep ran out of budget would mean asking about them again for ever.
|
|
||||||
if err := inv.MarkCollected(ctx, r.LetGo); err != nil {
|
|
||||||
r.Stopped = fmt.Sprintf("the store let go of %d artifact(s) and the record of it did not keep: %v",
|
|
||||||
len(r.LetGo), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
|
||||||
// what it let go of — after a build. Says what it did and what it could not; returns nothing,
|
|
||||||
// because nothing upstream should branch on it.
|
|
||||||
func collect(ctx context.Context, inv *inventory.Inventory) {
|
|
||||||
references, err := inv.ToCollect(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
kept, err := inv.KeptArchives(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(references) == 0 && len(kept) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not read the catalogue to find the artifact store: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// As the mesh reaches it from the network. Empty means the store is not on the network — on a
|
|
||||||
// mesh being raised it is not yet, and there the store holds one build of anything and has
|
|
||||||
// nothing to collect.
|
|
||||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
|
||||||
if err != nil || address == "" {
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not find the artifact store to collect from: %v\n", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
r := sweep(ctx, inv, artifacts.Store{Address: address}, references, kept, afterBuild)
|
|
||||||
sayCollected(os.Stderr, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sayCollected says what one sweep after a build did.
|
|
||||||
func sayCollected(w io.Writer, r sweepResult) {
|
|
||||||
if r.HoldersWritten > 0 {
|
|
||||||
fmt.Fprintf(w, "the artifact store now holds %d more kept archive(s) by a manifest\n", r.HoldersWritten)
|
|
||||||
}
|
|
||||||
if r.Missing > 0 {
|
|
||||||
fmt.Fprintf(w, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
|
||||||
"`collection` lists them\n", r.Missing)
|
|
||||||
}
|
|
||||||
if len(r.LetGo) > 0 {
|
|
||||||
// **Each one named** (novox/hq issue 1933): a count could not say which archive a machine's 404 was let go of.
|
|
||||||
fmt.Fprintf(w, "the artifact store let go of %d artifact(s) the mesh no longer keeps: %s\n", len(r.LetGo),
|
|
||||||
strings.Join(r.LetGo, ", "))
|
|
||||||
}
|
|
||||||
if r.Stopped != "" && !r.Bounded {
|
|
||||||
fmt.Fprintln(w, r.Stopped)
|
|
||||||
}
|
|
||||||
if r.Held && r.Left > 0 {
|
|
||||||
fmt.Fprintf(w, "%d more to collect; the next build asks again\n", r.Left)
|
|
||||||
}
|
|
||||||
if r.Skipped > 0 {
|
|
||||||
fmt.Fprintf(w, "%d artifact(s) the sweep will not address were skipped\n", r.Skipped)
|
|
||||||
}
|
|
||||||
if r.Indexes > 0 {
|
|
||||||
fmt.Fprintf(w, "%d eligible index(es) wait for a collect a person confirms, which takes their platforms too\n", r.Indexes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
|
|
||||||
// Answers how many holders it wrote and how many kept archives the store does not have.
|
|
||||||
//
|
|
||||||
// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
|
|
||||||
// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
|
|
||||||
// reference the store cannot be asked about is skipped as the deletion loop skips one
|
|
||||||
// (novox/hq issue 226).
|
|
||||||
func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
|
|
||||||
for _, reference := range kept {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
|
|
||||||
}
|
|
||||||
did, err := store.Hold(ctx, reference)
|
|
||||||
switch {
|
|
||||||
case err == nil:
|
|
||||||
if did {
|
|
||||||
wrote++
|
|
||||||
}
|
|
||||||
case errors.Is(err, artifacts.Gone):
|
|
||||||
missing++
|
|
||||||
case errors.Is(err, artifacts.ErrNotOurs):
|
|
||||||
default:
|
|
||||||
return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return wrote, missing, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
|
|
||||||
// several times a day converges within days of this landing; small enough that no single build
|
|
||||||
// waits on the whole backlog.
|
|
||||||
const mostPerSweep = 200
|
|
||||||
|
|
||||||
// sweepBudget is the longest a sweep will keep a build waiting.
|
|
||||||
const sweepBudget = 60 * time.Second
|
|
||||||
@@ -1,166 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/artifacts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
|
|
||||||
// (novox/hq issue 253, ADR 0189).
|
|
||||||
//
|
|
||||||
// **The question to answer before the store's collector runs for real.** The collector deletes
|
|
||||||
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
|
|
||||||
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
|
|
||||||
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
|
|
||||||
// dry run go.
|
|
||||||
//
|
|
||||||
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
|
|
||||||
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
|
|
||||||
// new verb in the seat's row.
|
|
||||||
|
|
||||||
type collectionReport struct {
|
|
||||||
// Store is the artifact store as this machine reached it; empty when it is not on the network.
|
|
||||||
Store string `json:"store"`
|
|
||||||
// KeptArchives is how many archives the mesh keeps, for either reason.
|
|
||||||
KeptArchives int `json:"kept_archives"`
|
|
||||||
// Held is how many of them the store holds by their manifest.
|
|
||||||
Held int `json:"held"`
|
|
||||||
// Unheld are the kept archives the collector would delete tonight if it ran for real.
|
|
||||||
Unheld []string `json:"unheld"`
|
|
||||||
// Missing are kept archives the store does not have at all.
|
|
||||||
Missing []string `json:"missing"`
|
|
||||||
// Unasked is how many could not be asked about, and why the asking stopped.
|
|
||||||
Unasked int `json:"unasked"`
|
|
||||||
Stopped string `json:"stopped,omitempty"`
|
|
||||||
// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
|
|
||||||
// collected — split by kind.
|
|
||||||
Eligible int `json:"eligible"`
|
|
||||||
EligibleImages int `json:"eligible_images"`
|
|
||||||
EligibleArchives int `json:"eligible_archives"`
|
|
||||||
// SafeToCollect is whether every kept archive was asked about and every one is held.
|
|
||||||
SafeToCollect bool `json:"safe_to_collect"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func collectionCommand(ctx context.Context, args []string) error {
|
|
||||||
set := flag.NewFlagSet("collection", flag.ContinueOnError)
|
|
||||||
asJSON := set.Bool("json", false, "answer as JSON")
|
|
||||||
positionals, err := parseAround(set, args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(positionals) != 0 {
|
|
||||||
return errors.New("collection [--json]")
|
|
||||||
}
|
|
||||||
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
inv := open.inventory
|
|
||||||
|
|
||||||
kept, err := inv.KeptArchives(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
eligible, err := inv.ToCollect(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
|
|
||||||
for _, reference := range eligible {
|
|
||||||
if strings.Contains(reference, "/blobs/") {
|
|
||||||
report.EligibleArchives++
|
|
||||||
} else {
|
|
||||||
report.EligibleImages++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if report.Store == "" {
|
|
||||||
report.Unasked = len(kept)
|
|
||||||
report.Stopped = "this mesh has no artifact store on its network"
|
|
||||||
} else {
|
|
||||||
report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
|
|
||||||
}
|
|
||||||
report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
|
|
||||||
|
|
||||||
if *asJSON {
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
|
||||||
encoder.SetIndent("", " ")
|
|
||||||
return encoder.Encode(report)
|
|
||||||
}
|
|
||||||
printCollection(report)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// askHeld asks the store about each kept archive, stopping at the first answer that is not about
|
|
||||||
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
|
|
||||||
// identical failures says less than one line.
|
|
||||||
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
|
|
||||||
for i, reference := range kept {
|
|
||||||
held, err := store.Held(ctx, reference)
|
|
||||||
switch {
|
|
||||||
case err == nil && held:
|
|
||||||
report.Held++
|
|
||||||
case err == nil:
|
|
||||||
report.Unheld = append(report.Unheld, reference)
|
|
||||||
case errors.Is(err, artifacts.Gone):
|
|
||||||
report.Missing = append(report.Missing, reference)
|
|
||||||
case errors.Is(err, artifacts.ErrNotOurs):
|
|
||||||
// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
|
|
||||||
report.Unasked++
|
|
||||||
default:
|
|
||||||
return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return report.Unasked, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func printCollection(r collectionReport) {
|
|
||||||
store := r.Store
|
|
||||||
if store == "" {
|
|
||||||
store = "(not on the network)"
|
|
||||||
}
|
|
||||||
fmt.Printf("artifact store %s\n", store)
|
|
||||||
fmt.Printf("kept archives %d\n", r.KeptArchives)
|
|
||||||
fmt.Printf(" held %d\n", r.Held)
|
|
||||||
fmt.Printf(" unheld %d\n", len(r.Unheld))
|
|
||||||
fmt.Printf(" missing %d\n", len(r.Missing))
|
|
||||||
if r.Unasked > 0 {
|
|
||||||
fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped)
|
|
||||||
}
|
|
||||||
fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
|
|
||||||
if len(r.Unheld) > 0 {
|
|
||||||
fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
|
|
||||||
for _, reference := range r.Unheld {
|
|
||||||
fmt.Printf(" %s\n", reference)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(r.Missing) > 0 {
|
|
||||||
fmt.Println("\nmissing — kept by the mesh, not in the store:")
|
|
||||||
for _, reference := range r.Missing {
|
|
||||||
fmt.Printf(" %s\n", reference)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Println()
|
|
||||||
if r.SafeToCollect {
|
|
||||||
fmt.Println("every kept archive is held: the store's collector may run for real")
|
|
||||||
} else {
|
|
||||||
fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"go/ast"
|
|
||||||
"go/parser"
|
|
||||||
"go/token"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1389: the controller's own composition is gone. No function of the package is named for it any more, and no function
|
|
||||||
// calls one by that name: a name reappearing is a second copy of what mesh-sdk's go/compose holds (novox/hq ADR 0453).
|
|
||||||
func TestTheControllersOwnCompositionIsGone(t *testing.T) {
|
|
||||||
gone := map[string]bool{"planFor": true, "declarationWith": true, "composedAndValidated": true, "generators": true,
|
|
||||||
"renderingFor": true, "theRestOfTheMesh": true, "whoResolves": true, "grantsFor": true, "namesInTheMesh": true,
|
|
||||||
"onThePrivateNetwork": true, "passSeatsOn": true, "theMeshPassesSeatsOn": true, "assignmentsOf": true,
|
|
||||||
"licencesFor": true, "seatsOn": true, "composeBusUsers": true, "adoptionFor": true, "localNetworksOf": true}
|
|
||||||
files, err := filepath.Glob("*.go")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
fset := token.NewFileSet()
|
|
||||||
var found []string
|
|
||||||
for _, f := range files {
|
|
||||||
if strings.HasSuffix(f, "_test.go") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
file, err := parser.ParseFile(fset, f, nil, 0)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, d := range file.Decls {
|
|
||||||
fn, ok := d.(*ast.FuncDecl)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if gone[fn.Name.Name] {
|
|
||||||
found = append(found, f+": "+fn.Name.Name+" is declared")
|
|
||||||
}
|
|
||||||
if fn.Body == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
|
||||||
if c, ok := n.(*ast.CallExpr); ok {
|
|
||||||
if id, ok := c.Fun.(*ast.Ident); ok && gone[id.Name] {
|
|
||||||
found = append(found, f+": "+fn.Name.Name+" calls "+id.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(found)
|
|
||||||
for _, one := range found {
|
|
||||||
t.Errorf("%s: the controller's own composition is back; compose through mesh-sdk (compose_sdk.go)", one)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,114 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/base64"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
)
|
|
||||||
|
|
||||||
// aCoreInput is an Input of one machine that takes a core module whose credential the vault has not boxed: the composition
|
|
||||||
// stands in a vault box for it, and the machine holds the controller's own seal of it.
|
|
||||||
func aCoreInput(t *testing.T) (*meshInput, string) {
|
|
||||||
t.Helper()
|
|
||||||
raw := make([]byte, 32)
|
|
||||||
raw[0] = 9
|
|
||||||
key := base64.StdEncoding.EncodeToString(raw)
|
|
||||||
client := catalogue.Manifest{Module: "mesh-provisioner-client", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}},
|
|
||||||
OwnSecrets: catalogue.OwnSecrets{broker.ProvisionerSecret: {Path: "/var/lib/mesh-provisioner-client/broker"}}}
|
|
||||||
return &meshInput{Input: compose.Input{
|
|
||||||
Catalogue: map[string]catalogue.Manifest{client.Module: client},
|
|
||||||
Nodes: []compose.Node{{Name: "laptop", Assigned: []string{client.Module}, SealingKey: key, Stated: true,
|
|
||||||
Contract: compose.VaultBoxContract}},
|
|
||||||
OwnSecrets: []compose.OwnSecretRow{{Node: "laptop", Module: client.Module, Name: broker.ProvisionerSecret,
|
|
||||||
Origin: "made", SealedTo: key, Sealed: "SEALED-BY-THE-CONTROLLER"}},
|
|
||||||
}}, client.Module
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1385 (the lead's answer 1): where the vault is silent and no box is kept, the declaration carries the controller's own seal
|
|
||||||
// for the core credential, as ADR 0396 has it, and stands in for nothing. It failed first: the composition's declaration
|
|
||||||
// carried the stand-in box, and a send was refused.
|
|
||||||
func TestACoreCredentialTheVaultDidNotBoxIsSealedByTheController(t *testing.T) {
|
|
||||||
in, module := aCoreInput(t)
|
|
||||||
d, _, err := compose.ComposeIn(in.Input, "laptop", compose.Foreseeing)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(d.CoreBoxesWanted) != 1 || len(stillForeseen(d)) == 0 {
|
|
||||||
t.Fatalf("the composition does not stand in for the vault's box: %v", d.CoreBoxesWanted)
|
|
||||||
}
|
|
||||||
sealed, _, err := sealedByTheController(in, "laptop", d, []string{module})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if left := stillForeseen(sealed); len(left) != 0 {
|
|
||||||
t.Fatalf("the declaration still stands in for %v", left)
|
|
||||||
}
|
|
||||||
var file map[string]any
|
|
||||||
for _, r := range sealed.Resources {
|
|
||||||
if r["id"] == module+"."+catalogue.NeedID(broker.ProvisionerSecret) {
|
|
||||||
file = r
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if file == nil || file["sealed"] != "SEALED-BY-THE-CONTROLLER" || file[catalogue.VaultSealedField] != nil {
|
|
||||||
t.Fatalf("the core credential is carried as %v: the controller's own seal, not the vault's box", file)
|
|
||||||
}
|
|
||||||
// And a module the vault did answer for is left as it is.
|
|
||||||
other, _, err := sealedByTheController(in, "laptop", d, []string{"another-module"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(stillForeseen(other)) == 0 {
|
|
||||||
t.Fatal("a module the vault answered for was sealed by the controller")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1383 (the lead's answer 4): the machine that runs the bus is refused where no user has a credential, with busUserList's
|
|
||||||
// words, and a machine that does not run the bus is not. It failed first: the Input stood a list in and the bus node composed.
|
|
||||||
func TestTheMachineThatRunsTheBusIsRefusedWhereNoUserHasACredential(t *testing.T) {
|
|
||||||
bus := catalogue.Manifest{Module: "nats", Version: "1", BusUsers: "/etc/nats/users.conf",
|
|
||||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}
|
|
||||||
other := catalogue.Manifest{Module: "files", Version: "1"}
|
|
||||||
why := errors.New("this machine runs the bus and not one user has a credential, so the composed list would refuse every connection in the mesh")
|
|
||||||
in := &meshInput{busRefused: why, Input: compose.Input{
|
|
||||||
Catalogue: map[string]catalogue.Manifest{"nats": bus, "files": other},
|
|
||||||
Nodes: []compose.Node{{Name: "anchor", Assigned: []string{"nats"}}, {Name: "laptop", Assigned: []string{"files"}}},
|
|
||||||
}}
|
|
||||||
if err := refusedBus(in, "anchor"); err == nil || !strings.Contains(err.Error(), "not one user has a credential") {
|
|
||||||
t.Fatalf("the machine that runs the bus is not refused: %v", err)
|
|
||||||
}
|
|
||||||
if err := refusedBus(in, "laptop"); err != nil {
|
|
||||||
t.Fatalf("a machine that does not run the bus is refused: %v", err)
|
|
||||||
}
|
|
||||||
if err := refusedBus(&meshInput{Input: in.Input}, "anchor"); err != nil {
|
|
||||||
t.Fatalf("a bus with users is refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The controller's seal replaces the stand-in box whatever path the resource carries: the path is the manifest's with the
|
|
||||||
// machine's words filled in, not the manifest's own. It failed first, with the path compared (the live try: "composed twice and
|
|
||||||
// still names ... asked of the vault by the next send").
|
|
||||||
func TestTheControllersSealReplacesTheBoxWhateverPathTheFileCarries(t *testing.T) {
|
|
||||||
in, module := aCoreInput(t)
|
|
||||||
d, _, err := compose.ComposeIn(in.Input, "laptop", compose.Foreseeing)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, r := range d.Resources {
|
|
||||||
if _, box := r[catalogue.VaultSealedField]; box {
|
|
||||||
r["path"] = "/home/someone/.var/lib/mesh-provisioner-client/broker"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sealed, _, err := sealedByTheController(in, "laptop", d, []string{module})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if left := stillForeseen(sealed); len(left) != 0 {
|
|
||||||
t.Fatalf("the declaration still stands in for %v", left)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
)
|
|
||||||
|
|
||||||
// grantsOf is the grants a provider creates for its consumers, the consumers whose identity overflows the provision's bound
|
|
||||||
// and the credentials bound to another consumer, as mesh-sdk's rendering gives them for the machine (it replaced the
|
|
||||||
// controller's grantsFor, issue 1389).
|
|
||||||
func grantsOf(ctx context.Context, open *stores, node string) ([]catalogue.Grant, []catalogue.Overflow, []catalogue.Unbound, error) {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
rendered, err := compose.Render(in.Input, node, compose.Reading)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
return rendered.Rendering.Grants, rendered.Withheld, rendered.Unbound, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// madeAndRendered sends to the machine (so everything a send makes is made) and answers its rendering as a question reads it
|
|
||||||
// afterwards: what the controller's Allocating rendering was.
|
|
||||||
func madeAndRendered(ctx context.Context, open *stores, node string) (catalogue.Rendering, error) {
|
|
||||||
if _, err := composeSend(ctx, open, node); err != nil {
|
|
||||||
return catalogue.Rendering{}, err
|
|
||||||
}
|
|
||||||
forgetInput(ctx)
|
|
||||||
return renderingOf(ctx, open, node)
|
|
||||||
}
|
|
||||||
@@ -1,461 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The controller's adapter from its stores to mesh-sdk's composition input (novox/hq ADR 0453 clause 3, design 47 R4b,
|
|
||||||
// issues 1382 and 1383).
|
|
||||||
//
|
|
||||||
// **The adapter reads, and perform writes.** readComposeInput asks the stores once for everything the composition reads
|
|
||||||
// and returns compose.Input, which the composition never changes; it makes, seals, allocates and records nothing. A
|
|
||||||
// pair credential, a port, an own secret or a core box that the next send would make is not made here: the composition
|
|
||||||
// names it foreseen, and compose_perform.go makes it for a send, then reads the stores again and composes again. A
|
|
||||||
// question (status, facts, probes, the merge gate's reading) never performs.
|
|
||||||
//
|
|
||||||
// **The Input is built once per operation.** A verb that composes every node reads the stores once and hands the Input to
|
|
||||||
// each node's composition of it.
|
|
||||||
|
|
||||||
// composeInputBuilt is called once for each Input the adapter builds: the seam a test counts the reads of an operation by
|
|
||||||
// (issue 1382, criterion 5).
|
|
||||||
var composeInputBuilt = func() {}
|
|
||||||
|
|
||||||
// readComposeInput reads the stores once and returns what the composition needs. ctx carries what a send keeps (the
|
|
||||||
// recorded modules kept at the build their machine runs, recorded_kept.go): the same contexts every send composes under.
|
|
||||||
func readComposeInput(ctx context.Context, open *stores) (*meshInput, error) {
|
|
||||||
in, shape, err := readMeshAndNodes(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// What each node resolves to, with no credential given (the settings are read before it): the modules whose settings are read, and the
|
|
||||||
// needs whose pair credentials are asked of the stores. Nothing is written by it.
|
|
||||||
unread, err := readSettings(ctx, open, &in)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
plans := map[string]compose.Resolved{}
|
|
||||||
for _, n := range in.Nodes {
|
|
||||||
if shape.refused[n.Name] != nil {
|
|
||||||
continue // refused for what it keeps: nothing of it is read further
|
|
||||||
}
|
|
||||||
resolved, err := compose.Resolve(in, n.Name)
|
|
||||||
switch {
|
|
||||||
case compose.IsUnresolvable(err):
|
|
||||||
// Their set does not compose: nothing of theirs is read further, as the controller never read a thing for a
|
|
||||||
// node that did not resolve.
|
|
||||||
continue
|
|
||||||
case err != nil:
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
plans[n.Name] = resolved
|
|
||||||
}
|
|
||||||
if err := unreadSettings(unread, plans); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var busRefused error
|
|
||||||
wouldMake, err := readCredentialsAndBuilds(ctx, open, &in, plans, shape, &busRefused)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
composeInputBuilt()
|
|
||||||
return &meshInput{Input: in, wouldMake: wouldMake, busRefused: busRefused, refused: shape.refused}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// meshInput is the Input an operation composes over, and the pair credentials its reading found none held for: the ones the
|
|
||||||
// next send makes, which perform makes.
|
|
||||||
type meshInput struct {
|
|
||||||
compose.Input
|
|
||||||
wouldMake []inventory.PairCredential
|
|
||||||
// busRefused is why the bus's user list cannot be composed (no user has a credential): the machine that runs the bus is
|
|
||||||
// refused with it, as the controller's own composition refused it, and no other machine is.
|
|
||||||
busRefused error
|
|
||||||
// refused is, by node, why that machine alone cannot be composed (nodeRefusal).
|
|
||||||
refused map[string]error
|
|
||||||
}
|
|
||||||
|
|
||||||
// nodeOf is the node of that name in an Input.
|
|
||||||
func nodeOf(in compose.Input, name string) (compose.Node, bool) {
|
|
||||||
for _, n := range in.Nodes {
|
|
||||||
if n.Name == name {
|
|
||||||
return n, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return compose.Node{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// inputShape is what the first half of the reading found that the second half needs.
|
|
||||||
type inputShape struct {
|
|
||||||
sent map[string]map[string]string // by node: module to the commit last sent
|
|
||||||
kept map[string]map[string]string // by node: module to the commit kept
|
|
||||||
current map[string]inventory.CurrentBuild
|
|
||||||
// refused is, by node, why a recorded module cannot be kept at the build the machine runs (keepRecorded's refusal, in
|
|
||||||
// its words): that machine alone is refused with it, as planFor refused it, and nothing else of the mesh is.
|
|
||||||
refused map[string]error
|
|
||||||
}
|
|
||||||
|
|
||||||
// readMeshAndNodes is the mesh-wide and per-node half of the Input (issue 1382).
|
|
||||||
func readMeshAndNodes(ctx context.Context, open *stores) (compose.Input, inputShape, error) {
|
|
||||||
inv := open.inventory
|
|
||||||
var in compose.Input
|
|
||||||
shape := inputShape{sent: map[string]map[string]string{}, kept: map[string]map[string]string{}, refused: map[string]error{}}
|
|
||||||
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
in.Catalogue = shelf
|
|
||||||
if in.Holdings, err = inv.Holdings(ctx); err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
if in.Licences, err = readLicences(ctx, open); err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
places, err := inv.Overlays(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
placed := map[string]inventory.Overlay{}
|
|
||||||
for _, p := range places {
|
|
||||||
placed[p.Name] = p
|
|
||||||
}
|
|
||||||
if shape.current, err = inv.CurrentBuilds(ctx); err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// The range the mesh allocates from and what its internal names end in, read from the environment here and nowhere else.
|
|
||||||
in.OverlayCIDR = strings.TrimSpace(os.Getenv(OverlayCIDRVar))
|
|
||||||
in.InternalSuffix = overlay.Suffix()
|
|
||||||
if in.Tunnels, err = readTunnels(ctx, inv); err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
if in.Adopted, err = readAdopted(ctx, inv); err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
carried, err := inv.CarriedPeers(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
for _, c := range carried {
|
|
||||||
in.CarriedPeers = append(in.CarriedPeers, compose.CarriedPeer{
|
|
||||||
PublicKey: c.PublicKey, Address: c.Address, EnrolledAs: c.EnrolledAs, Named: c.Named})
|
|
||||||
}
|
|
||||||
if in.Joining, err = inv.NodesWithALiveToken(ctx); err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
|
|
||||||
keepsSecrets := false
|
|
||||||
for _, m := range shelf {
|
|
||||||
keepsSecrets = keepsSecrets || m.Keeps != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, n := range nodes {
|
|
||||||
var refusal error
|
|
||||||
node, err := readNode(ctx, open, n, placed[n.Name], &refusal)
|
|
||||||
if err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
if refusal != nil {
|
|
||||||
shape.refused[n.Name] = refusal
|
|
||||||
}
|
|
||||||
shape.sent[n.Name], shape.kept[n.Name] = node.SentBuilds, node.KeptBuilds
|
|
||||||
in.Nodes = append(in.Nodes, node)
|
|
||||||
}
|
|
||||||
if keepsSecrets {
|
|
||||||
if in.OperatorExport, err = inv.OperatorExport(ctx); err != nil {
|
|
||||||
return in, shape, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return in, shape, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readNode is one node of the Input: everything the composition reads about it, in the order the controller read it.
|
|
||||||
func readNode(ctx context.Context, open *stores, n inventory.Node, place inventory.Overlay,
|
|
||||||
refused *error) (compose.Node, error) {
|
|
||||||
inv := open.inventory
|
|
||||||
out := compose.Node{
|
|
||||||
Name: n.Name,
|
|
||||||
Place: compose.Place{Key: place.Key, Endpoint: place.Endpoint, Site: place.Site, Hub: place.Hub,
|
|
||||||
Address: place.Address},
|
|
||||||
Account: n.Account, AccountHome: n.AccountHome, AgentAccount: n.AgentAccount, AgentAccountHome: n.AgentAccountHome,
|
|
||||||
Adopted: n.Adopted,
|
|
||||||
}
|
|
||||||
var err error
|
|
||||||
if out.Assigned, err = inv.Assigned(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.Capabilities, err = inv.ProfileOf(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.PublicDomain, err = inv.PublicDomainOf(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.Pins, err = inv.PinsFor(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.Bound, err = inv.BindingsFor(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
given, err := inv.PortsFor(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
for _, a := range given {
|
|
||||||
if out.Ports == nil {
|
|
||||||
out.Ports = map[string]map[int]int{}
|
|
||||||
}
|
|
||||||
if out.Ports[a.Module] == nil {
|
|
||||||
out.Ports[a.Module] = map[int]int{}
|
|
||||||
}
|
|
||||||
out.Ports[a.Module][a.Wanted] = a.Machine
|
|
||||||
}
|
|
||||||
if out.SealingKey, err = inv.SealingKeyOf(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if n.Adopted {
|
|
||||||
if out.Taken, err = inv.Taken(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if out.OutwardLinks, err = inv.OutwardLinksOf(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
account, had, err := inv.MachineOf(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if had && account.Machine != nil {
|
|
||||||
out.NetworksSaid = true
|
|
||||||
for _, a := range account.Machine.Networks {
|
|
||||||
out.Networks = append(out.Networks, catalogue.AttachedNetwork{Link: a.Link, Network: a.Network})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if out.Sequence, err = inv.Sequence(ctx, n.ID); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.Epoch, err = inv.SentEpoch(ctx, n.ID); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.Generation, err = inv.SentGeneration(ctx, n.ID); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.ReadsModules, err = inv.ReadsModules(ctx, n.ID); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
runs, err := inv.AssignmentRuns(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if len(runs) > 0 {
|
|
||||||
out.Runs = make(map[string]compose.Run, len(runs))
|
|
||||||
for module, ref := range runs {
|
|
||||||
out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A recorded module kept at the build this machine runs, in the context a send composes under (recorded_kept.go): its
|
|
||||||
// manifest and the commit, applied to this node's own resolution only.
|
|
||||||
// What cannot be kept refuses this machine alone (keepRecorded's refusal, as planFor returned it).
|
|
||||||
kept, err := recordedKept(ctx, open, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
*refused = err
|
|
||||||
} else if len(kept) > 0 {
|
|
||||||
out.KeptBuilds = kept
|
|
||||||
if out.Kept, err = keptManifests(ctx, open, n.Name, kept); err != nil {
|
|
||||||
*refused = err
|
|
||||||
out.KeptBuilds, out.Kept = nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stated, has, err := inv.HealthOf(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
out.Stated = has
|
|
||||||
if has {
|
|
||||||
out.Contract = stated.Contract
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptManifests is the manifest of each module kept at the build a machine runs: keepRecorded's reads, and its refusals in its
|
|
||||||
// words, without putting any of it into a shelf (that is the node's own Kept, applied by the composition).
|
|
||||||
func keptManifests(ctx context.Context, open *stores, node string, kept map[string]string) (map[string]catalogue.Manifest, error) {
|
|
||||||
// applyKept puts each manifest into the shelf it is given and refuses a build that cannot be read; the shelf here is a
|
|
||||||
// scratch one, whose entries are the kept manifests.
|
|
||||||
scratch := map[string]catalogue.Manifest{}
|
|
||||||
if err := applyKept(ctx, open, node, kept, scratch); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return scratch, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func readTunnels(ctx context.Context, inv *inventory.Inventory) (map[string]compose.FoundTunnel, error) {
|
|
||||||
found, err := inv.Tunnels(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(found) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
out := make(map[string]compose.FoundTunnel, len(found))
|
|
||||||
for name, t := range found {
|
|
||||||
out[name] = compose.FoundTunnel{Tunnel: tunnelOf(t.Tunnel), NodeAdopted: t.NodeAdopted}
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func readAdopted(ctx context.Context, inv *inventory.Inventory) (*compose.AdoptedTunnel, error) {
|
|
||||||
tunnel, hub, adopted, err := inv.AdoptedTunnel(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if hub == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return &compose.AdoptedTunnel{Tunnel: tunnelOf(tunnel), Hub: hub, Adopted: adopted}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func tunnelOf(t inventory.Tunnel) compose.Tunnel {
|
|
||||||
out := compose.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, MTU: t.MTU,
|
|
||||||
Address: t.Address, Range: t.Range, PublicKey: t.PublicKey}
|
|
||||||
for _, p := range t.Peers {
|
|
||||||
out.Peers = append(out.Peers, compose.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// readLicences is every licence the mesh holds, with who uses it and the key sealed to each holder: given even where empty,
|
|
||||||
// so a holder the mesh holds no key for composes as that, and not as a key that was not asked for.
|
|
||||||
func readLicences(ctx context.Context, open *stores) ([]compose.Licence, error) {
|
|
||||||
held, err := open.Licences(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
all, err := held.All(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []compose.Licence
|
|
||||||
for _, one := range all {
|
|
||||||
l := compose.Licence{Name: one.Name, Serves: one.Serves, Keys: map[compose.Holder]string{}}
|
|
||||||
holders, err := held.HoldersOf(ctx, one.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, h := range holders {
|
|
||||||
holder := compose.Holder{Node: h.Node, Module: h.Module}
|
|
||||||
l.Holders = append(l.Holders, holder)
|
|
||||||
key, err := held.KeyFor(ctx, one.Name, h.Node, h.Module)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
l.Keys[holder] = key
|
|
||||||
}
|
|
||||||
node, module, err := held.ManagerOf(ctx, one.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
l.Manager = compose.Holder{Node: node, Module: module}
|
|
||||||
out = append(out, l)
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readSettings is each node's settings layers for every module that has any (a row, or config in mesh-config), read BEFORE any node is resolved: the first pass
|
|
||||||
// resolves a node whose set takes a provider's served facts from the node that runs it, with that node's settings on them
|
|
||||||
// (servedOnNode), so a mesh whose provider says ${setting:domain} resolves only with the settings in the Input. A module's
|
|
||||||
// settings that fail to read are returned in unread, by node and module, and are an error only for a module a node resolves to
|
|
||||||
// (unreadSettings): the controller read none for a module nothing runs.
|
|
||||||
func readSettings(ctx context.Context, open *stores, in *compose.Input) (unread map[string]map[string]error, err error) {
|
|
||||||
have, err := open.inventory.ModulesWithSettings(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
names, err := modulesToRead(in.Catalogue, have, func(module string) (bool, error) {
|
|
||||||
return open.inventory.ReadsConfig(ctx, module)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for i := range in.Nodes {
|
|
||||||
n := &in.Nodes[i]
|
|
||||||
for _, module := range names {
|
|
||||||
layers, err := open.inventory.SettingsFor(ctx, n.Name, module)
|
|
||||||
if err != nil {
|
|
||||||
if unread == nil {
|
|
||||||
unread = map[string]map[string]error{}
|
|
||||||
}
|
|
||||||
if unread[n.Name] == nil {
|
|
||||||
unread[n.Name] = map[string]error{}
|
|
||||||
}
|
|
||||||
unread[n.Name][module] = err
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if len(layers) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if n.Settings == nil {
|
|
||||||
n.Settings = map[string][]catalogue.Layer{}
|
|
||||||
}
|
|
||||||
n.Settings[module] = layers
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return unread, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// modulesToRead is the modules on the shelf whose settings are read for a node: those with a settings row, and those in
|
|
||||||
// the config form whose values mesh-config holds (readsConfig), which have no row when they were never in the settings
|
|
||||||
// form (issue 1493: brother's scanner and printer, set only in mesh-config, composed empty). SettingsFor is where
|
|
||||||
// mesh-config's values are laid in, so a gate on the settings table alone leaves them out.
|
|
||||||
func modulesToRead(shelf map[string]catalogue.Manifest, have map[string]bool, readsConfig func(string) (bool, error)) ([]string, error) {
|
|
||||||
names := make([]string, 0, len(have))
|
|
||||||
for module, m := range shelf {
|
|
||||||
if have[module] {
|
|
||||||
names = append(names, module)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !m.DeclaresConfig() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
reads, err := readsConfig(module)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if reads {
|
|
||||||
names = append(names, module)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// unreadSettings is the error of a module's settings that did not read, for a module the node resolves to.
|
|
||||||
func unreadSettings(unread map[string]map[string]error, plans map[string]compose.Resolved) error {
|
|
||||||
names := make([]string, 0, len(plans))
|
|
||||||
for node := range plans {
|
|
||||||
names = append(names, node)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
for _, node := range names {
|
|
||||||
for _, m := range plans[node].Plan.Modules {
|
|
||||||
if err := unread[node][m.Module]; err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,285 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The second half of the adapter (issue 1383): the credential and build fields of the Input, read and never made.
|
|
||||||
|
|
||||||
// readCredentialsAndBuilds fills what the first half left: the credentials, the own secrets, the vault's boxes, the bus,
|
|
||||||
// each node's serving key and certificate, and the builds.
|
|
||||||
func readCredentialsAndBuilds(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved,
|
|
||||||
shape inputShape, busRefused *error) ([]inventory.PairCredential, error) {
|
|
||||||
if err := readIdentity(ctx, open, in); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
wouldMake, err := readCredentials(ctx, open, in, plans, busRefused)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return wouldMake, readBuilds(ctx, open, in, plans, shape)
|
|
||||||
}
|
|
||||||
|
|
||||||
// readIdentity is the mesh's authority and each node's serving key and the certificate kept for it, for a mesh in which a
|
|
||||||
// module asks for a certificate. It establishes and issues nothing: a certificate not kept yet is composed with a stand-in, and
|
|
||||||
// issued by the send (certificateFor).
|
|
||||||
func readIdentity(ctx context.Context, open *stores, in *compose.Input) error {
|
|
||||||
needs := false
|
|
||||||
for _, m := range in.Catalogue {
|
|
||||||
needs = needs || m.Certificate != nil
|
|
||||||
}
|
|
||||||
if !needs {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
ident, err := open.Identity(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if certificate, known, err := ident.KnownAuthority(ctx); err != nil {
|
|
||||||
return err
|
|
||||||
} else if known {
|
|
||||||
in.Authority = certificate
|
|
||||||
}
|
|
||||||
nodes, err := open.inventory.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ids := map[string]string{}
|
|
||||||
for _, n := range nodes {
|
|
||||||
ids[n.Name] = n.ID
|
|
||||||
}
|
|
||||||
for i := range in.Nodes {
|
|
||||||
n := &in.Nodes[i]
|
|
||||||
if n.ServingKey, err = ident.ServingKeyOf(ctx, ids[n.Name]); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if n.ServingKey == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if certificate, known, err := ident.KeptCertificate(ctx, overlay.InternalName(n.Name), n.ServingKey); err != nil {
|
|
||||||
return err
|
|
||||||
} else if known {
|
|
||||||
n.Certificate = certificate
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readCredentials is how each pair credential a node takes from elsewhere stands, the credentials made for each provider's
|
|
||||||
// consumers, the own secrets, the vault's boxes and the bus. It reads and never makes: a credential a send would make is
|
|
||||||
// left out (the composition's would-make), a credential only a person gives and nobody gave is not-given, and one the send
|
|
||||||
// would be refused is refused with the controller's words.
|
|
||||||
func readCredentials(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved,
|
|
||||||
busRefused *error) ([]inventory.PairCredential, error) {
|
|
||||||
var wouldMake []inventory.PairCredential
|
|
||||||
inv := open.inventory
|
|
||||||
shelf := in.Catalogue
|
|
||||||
nodeOf := map[string]compose.Node{}
|
|
||||||
for _, n := range in.Nodes {
|
|
||||||
nodeOf[n.Name] = n
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(plans))
|
|
||||||
for name := range plans {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
for _, name := range names {
|
|
||||||
resolved := plans[name]
|
|
||||||
nodeShelf := shelfWith(shelf, nodeOf[name].Kept)
|
|
||||||
for _, need := range resolved.Plan.Needs {
|
|
||||||
if need.ByRecord {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
p := inventory.PairCredential{Name: need.Name, Consumer: name, ConsumerModule: need.For,
|
|
||||||
Provider: need.From, Local: need.Local}
|
|
||||||
if module, outside := issuedOutsideBy(nodeShelf, need); outside {
|
|
||||||
p.Kind, p.ProviderModule = inventory.PairOutside, module
|
|
||||||
} else if need.SharedOwn != "" {
|
|
||||||
p.Kind, p.Own = inventory.PairShared, need.SharedOwn
|
|
||||||
p.ProviderModule = sharedProviderModule(resolved.Plan, shelf, need)
|
|
||||||
}
|
|
||||||
state, err := inv.PairCredentialState(ctx, p)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
c := compose.Credential{Provision: need.Name, Consumer: name, ConsumerModule: need.For,
|
|
||||||
Provider: need.From, Local: need.Local}
|
|
||||||
switch state.State {
|
|
||||||
case inventory.PairHeld:
|
|
||||||
c.State, c.Sealed = compose.CredentialHeld, state.Sealed
|
|
||||||
case inventory.PairNotGiven:
|
|
||||||
c.State, c.Why = compose.CredentialNotGiven, state.Why
|
|
||||||
case inventory.PairRefused:
|
|
||||||
c.State, c.Why = compose.CredentialRefused, state.Why
|
|
||||||
default:
|
|
||||||
// Left out: a need with no entry is the credential the next send makes, and perform makes it.
|
|
||||||
wouldMake = append(wouldMake, p)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
in.Credentials = append(in.Credentials, c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The credentials made for each provider's consumers, so the grants it must create are read from them.
|
|
||||||
for _, n := range in.Nodes {
|
|
||||||
rows, err := inv.SecretsFrom(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, s := range rows {
|
|
||||||
in.Issued = append(in.Issued, compose.Issued{Provision: s.Name, Consumer: s.Consumer,
|
|
||||||
ConsumerModule: s.ConsumerModule, Local: s.Local, Provider: n.Name, ForProvider: s.ForProvider})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
own, err := inv.OwnSecretRows(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, r := range own {
|
|
||||||
in.OwnSecrets = append(in.OwnSecrets, compose.OwnSecretRow{Node: r.Node, Module: r.Module, Name: r.Name,
|
|
||||||
Origin: r.Origin, SealedTo: r.SealedTo, Sealed: r.Sealed})
|
|
||||||
}
|
|
||||||
for _, n := range in.Nodes {
|
|
||||||
boxes, err := inv.KeptCoreBoxes(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for module, byName := range boxes {
|
|
||||||
for name, b := range byName {
|
|
||||||
in.KeptCoreBoxes = append(in.KeptCoreBoxes, compose.CoreBox{Node: n.Name, Module: module, Name: name,
|
|
||||||
Generation: b.Generation, SealedTo: b.SealedTo, Box: b.Box})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Slice(in.KeptCoreBoxes, func(i, j int) bool {
|
|
||||||
a, b := in.KeptCoreBoxes[i], in.KeptCoreBoxes[j]
|
|
||||||
if a.Node != b.Node {
|
|
||||||
return a.Node < b.Node
|
|
||||||
}
|
|
||||||
if a.Module != b.Module {
|
|
||||||
return a.Module < b.Module
|
|
||||||
}
|
|
||||||
return a.Name < b.Name
|
|
||||||
})
|
|
||||||
|
|
||||||
// The bus: its user list as the controller composes it, for the machine that runs it, and who the mesh has minted a
|
|
||||||
// credential for.
|
|
||||||
var bus []catalogue.Manifest
|
|
||||||
for _, m := range shelf {
|
|
||||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
|
||||||
bus = append(bus, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(bus) > 0 {
|
|
||||||
list, missing, err := busUserList(ctx, inv, bus)
|
|
||||||
if err != nil && !strings.Contains(err.Error(), "not one user has a credential") {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// The machine that runs the bus is refused, as the controller's own composition refused it: a list that
|
|
||||||
// refuses every connection in the mesh is never composed (refusedBus); the other machines compose.
|
|
||||||
*busRefused = err
|
|
||||||
}
|
|
||||||
in.BusUsers, in.BusMissing = list, missing
|
|
||||||
}
|
|
||||||
if in.BusMinted, err = inv.BusMinted(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if in.BusMemberships, err = inv.BusMemberships(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return wouldMake, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readBuilds is the build half of the Input: every repository the mesh built, each module's current build, and the build made
|
|
||||||
// from a commit of a module for the machines whose node-engine reads them.
|
|
||||||
func readBuilds(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved, shape inputShape) error {
|
|
||||||
inv := open.inventory
|
|
||||||
held, err := inv.Held(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for repository := range held {
|
|
||||||
in.Built = append(in.Built, repository)
|
|
||||||
}
|
|
||||||
sort.Strings(in.Built)
|
|
||||||
in.CurrentBuilds = make(map[string]string, len(shape.current))
|
|
||||||
for module, b := range shape.current {
|
|
||||||
in.CurrentBuilds[module] = b.Commit
|
|
||||||
}
|
|
||||||
// The build made from each commit a declaration may carry, for the machines that read a build per module.
|
|
||||||
for _, n := range in.Nodes {
|
|
||||||
if !n.ReadsModules {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
resolved, ok := plans[n.Name]
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, m := range resolved.Plan.Modules {
|
|
||||||
for _, commit := range []string{shape.current[m.Module].Commit, shape.kept[n.Name][m.Module], shape.sent[n.Name][m.Module]} {
|
|
||||||
if commit == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, done := in.BuildIDs[m.Module][commit]; done {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
build, err := inv.BuildOf(ctx, m.Module, commit)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if in.BuildIDs == nil {
|
|
||||||
in.BuildIDs = map[string]map[string]string{}
|
|
||||||
}
|
|
||||||
if in.BuildIDs[m.Module] == nil {
|
|
||||||
in.BuildIDs[m.Module] = map[string]string{}
|
|
||||||
}
|
|
||||||
in.BuildIDs[m.Module][commit] = build
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// shelfWith is the shelf with each module a node keeps at the build it runs put in place of the current one: a copy.
|
|
||||||
func shelfWith(shelf map[string]catalogue.Manifest, kept map[string]catalogue.Manifest) map[string]catalogue.Manifest {
|
|
||||||
if len(kept) == 0 {
|
|
||||||
return shelf
|
|
||||||
}
|
|
||||||
out := make(map[string]catalogue.Manifest, len(shelf)+len(kept))
|
|
||||||
for name, m := range shelf {
|
|
||||||
out[name] = m
|
|
||||||
}
|
|
||||||
for name, m := range kept {
|
|
||||||
out[name] = m
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// sharedProviderModule is which module answers a shared credential on the providing node: the one in the plan, else the first
|
|
||||||
// the catalogue names (providerModuleOf, in an order that does not move between calls).
|
|
||||||
func sharedProviderModule(plan catalogue.Resolution, shelf map[string]catalogue.Manifest, n catalogue.Needed) string {
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
if _, shared := m.SharedCredentialOf(n.Name); shared {
|
|
||||||
return m.Module
|
|
||||||
}
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(shelf))
|
|
||||||
for name := range shelf {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
for _, name := range names {
|
|
||||||
if _, shared := shelf[name].SharedCredentialOf(n.Name); shared {
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
@@ -1,354 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// tableState is every table's rows and every sequence's last value, as the adapter's reads must leave them.
|
|
||||||
func tableState(t *testing.T, inv *inventory.Inventory) map[string]string {
|
|
||||||
t.Helper()
|
|
||||||
out := map[string]string{}
|
|
||||||
for _, row := range inv.QueryForTest(t, `select table_name from information_schema.tables
|
|
||||||
where table_schema = 'public' and table_type = 'BASE TABLE' order by table_name`) {
|
|
||||||
name := strings.Trim(row[0], `"`)
|
|
||||||
count := inv.QueryForTest(t, fmt.Sprintf(`select count(*)::text from %q`, name))
|
|
||||||
out["rows:"+name] = fmt.Sprint(count)
|
|
||||||
digest := inv.QueryForTest(t, fmt.Sprintf(`select coalesce(md5(string_agg(t::text, '|' order by t::text)), '') from %q t`, name))
|
|
||||||
out["content:"+name] = fmt.Sprint(digest)
|
|
||||||
}
|
|
||||||
for _, row := range inv.QueryForTest(t, `select sequencename, coalesce(last_value::text, '') from pg_sequences
|
|
||||||
where schemaname = 'public' order by sequencename`) {
|
|
||||||
out["sequence:"+row[0]] = row[1]
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1382/AC4 and 1383/AC6: building the Input writes nothing. The store's rows, their contents and its sequences are the same
|
|
||||||
// before and after, on a mesh whose credentials are not made, whose ports are not given and which keeps a recorded build.
|
|
||||||
func TestBuildingTheInputWritesNothing(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
// A consumer whose credential nothing has made yet, on a mesh that already composed.
|
|
||||||
register(t, open, catalogue.Manifest{Module: "reports", Version: "1", Requires: []string{"s3-bucket"},
|
|
||||||
Resources: []map[string]any{{"id": "config", "type": "file", "path": "/etc/reports/config.json", "mode": "0600",
|
|
||||||
"content": "{}", "merge": "json"}}})
|
|
||||||
if _, err := inv.Assign(ctx, "laptop", "reports"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
before := tableState(t, inv)
|
|
||||||
if _, err := readComposeInput(ctx, open); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
after := tableState(t, inv)
|
|
||||||
var moved []string
|
|
||||||
for k, v := range after {
|
|
||||||
if before[k] != v {
|
|
||||||
moved = append(moved, k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(moved)
|
|
||||||
if len(moved) > 0 {
|
|
||||||
t.Fatalf("building the Input wrote: %v", moved)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// a consumer of the s3-bucket aMeshWithSecrets' provider offers, with a file the credential goes into.
|
|
||||||
func aConsumerOfTheBucket(name string) catalogue.Manifest {
|
|
||||||
return catalogue.Manifest{Module: name, Version: "1", Requires: []string{"s3-bucket"},
|
|
||||||
Secrets: map[string]string{"s3-bucket": "/etc/" + name + "/s3"},
|
|
||||||
Resources: []map[string]any{{"id": "config", "type": "file", "path": "/etc/" + name + "/config.json", "mode": "0600",
|
|
||||||
"content": "{}", "merge": "json"}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1383/AC1: of a provider's three consumers, two have their credentials made and one has not: Issued holds the two rows, with
|
|
||||||
// the copy sealed to the provider, the third is a credential the next send makes, and no row was inserted for it (planFor
|
|
||||||
// inserted it, plan.go: SecretFor in every mode).
|
|
||||||
func TestAPairCredentialNoRowHoldsIsAWouldMakeAndNoRowIsInserted(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
register(t, open, aConsumerOfTheBucket("reports"))
|
|
||||||
if _, err := inv.Assign(ctx, "laptop", "reports"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// A send makes the credentials of files and reports; a question makes none.
|
|
||||||
if _, err := composeSend(ctx, open, "laptop"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
register(t, open, aConsumerOfTheBucket("metrics"))
|
|
||||||
if _, err := inv.Assign(ctx, "laptop", "metrics"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
before := tableState(t, inv)
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var rows []compose.Issued
|
|
||||||
for _, r := range in.Issued {
|
|
||||||
if r.Provider == "anchor" && r.Provision == "s3-bucket" {
|
|
||||||
rows = append(rows, r)
|
|
||||||
if r.ForProvider == "" {
|
|
||||||
t.Errorf("the row of %s carries no copy sealed to the provider", r.ConsumerModule)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(rows) != 2 {
|
|
||||||
t.Fatalf("Issued holds %d row(s) for anchor's s3-bucket, want files and reports: %+v", len(rows), rows)
|
|
||||||
}
|
|
||||||
var would []string
|
|
||||||
for _, p := range in.wouldMake {
|
|
||||||
would = append(would, p.ConsumerModule)
|
|
||||||
}
|
|
||||||
if len(would) != 1 || would[0] != "metrics" {
|
|
||||||
t.Fatalf("the credentials the next send makes are %v, want metrics", would)
|
|
||||||
}
|
|
||||||
for _, c := range in.Credentials {
|
|
||||||
if c.ConsumerModule == "metrics" {
|
|
||||||
t.Errorf("a credential nobody holds has an entry: %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
after := tableState(t, inv)
|
|
||||||
for k, v := range after {
|
|
||||||
if before[k] != v {
|
|
||||||
t.Errorf("reading the Input moved %s", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1383/AC2: a credential only a person gives that nobody gave is not given, and so is one given under a key the machine no
|
|
||||||
// longer has; a credential given by a person for the pair and sealed to a key one end no longer has is refused: in the words
|
|
||||||
// plan.go and secrets.go use.
|
|
||||||
func TestACredentialOnlyAPersonGivesIsNotGivenAndAGivenOneUnderAnOldKeyIsSaidSo(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
register(t, open, catalogue.Manifest{Module: "indexers", Version: "1", Provides: []catalogue.Offer{{
|
|
||||||
Name: "indexer-api", Scope: catalogue.ScopeMesh, Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "grabber", Version: "1", Requires: []string{"indexer-api"},
|
|
||||||
Secrets: map[string]string{"indexer-api": "/etc/grabber/key"}})
|
|
||||||
register(t, open, aConsumerOfTheBucket("ledger"))
|
|
||||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1", Provides: catalogue.FromAnywhere("s3-bucket")})
|
|
||||||
for _, a := range [][2]string{{"anchor", "indexers"}, {"laptop", "grabber"}, {"anchor", "objects"}, {"laptop", "ledger"}} {
|
|
||||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
|
||||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stateOf := func(provision, consumerModule string) (compose.Credential, bool) {
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, c := range in.Credentials {
|
|
||||||
if c.Provision == provision && c.ConsumerModule == consumerModule {
|
|
||||||
return c, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return compose.Credential{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
c, found := stateOf("indexer-api", "grabber")
|
|
||||||
if !found || c.State != compose.CredentialNotGiven || !strings.Contains(c.Why, "nobody gave it") {
|
|
||||||
t.Fatalf("a credential nobody gave reads as %+v", c)
|
|
||||||
}
|
|
||||||
if err := inv.AcceptSecretForPair(ctx, "indexer-api", "laptop", "grabber", "anchor", "", "the-key"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if c, found := stateOf("indexer-api", "grabber"); !found || c.State != compose.CredentialHeld || c.Sealed == "" {
|
|
||||||
t.Fatalf("a credential a person gave reads as %+v", c)
|
|
||||||
}
|
|
||||||
// The consumer's machine generated a key of its own since.
|
|
||||||
if err := inv.RecordSealingKey(ctx, nodeID(t, open, "laptop"), aPublicKey(t)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if c, found := stateOf("indexer-api", "grabber"); !found || c.State != compose.CredentialNotGiven ||
|
|
||||||
!strings.Contains(c.Why, "a sealing key one end no longer has") {
|
|
||||||
t.Fatalf("a credential given under a key the machine no longer has reads as %+v", c)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A credential for a pair, given by a person, under a key one end has since replaced: the mesh cannot make another.
|
|
||||||
if err := inv.RecordSealingKey(ctx, nodeID(t, open, "laptop"), aPublicKey(t)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.AcceptSecretForPair(ctx, "s3-bucket", "laptop", "ledger", "anchor", "", "the-bucket-key"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RecordSealingKey(ctx, nodeID(t, open, "laptop"), aPublicKey(t)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if c, found := stateOf("s3-bucket", "ledger"); !found || c.State != compose.CredentialRefused ||
|
|
||||||
!strings.Contains(c.Why, "was accepted from a person, and a sealing key at one end has changed since") {
|
|
||||||
t.Fatalf("a credential given for the pair under an old key reads as %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1383/AC3: the own secrets carry their origin, the key they are sealed to and the sealed value; the vault's kept boxes, the
|
|
||||||
// licences (with their keys given, so an empty key means the mesh holds none), the users the bus has minted and the
|
|
||||||
// memberships equal what the inventory answers.
|
|
||||||
func TestTheAdaptersInputCarriesTheOwnSecretsTheBoxesTheLicencesAndTheBus(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
key, err := inv.SealingKeyOf(ctx, "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sealed, err := inv.SecretForModule(ctx, "anchor", "objects", "admin")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
box := json.RawMessage(`{"v":1,"eph":"e","iv":"i","tag":"t","ct":"c"}`)
|
|
||||||
if err := inv.KeepCoreBox(ctx, "anchor", inventory.CoreBox{Module: "objects", Name: "admin", Generation: 4,
|
|
||||||
SealedTo: key, Box: box}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.objects", Kind: "module", Node: "anchor",
|
|
||||||
Module: "objects"}, "a-password"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.PutBusMembership(ctx, "laptop", "sealed-membership"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
held, err := open.Licences(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := held.Add(ctx, "claude", "static", map[string]any{"model": "x"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var own *compose.OwnSecretRow
|
|
||||||
for i := range in.OwnSecrets {
|
|
||||||
if r := &in.OwnSecrets[i]; r.Node == "anchor" && r.Module == "objects" && r.Name == "admin" {
|
|
||||||
own = r
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if own == nil || own.Origin != "made" || own.SealedTo != key || own.Sealed != sealed {
|
|
||||||
t.Errorf("the own secret reads as %+v; the inventory holds %q sealed to %q", own, sealed, key)
|
|
||||||
}
|
|
||||||
var kept *compose.CoreBox
|
|
||||||
for i := range in.KeptCoreBoxes {
|
|
||||||
if b := &in.KeptCoreBoxes[i]; b.Node == "anchor" && b.Module == "objects" && b.Name == "admin" {
|
|
||||||
kept = b
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if kept == nil || kept.Generation != 4 || kept.SealedTo != key || string(kept.Box) != string(box) {
|
|
||||||
t.Errorf("the kept box reads as %+v", kept)
|
|
||||||
}
|
|
||||||
if got := strings.Join(in.BusMinted, ","); !strings.Contains(got, "anchor.objects") {
|
|
||||||
t.Errorf("BusMinted is %v", in.BusMinted)
|
|
||||||
}
|
|
||||||
if in.BusMemberships["laptop"] != "sealed-membership" {
|
|
||||||
t.Errorf("BusMemberships is %v", in.BusMemberships)
|
|
||||||
}
|
|
||||||
if len(in.Licences) != 1 || in.Licences[0].Name != "claude" || in.Licences[0].Keys == nil {
|
|
||||||
t.Errorf("Licences read as %+v", in.Licences)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1383/AC4: the bus's user list is what busUserList gives for the machine that runs the bus, and the Input carries none when
|
|
||||||
// no module of the shelf runs the bus.
|
|
||||||
func TestTheBusUsersAreWhatTheControllerComposesForTheMachineThatRunsTheBus(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if in.BusUsers != "" || len(in.BusMissing) != 0 {
|
|
||||||
t.Fatalf("a mesh whose shelf runs no bus carries a bus user list: %q %v", in.BusUsers, in.BusMissing)
|
|
||||||
}
|
|
||||||
bus := catalogue.Manifest{Module: "nats", Version: "1", BusUsers: "/etc/nats/users.conf",
|
|
||||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}
|
|
||||||
register(t, open, bus)
|
|
||||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "controller", Kind: "controller"}, "pw"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
in, err = readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
list, missing, _ := busUserList(ctx, open.inventory, []catalogue.Manifest{bus})
|
|
||||||
if in.BusUsers != list || fmt.Sprint(in.BusMissing) != fmt.Sprint(missing) {
|
|
||||||
t.Errorf("the Input carries %q (%v); busUserList gives %q (%v)", in.BusUsers, in.BusMissing, list, missing)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1383/AC5: Built, CurrentBuilds and BuildIDs equal the inventory's Held, CurrentBuilds and BuildOf, and the serving key and
|
|
||||||
// the certificate are the machine's.
|
|
||||||
func TestTheAdaptersInputCarriesTheBuildsAndTheMachinesKey(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
held, err := inv.Held(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var wantBuilt []string
|
|
||||||
for repository := range held {
|
|
||||||
wantBuilt = append(wantBuilt, repository)
|
|
||||||
}
|
|
||||||
sort.Strings(wantBuilt)
|
|
||||||
if fmt.Sprint(in.Built) != fmt.Sprint(wantBuilt) && !(len(in.Built) == 0 && len(wantBuilt) == 0) {
|
|
||||||
t.Errorf("Built is %v, the inventory's %v", in.Built, wantBuilt)
|
|
||||||
}
|
|
||||||
current, err := inv.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for module, b := range current {
|
|
||||||
if in.CurrentBuilds[module] != b.Commit {
|
|
||||||
t.Errorf("CurrentBuilds[%s] is %q, the inventory's %q", module, in.CurrentBuilds[module], b.Commit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for module, byCommit := range in.BuildIDs {
|
|
||||||
for commit, build := range byCommit {
|
|
||||||
if want, _ := inv.BuildOf(ctx, module, commit); want != build {
|
|
||||||
t.Errorf("BuildIDs[%s][%s] is %q, the inventory's %q", module, commit, build, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The lead's answer 4: where the mesh holds no credential for any bus user, the machine that runs the bus is refused with
|
|
||||||
// busUserList's words (the controller's composition refused it), the Input still builds, and another machine composes.
|
|
||||||
func TestTheBusMachineIsRefusedWhereNoUserHasACredentialAndTheOthersCompose(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
bus := catalogue.Manifest{Module: "nats", Version: "1", BusUsers: "/etc/nats/users.conf",
|
|
||||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}
|
|
||||||
register(t, open, bus)
|
|
||||||
register(t, open, aWebModule())
|
|
||||||
if _, err := assign(ctx, open, "anchor", "nats"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "laptop", "web"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := composeQuestion(ctx, open, "anchor", compose.Foreseeing, false); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), "not one user has a credential") {
|
|
||||||
t.Fatalf("the bus machine is composed: %v", err)
|
|
||||||
}
|
|
||||||
if _, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false); err != nil {
|
|
||||||
t.Fatalf("a machine that does not run the bus is refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,329 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1382: the adapter's Input names, for every node, what the inventory answers for it, each compared with the read named
|
|
||||||
// beside the field in go/compose/input.go (novox/hq ADR 0453 clause 3).
|
|
||||||
func TestTheAdaptersInputNamesWhatTheInventoryAnswersForEachNode(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
if err := inv.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := inv.PortFor(ctx, "laptop", "files", 8080, false); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(in.Catalogue, shelf) {
|
|
||||||
t.Error("the Input's Catalogue is not the shelf the inventory holds")
|
|
||||||
}
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(in.Nodes) != len(nodes) {
|
|
||||||
t.Fatalf("the Input names %d node(s), the inventory has %d", len(in.Nodes), len(nodes))
|
|
||||||
}
|
|
||||||
places, err := inv.Overlays(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, record := range nodes {
|
|
||||||
n, ok := nodeOf(in.Input, record.Name)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("%s is not in the Input", record.Name)
|
|
||||||
}
|
|
||||||
for _, p := range places {
|
|
||||||
if p.Name == record.Name {
|
|
||||||
if want := (compose.Place{Key: p.Key, Endpoint: p.Endpoint, Site: p.Site, Hub: p.Hub, Address: p.Address}); n.Place != want {
|
|
||||||
t.Errorf("%s: Place is %+v, the inventory's %+v", record.Name, n.Place, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assigned, _ := inv.Assigned(ctx, record.Name)
|
|
||||||
if !reflect.DeepEqual(n.Assigned, assigned) {
|
|
||||||
t.Errorf("%s: Assigned is %v, the inventory's %v", record.Name, n.Assigned, assigned)
|
|
||||||
}
|
|
||||||
caps, _ := inv.ProfileOf(ctx, record.Name)
|
|
||||||
if !reflect.DeepEqual(n.Capabilities, caps) {
|
|
||||||
t.Errorf("%s: Capabilities are %v, the inventory's %v", record.Name, n.Capabilities, caps)
|
|
||||||
}
|
|
||||||
domain, _ := inv.PublicDomainOf(ctx, record.Name)
|
|
||||||
if n.PublicDomain != domain {
|
|
||||||
t.Errorf("%s: PublicDomain is %q, the inventory's %q", record.Name, n.PublicDomain, domain)
|
|
||||||
}
|
|
||||||
if n.Account != record.Account || n.AccountHome != record.AccountHome || n.AgentAccount != record.AgentAccount ||
|
|
||||||
n.AgentAccountHome != record.AgentAccountHome || n.Adopted != record.Adopted {
|
|
||||||
t.Errorf("%s: the accounts or the mode differ from the inventory's: %+v", record.Name, n)
|
|
||||||
}
|
|
||||||
pins, _ := inv.PinsFor(ctx, record.Name)
|
|
||||||
if len(pins) != len(n.Pins) || (len(pins) > 0 && !reflect.DeepEqual(n.Pins, pins)) {
|
|
||||||
t.Errorf("%s: Pins are %v, the inventory's %v", record.Name, n.Pins, pins)
|
|
||||||
}
|
|
||||||
bound, _ := inv.BindingsFor(ctx, record.Name)
|
|
||||||
if len(bound) != len(n.Bound) || (len(bound) > 0 && !reflect.DeepEqual(n.Bound, bound)) {
|
|
||||||
t.Errorf("%s: Bound is %v, the inventory's %v", record.Name, n.Bound, bound)
|
|
||||||
}
|
|
||||||
key, _ := inv.SealingKeyOf(ctx, record.Name)
|
|
||||||
if n.SealingKey != key {
|
|
||||||
t.Errorf("%s: SealingKey is %q, the inventory's %q", record.Name, n.SealingKey, key)
|
|
||||||
}
|
|
||||||
ports, _ := inv.PortsFor(ctx, record.Name)
|
|
||||||
want := map[string]map[int]int{}
|
|
||||||
for _, a := range ports {
|
|
||||||
if want[a.Module] == nil {
|
|
||||||
want[a.Module] = map[int]int{}
|
|
||||||
}
|
|
||||||
want[a.Module][a.Wanted] = a.Machine
|
|
||||||
}
|
|
||||||
if len(want) != len(n.Ports) || (len(want) > 0 && !reflect.DeepEqual(n.Ports, want)) {
|
|
||||||
t.Errorf("%s: Ports are %v, the inventory's %v", record.Name, n.Ports, want)
|
|
||||||
}
|
|
||||||
// The settings of every module it resolves to, layer for layer.
|
|
||||||
resolved, err := compose.Resolve(in.Input, record.Name)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, m := range resolved.Plan.Modules {
|
|
||||||
layers, err := inv.SettingsFor(ctx, record.Name, m.Module)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(layers) != len(n.Settings[m.Module]) || (len(layers) > 0 && !reflect.DeepEqual(n.Settings[m.Module], layers)) {
|
|
||||||
t.Errorf("%s: the settings of %s are %v, the inventory's %v", record.Name, m.Module, n.Settings[m.Module], layers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if domain := nodeDomain(in.Input, "anchor"); domain != "example.test" {
|
|
||||||
t.Errorf("anchor's PublicDomain is %q", domain)
|
|
||||||
}
|
|
||||||
if got := nodePorts(in.Input, "laptop"); got["files"][8080] == 0 {
|
|
||||||
t.Errorf("laptop's port for files:8080 is not in the Input: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func nodeDomain(in compose.Input, name string) string {
|
|
||||||
n, _ := nodeOf(in, name)
|
|
||||||
return n.PublicDomain
|
|
||||||
}
|
|
||||||
|
|
||||||
func nodePorts(in compose.Input, name string) map[string]map[int]int {
|
|
||||||
n, _ := nodeOf(in, name)
|
|
||||||
return n.Ports
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1382/AC2: a node keeping a recorded module at the build it runs has that module in Kept and KeptBuilds, and the Input's
|
|
||||||
// Catalogue is the current shelf, not the kept manifest (R4a review round 1, finding 1).
|
|
||||||
func TestANodeKeepingARecordedBuildHasItInKeptAndTheCatalogueStaysCurrent(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
start := time.Now().Add(-time.Hour)
|
|
||||||
image := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64)
|
|
||||||
for _, b := range []link.BuildResult{
|
|
||||||
aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, map[string][2]string{"server": {image, ""}}),
|
|
||||||
} {
|
|
||||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := inv.Assign(ctx, "anchor", "postgres"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"postgres": "c1111111"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute),
|
|
||||||
map[string][2]string{"server": {image, "runtime"}})); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A send that is not a person's push keeps the build the machine runs.
|
|
||||||
in, err := readComposeInput(keepingRecorded(ctx), open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
anchor, _ := nodeOf(in.Input, "anchor")
|
|
||||||
if anchor.KeptBuilds["postgres"] != "c1111111" {
|
|
||||||
t.Fatalf("anchor keeps %v, want postgres at c1111111", anchor.KeptBuilds)
|
|
||||||
}
|
|
||||||
kept, found := anchor.Kept["postgres"]
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("the kept manifest of postgres is not in anchor's Kept: %v", anchor.Kept)
|
|
||||||
}
|
|
||||||
current, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(in.Catalogue["postgres"], current["postgres"]) {
|
|
||||||
t.Error("the Input's Catalogue is not the current shelf")
|
|
||||||
}
|
|
||||||
if reflect.DeepEqual(kept, current["postgres"]) {
|
|
||||||
t.Error("the kept manifest is the current one: nothing is kept")
|
|
||||||
}
|
|
||||||
if laptop, _ := nodeOf(in.Input, "laptop"); len(laptop.Kept) != 0 || len(laptop.KeptBuilds) != 0 {
|
|
||||||
t.Errorf("laptop keeps what only anchor runs: %v", laptop.KeptBuilds)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A person's push keeps nothing.
|
|
||||||
pushed, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if a, _ := nodeOf(pushed.Input, "anchor"); len(a.Kept) != 0 || len(a.KeptBuilds) != 0 {
|
|
||||||
t.Errorf("a person's push keeps %v", a.KeptBuilds)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1382/AC3: the seats on record, the tunnels, the adopted tunnel, its carried peers and the nodes joining equal the
|
|
||||||
// inventory's answers; the range and the suffix are read from the environment once, here.
|
|
||||||
func TestTheAdaptersInputCarriesTheNetworkAsTheInventoryHoldsIt(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
t.Setenv(OverlayCIDRVar, "10.99.0.0/16")
|
|
||||||
t.Setenv("MESH_INTERNAL_SUFFIX", "mesh.test")
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
holdings, err := inv.Holdings(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(in.Holdings, holdings) {
|
|
||||||
t.Errorf("Holdings are %v, the inventory's %v", in.Holdings, holdings)
|
|
||||||
}
|
|
||||||
tunnels, err := inv.Tunnels(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(tunnels) != len(in.Tunnels) {
|
|
||||||
t.Errorf("Tunnels are %v, the inventory's %v", in.Tunnels, tunnels)
|
|
||||||
}
|
|
||||||
if _, hub, _, err := inv.AdoptedTunnel(ctx); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
} else if (hub == "") != (in.Adopted == nil) {
|
|
||||||
t.Errorf("Adopted is %v, the inventory's hub is %q", in.Adopted, hub)
|
|
||||||
}
|
|
||||||
carried, err := inv.CarriedPeers(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(carried) != len(in.CarriedPeers) {
|
|
||||||
t.Errorf("CarriedPeers are %v, the inventory's %v", in.CarriedPeers, carried)
|
|
||||||
}
|
|
||||||
joining, err := inv.NodesWithALiveToken(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(joining) != len(in.Joining) {
|
|
||||||
t.Errorf("Joining is %v, the inventory's %v", in.Joining, joining)
|
|
||||||
}
|
|
||||||
if in.OverlayCIDR != "10.99.0.0/16" || in.InternalSuffix != "mesh.test" {
|
|
||||||
t.Errorf("the range and the suffix are %q and %q", in.OverlayCIDR, in.InternalSuffix)
|
|
||||||
}
|
|
||||||
// Read once: what the environment says afterwards moves nothing already read.
|
|
||||||
t.Setenv(OverlayCIDRVar, "10.1.0.0/16")
|
|
||||||
if in.OverlayCIDR != "10.99.0.0/16" {
|
|
||||||
t.Error("the range moved with the environment")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The lead's equality finding (build 1792487777757012513): a provider on one machine serves a fact that says ${setting:domain},
|
|
||||||
// the setting is set for it, and a consumer on another machine takes that fact. The first pass of the adapter resolved every
|
|
||||||
// node with no settings in the Input, so the served fact could not be settled and the whole Input failed. The settings are in
|
|
||||||
// the Input before any node is resolved, and the Input builds, with the same plan for the consumer as the controller's own
|
|
||||||
// composition. It failed first, with the finding's words.
|
|
||||||
func TestAServedFactThatSaysASettingIsResolvedWithTheSettingInTheInput(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
|
|
||||||
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
|
|
||||||
Serves: map[string]map[string]any{"acme-ca": {"domain": "${setting:domain}"}}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1", Requires: []string{"acme-ca"}})
|
|
||||||
// The setting is there before the machines are assigned: an assignment resolves the mesh, and a served fact that says
|
|
||||||
// ${setting:domain} resolves only with it.
|
|
||||||
if err := open.inventory.KeepSettings(ctx, "", "step-ca", map[string]any{"domain": "example.test"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "laptop", "route-proxy"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("the Input cannot be built: %v", err)
|
|
||||||
}
|
|
||||||
if anchor, _ := nodeOf(in.Input, "anchor"); len(anchor.Settings["step-ca"]) == 0 {
|
|
||||||
t.Fatalf("the Input does not carry the setting of the provider: %v", anchor.Settings)
|
|
||||||
}
|
|
||||||
oldPlan, _, err := resolveNode(ctx, open, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
newPlan, _, err := resolveNode(withInputCache(ctx), open, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("the consumer does not resolve: %v", err)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(oldPlan.Needs, newPlan.Needs) {
|
|
||||||
t.Errorf("the consumer's needs are %+v, the controller's own %+v", newPlan.Needs, oldPlan.Needs)
|
|
||||||
}
|
|
||||||
t.Logf("a served fact that says a setting resolves with the setting in the Input: the Input builds and the consumer's needs equal planFor's")
|
|
||||||
}
|
|
||||||
|
|
||||||
// A setting that nothing sets, in a fact a provider serves, is not the adapter's to hide: the controller's own composition
|
|
||||||
// refuses it as well (theRestOfTheMesh returns servedOnNode's error), and the adapter refuses with the same words rather than
|
|
||||||
// building an Input that composes a placeholder.
|
|
||||||
func TestAServedFactThatSaysAnUnsetSettingIsRefusedAsTheControllerRefusesIt(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
|
|
||||||
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
|
|
||||||
Serves: map[string]map[string]any{"acme-ca": {"domain": "${setting:domain}"}}})
|
|
||||||
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1", Requires: []string{"acme-ca"}})
|
|
||||||
// Assigned with the setting there, then the setting is cleared: the mesh the controller holds when someone clears it.
|
|
||||||
if err := open.inventory.KeepSettings(ctx, "", "step-ca", map[string]any{"domain": "example.test"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "laptop", "route-proxy"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.ClearSettings(ctx, "", "step-ca"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _, oldErr := resolveNode(ctx, open, "laptop")
|
|
||||||
_, newErr := readComposeInput(ctx, open)
|
|
||||||
if (oldErr == nil) != (newErr == nil) {
|
|
||||||
t.Fatalf("the controller's composition says %v, the adapter says %v", oldErr, newErr)
|
|
||||||
}
|
|
||||||
if oldErr != nil && !strings.Contains(newErr.Error(), "nothing sets \"domain\"") {
|
|
||||||
t.Errorf("the adapter's words are %q, the controller's %q", newErr, oldErr)
|
|
||||||
}
|
|
||||||
t.Logf("an unset setting in a served fact is refused alike: the controller says %v, the adapter says %v", oldErr != nil, newErr != nil)
|
|
||||||
}
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Issue 1493: a module in the config form whose value is set only in mesh-config, with no settings row, reaches the
|
|
||||||
// node's settings in the composition's input.
|
|
||||||
func TestAConfigModuleWithoutASettingsRowIsReadFromConfig(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
register(t, open, configForm(t, "brother"))
|
|
||||||
if _, err := assign(ctx, open, "anchor", "brother"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
inv.ExecForTest(t, `insert into config_move (moved_by, why) values ('a test', 'issue 1493')`)
|
|
||||||
inv.ReadsConfigFrom(func(_ context.Context, node, module string) (map[string]any, error) {
|
|
||||||
if node == "anchor" && module == "brother" {
|
|
||||||
return map[string]any{"level": "debug"}, nil
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
})
|
|
||||||
in, _, err := readMeshAndNodes(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := readSettings(ctx, open, &in); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, n := range in.Nodes {
|
|
||||||
if n.Name != "anchor" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
layers := n.Settings["brother"]
|
|
||||||
if len(layers) != 1 || layers[0].Values["level"] != "debug" {
|
|
||||||
t.Fatalf("anchor's settings for brother are %+v, want level debug from mesh-config", layers)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Fatal("no anchor node in the input")
|
|
||||||
}
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// configForm is a module in the config form, shaped as a real one (inventory's config_move_test).
|
|
||||||
func configForm(t *testing.T, name string) catalogue.Manifest {
|
|
||||||
t.Helper()
|
|
||||||
m, err := catalogue.ParseManifest([]byte(`{"module": "` + name + `", "version": "1",
|
|
||||||
"config": {"level": {"type": "string", "description": "how much it says", "default": "info"}},
|
|
||||||
"resources": [{"id": "conf", "type": "file", "path": "/etc/` + name + `.json", "mode": "0640",
|
|
||||||
"content": "{\"level\": \"info\"}", "merge": "json", "takes-config": ["level"]}]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// Issue 1493: a module in the config form whose values are set only in mesh-config has no settings row, and its
|
|
||||||
// settings are still read, because SettingsFor is where mesh-config's values are laid in.
|
|
||||||
func TestSettingsAreReadForAConfigModuleWithoutASettingsRow(t *testing.T) {
|
|
||||||
shelf := map[string]catalogue.Manifest{
|
|
||||||
"brother": configForm(t, "brother"), // config only in mesh-config: no settings row
|
|
||||||
"older": {Module: "older", Version: "1"}, // settings form, with a row
|
|
||||||
"idle": {Module: "idle", Version: "1"}, // settings form, no row: nothing to read
|
|
||||||
"newer": configForm(t, "newer"), // config form, but the switch is off for it
|
|
||||||
}
|
|
||||||
have := map[string]bool{"older": true, "gone": true} // "gone" has a row and is not on the shelf
|
|
||||||
reads := func(module string) (bool, error) { return module == "brother", nil }
|
|
||||||
got, err := modulesToRead(shelf, have, reads)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if want := []string{"brother", "older"}; !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("settings are read for %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
if _, err := modulesToRead(shelf, have, func(string) (bool, error) { return false, errors.New("store down") }); err == nil {
|
|
||||||
t.Error("a failing ReadsConfig is not returned")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
)
|
|
||||||
|
|
||||||
// made is what a send made in a store, by key and never by value: the machine ports given, the pair credentials and the
|
|
||||||
// own secrets.
|
|
||||||
type made struct {
|
|
||||||
Ports []string
|
|
||||||
Pairs []string
|
|
||||||
OwnSecrets []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// madeIn is what a store holds that a send makes, by key and never by value.
|
|
||||||
func madeIn(ctx context.Context, open *stores) (made, error) {
|
|
||||||
var out made
|
|
||||||
inv := open.inventory
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
for _, n := range nodes {
|
|
||||||
ports, err := inv.PortsFor(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
for _, p := range ports {
|
|
||||||
out.Ports = append(out.Ports, fmt.Sprintf("%s %s:%d->%d", n.Name, p.Module, p.Wanted, p.Machine))
|
|
||||||
}
|
|
||||||
rows, err := inv.SecretsFrom(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
for _, s := range rows {
|
|
||||||
out.Pairs = append(out.Pairs, fmt.Sprintf("%s %s on %s from %s local=%s", s.Name, s.ConsumerModule, s.Consumer, n.Name, s.Local))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
own, err := inv.OwnSecretRows(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
for _, r := range own {
|
|
||||||
out.OwnSecrets = append(out.OwnSecrets, fmt.Sprintf("%s %s/%s (%s)", r.Node, r.Module, r.Name, r.Origin))
|
|
||||||
}
|
|
||||||
sort.Strings(out.Ports)
|
|
||||||
sort.Strings(out.Pairs)
|
|
||||||
sort.Strings(out.OwnSecrets)
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1382/AC5: composing every node of one operation builds the Input once; without an operation's cache, once for each.
|
|
||||||
func TestComposingEveryNodeOfAnOperationBuildsTheInputOnce(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
built := 0
|
|
||||||
was := composeInputBuilt
|
|
||||||
composeInputBuilt = func() { built++ }
|
|
||||||
t.Cleanup(func() { composeInputBuilt = was })
|
|
||||||
ctx := withInputCache(t.Context())
|
|
||||||
nodes, err := open.inventory.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, n := range nodes {
|
|
||||||
if _, _, err := resolveNode(ctx, open, n.Name); err != nil && !unresolvable(err) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := composeQuestion(ctx, open, n.Name, compose.Reading, false); err != nil && !unresolvable(err) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := generatorsOf(ctx, open); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if built != 1 {
|
|
||||||
t.Fatalf("an operation over %d node(s) built the Input %d time(s)", len(nodes), built)
|
|
||||||
}
|
|
||||||
built = 0
|
|
||||||
if _, _, err := resolveNode(t.Context(), open, nodes[0].Name); err != nil && !unresolvable(err) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if built != 1 {
|
|
||||||
t.Fatalf("a composition with no cache built the Input %d time(s)", built)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,265 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// perform makes what a composition named foreseen (novox/hq ADR 0453 clause 3, design 47 R4b, issue 1385).
|
|
||||||
//
|
|
||||||
// **The composition makes nothing; a send does.** mesh-sdk's composition over an Input names what the next send would make
|
|
||||||
// (the ports it would assign, the credentials it would seal, the core boxes it would ask the vault for, the seats it would
|
|
||||||
// pass on, the certificate it would issue) and stands in for each. A send composes, performs, reads the stores again and
|
|
||||||
// composes again (send in compose_sdk.go), and refuses to send a declaration that still names anything foreseen. A question
|
|
||||||
// (status, facts, probes, the merge gate's reading) never reaches this file.
|
|
||||||
//
|
|
||||||
// What is made is exactly what the controller's own composition made on the way to a declaration it was going to send:
|
|
||||||
// - every pair credential no row holds, for every machine that resolves, since planning one node resolved them all;
|
|
||||||
// - the seats the mesh passes on, with the hand act (not on a throwaway mesh, whose hand act has nowhere to go);
|
|
||||||
// - the machine ports the node's modules were given none for;
|
|
||||||
// - the node's own secrets, by the controller's own rules (a bus credential nobody issued was refused by the composition);
|
|
||||||
// - the core credentials, asked of the vault every send, so a rotated one reaches the machine;
|
|
||||||
// - the certificate of a machine that was never certified.
|
|
||||||
//
|
|
||||||
// perform returns whether it made or recorded anything: the reason the caller reads the stores again.
|
|
||||||
func perform(ctx context.Context, open *stores, in *meshInput, node string, d compose.Declared) (bool, error) {
|
|
||||||
did := false
|
|
||||||
inv := open.inventory
|
|
||||||
|
|
||||||
// The seats passed on: recorded in the store with the controller's own hand act (ADR 0449 §4, issue 1317).
|
|
||||||
if len(d.Passes.Passes) > 0 {
|
|
||||||
var w io.Writer = os.Stderr
|
|
||||||
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
|
|
||||||
w = lw
|
|
||||||
}
|
|
||||||
for _, line := range d.Passes.Said {
|
|
||||||
fmt.Fprintln(w, line)
|
|
||||||
}
|
|
||||||
if performSeatPasses(ctx, seatPassingOn(inv, w), d.Passes.Passes) {
|
|
||||||
did = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every pair credential the stores hold none for: made once and kept, and sealed to both ends before it was ever written
|
|
||||||
// down. The error is the one planFor met: the machine that needs it, and why none could be made.
|
|
||||||
for _, p := range in.wouldMake {
|
|
||||||
if err := makePair(ctx, inv, p); err != nil {
|
|
||||||
return did, fmt.Errorf("%s on %s needs %s from %s and no credential could be made for it: %w",
|
|
||||||
p.ConsumerModule, p.Consumer, p.Name, p.Provider, err)
|
|
||||||
}
|
|
||||||
did = true
|
|
||||||
}
|
|
||||||
|
|
||||||
// The ports this node's modules were given none for.
|
|
||||||
if len(d.ForeseenPorts) > 0 {
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return did, err
|
|
||||||
}
|
|
||||||
for _, one := range d.ForeseenPorts {
|
|
||||||
module, port, ok := strings.Cut(one, ":")
|
|
||||||
wanted, perr := strconv.Atoi(port)
|
|
||||||
if !ok || perr != nil {
|
|
||||||
return did, fmt.Errorf("the composition named a port the controller cannot read: %q", one)
|
|
||||||
}
|
|
||||||
fixed := false
|
|
||||||
for _, l := range nodeShelf(in, shelf, node)[module].Listens {
|
|
||||||
if l.Port == wanted {
|
|
||||||
fixed = l.Fixed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := inv.PortFor(ctx, node, module, wanted, fixed); err != nil {
|
|
||||||
return did, fmt.Errorf("%s needs %d reachable on %s and it could not be assigned: %w", module, wanted, node, err)
|
|
||||||
}
|
|
||||||
did = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The node's own secrets: a module's superuser password, an administrator, an account. The composition refused what the
|
|
||||||
// send refuses (a bus credential nobody issued, a given value under an old key).
|
|
||||||
for _, one := range d.StoodIn {
|
|
||||||
module, name, ok := strings.Cut(one, "/")
|
|
||||||
if !ok || strings.Contains(one, " ") {
|
|
||||||
continue // a grant's sealed copy, the certificate or the bus's list: made below or by nobody
|
|
||||||
}
|
|
||||||
if _, err := inv.SecretForModule(ctx, node, module, name); err != nil {
|
|
||||||
return did, err
|
|
||||||
}
|
|
||||||
did = true
|
|
||||||
}
|
|
||||||
|
|
||||||
// The certificate of a machine that was never certified: issued and kept, with the authority when there is none yet.
|
|
||||||
for _, one := range d.StoodIn {
|
|
||||||
if strings.HasPrefix(one, "the certificate and the authority of ") {
|
|
||||||
if _, _, err := certificateFor(ctx, open, node); err != nil {
|
|
||||||
return did, err
|
|
||||||
}
|
|
||||||
did = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return did, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// makePair makes one pair credential by the controller's own read of it: the credential for the pair, or the provider's one
|
|
||||||
// credential sealed to this consumer too. A credential only a person gives is never made (it is not a would-make).
|
|
||||||
func makePair(ctx context.Context, inv *inventory.Inventory, p inventory.PairCredential) error {
|
|
||||||
var err error
|
|
||||||
switch p.Kind {
|
|
||||||
case inventory.PairShared:
|
|
||||||
_, err = inv.SharedSecretFor(ctx, p.Name, p.Consumer, p.ConsumerModule, p.Provider, p.ProviderModule, p.Local, p.Own)
|
|
||||||
case inventory.PairPlain:
|
|
||||||
_, err = inv.SecretFor(ctx, p.Name, p.Consumer, p.ConsumerModule, p.Provider, p.Local)
|
|
||||||
default:
|
|
||||||
return errors.New("a credential only a person gives is not made")
|
|
||||||
}
|
|
||||||
if inventory.IsNotGiven(err) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// stillForeseen says what a composed declaration still stands in for: the lists the composition names, and the stand-in itself
|
|
||||||
// anywhere in the body (a pair credential's stand-in is not in any list). A declaration that has any is never sent.
|
|
||||||
func stillForeseen(d compose.Declared) []string {
|
|
||||||
var left []string
|
|
||||||
left = append(left, d.Foreseen...)
|
|
||||||
left = append(left, d.StoodIn...)
|
|
||||||
for _, p := range d.ForeseenPorts {
|
|
||||||
left = append(left, "the port "+p)
|
|
||||||
}
|
|
||||||
for _, c := range d.CoreBoxesWanted {
|
|
||||||
left = append(left, "the vault's box for "+c.Module+"/"+c.Name)
|
|
||||||
}
|
|
||||||
for _, p := range d.Passes.Passes {
|
|
||||||
left = append(left, "the pass of "+p.Seat.Name+" to "+p.To.Module+" on "+p.To.Node)
|
|
||||||
}
|
|
||||||
if body, err := d.Body(); err == nil {
|
|
||||||
for _, marker := range []string{catalogue.ForeseenSealed, "asked of the vault by the next send", compose.ForeseenCertificate,
|
|
||||||
compose.ForeseenBusUsers} {
|
|
||||||
if strings.Contains(string(body), marker) {
|
|
||||||
left = append(left, "a stand-in: "+marker)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(left)
|
|
||||||
return slicesCompact(left)
|
|
||||||
}
|
|
||||||
|
|
||||||
func slicesCompact(in []string) []string {
|
|
||||||
var out []string
|
|
||||||
for _, s := range in {
|
|
||||||
if len(out) == 0 || out[len(out)-1] != s {
|
|
||||||
out = append(out, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// nodeShelf is the shelf as one node composes its own set: the current one with each module it keeps at the build it runs put
|
|
||||||
// in place of the current.
|
|
||||||
func nodeShelf(in *meshInput, shelf map[string]catalogue.Manifest, node string) map[string]catalogue.Manifest {
|
|
||||||
n, _ := nodeOf(in.Input, node)
|
|
||||||
return shelfWith(shelf, n.Kept)
|
|
||||||
}
|
|
||||||
|
|
||||||
// prepareSend is what a send does before it composes, every send and whatever the composition would have named: the
|
|
||||||
// controller's own composition did it on the way, so a module's file that names an own secret composes (mesh-sdk's
|
|
||||||
// composition can stand in for none: a file naming a secret not made yet is refused in every mode), and the core's credentials
|
|
||||||
// are asked of the vault on every send, so a rotated one reaches the machine and a silent vault is said (core_box.go).
|
|
||||||
//
|
|
||||||
// - Each own secret of a module the node runs (not one handed to the provisioner, not one issued outside the mesh, not one a
|
|
||||||
// person gives) that no row holds is made, unless the composition would refuse it (a bus credential nobody issued): that
|
|
||||||
// refusal is the composition's to say, in its words.
|
|
||||||
// - The core's credentials are asked of the vault, the boxes newly answered are kept, and the modules the vault did not answer
|
|
||||||
// for with no box kept are returned: the controller seals those itself until the vault holds them (ADR 0396).
|
|
||||||
func prepareSend(ctx context.Context, open *stores, node string) ([]string, error) {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if in.refused[node] != nil {
|
|
||||||
return nil, nil // empty-on-error: refused alone for what it keeps; the composition that follows says so, in its words
|
|
||||||
}
|
|
||||||
resolved, err := compose.Resolve(in.Input, node)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil // empty-on-error: it does not compose for its own reasons; the composition that follows says them
|
|
||||||
}
|
|
||||||
n, _ := nodeOf(in.Input, node)
|
|
||||||
hands := n.Stated && n.Contract >= compose.HandedContract
|
|
||||||
handing := catalogue.Rendering{Hands: hands, Provisioner: provisionerAccount(resolved.Plan)}
|
|
||||||
inv := open.inventory
|
|
||||||
did := false
|
|
||||||
if n.SealingKey != "" {
|
|
||||||
held := map[string]bool{}
|
|
||||||
for _, r := range in.OwnSecrets {
|
|
||||||
// Held only when sealed to the key the machine has now, or given by a person (which is not the mesh's
|
|
||||||
// to make again): a made row under an old key is re-made, as SecretForModule did.
|
|
||||||
if r.Node == node && (r.SealedTo == n.SealingKey || r.Origin == inventory.OriginAccepted) {
|
|
||||||
held[r.Module+"/"+r.Name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, m := range resolved.Plan.Modules {
|
|
||||||
if handing.Handing(m) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
plain := m.OwnSecrets.Plain()
|
|
||||||
names := make([]string, 0, len(plain))
|
|
||||||
for name := range plain {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
for _, name := range names {
|
|
||||||
if plain[name].IssuedBy == catalogue.IssuedOutside || held[m.Module+"/"+name] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
switch name {
|
|
||||||
case "broker":
|
|
||||||
if !slices.Contains(in.BusMinted, node+"."+m.Module) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
case compose.ProvisionerSecret:
|
|
||||||
if !slices.Contains(in.BusMinted, "provisioner."+node) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
// A bus process's account nobody issued still gets the stand-in, until the composition leaves
|
|
||||||
// the module off (issue 1753): the warning names the verb (issue 1226).
|
|
||||||
if line := processAccountWarning(m, node, name, in.BusMinted); line != "" {
|
|
||||||
sayWarning(ctx, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := inv.SecretForModule(ctx, node, m.Module, name); err == nil {
|
|
||||||
did = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var byController []string
|
|
||||||
if n.Stated && n.Contract >= compose.VaultBoxContract {
|
|
||||||
boxes, err := coreBoxesFor(ctx, inv, node, resolved.Plan.Modules, func(m catalogue.Manifest) bool {
|
|
||||||
return handing.Handing(m)
|
|
||||||
}, true)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
did = did || len(boxes.keep) > 0
|
|
||||||
byController = boxes.byController
|
|
||||||
}
|
|
||||||
if did {
|
|
||||||
// What was made is in the stores now: the composition reads them afresh.
|
|
||||||
forgetInput(ctx)
|
|
||||||
}
|
|
||||||
return byController, nil
|
|
||||||
}
|
|
||||||
@@ -1,235 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
func aWebModule() catalogue.Manifest {
|
|
||||||
return catalogue.Manifest{Module: "web", Version: "1",
|
|
||||||
Listens: []catalogue.Listening{{Port: 8080, From: catalogue.FromEverywhere}},
|
|
||||||
OwnSecrets: catalogue.OwnSecrets{"admin": {Path: "/etc/web/admin"}},
|
|
||||||
Resources: []map[string]any{
|
|
||||||
{"id": "server", "type": "container", "name": "web", "ports": []any{"8080"},
|
|
||||||
"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1385/AC1: a port the composition names foreseen is allocated with the inventory's PortFor for that module and port, a second
|
|
||||||
// composition names it no more, and the number is the one the controller's own Allocating composition gives in the same store.
|
|
||||||
func TestPerformAllocatesAForeseenPortAndTheNumberIsTheControllers(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aWebModule())
|
|
||||||
if _, err := assign(ctx, open, "laptop", "web"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
d, _, err := compose.ComposeIn(in.Input, "laptop", compose.Foreseeing)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(d.ForeseenPorts) != 1 || d.ForeseenPorts[0] != "web:8080" {
|
|
||||||
t.Fatalf("the composition names the ports %v foreseen, want web:8080", d.ForeseenPorts)
|
|
||||||
}
|
|
||||||
did, err := perform(ctx, open, in, "laptop", d)
|
|
||||||
if err != nil || !did {
|
|
||||||
t.Fatalf("perform did %v: %v", did, err)
|
|
||||||
}
|
|
||||||
held, err := open.inventory.PortsFor(ctx, "laptop")
|
|
||||||
if err != nil || len(held) != 1 || held[0].Module != "web" || held[0].Wanted != 8080 {
|
|
||||||
t.Fatalf("the port perform allocated reads %+v: %v", held, err)
|
|
||||||
}
|
|
||||||
in, err = readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if again, _, err := compose.ComposeIn(in.Input, "laptop", compose.Foreseeing); err != nil || len(again.ForeseenPorts) != 0 {
|
|
||||||
t.Fatalf("a second composition names %v foreseen: %v", again.ForeseenPorts, err)
|
|
||||||
}
|
|
||||||
// The number is the one the controller allocates in the same store.
|
|
||||||
plan, settings, err := resolveNode(ctx, open, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
old, err := declarationFor(ctx, open, "laptop", plan, settings)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
after, _, err := compose.ComposeIn(in.Input, "laptop", compose.Foreseeing)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, want := portsOf(after.Resources), portsOf(old.Resources); got != want {
|
|
||||||
t.Fatalf("the declaration publishes %s; the controller's allocation publishes %s", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func portsOf(resources []map[string]any) string {
|
|
||||||
var out []string
|
|
||||||
for _, r := range resources {
|
|
||||||
if r["type"] == "container" {
|
|
||||||
out = append(out, fmt.Sprint(r["name"], r["ports"]))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return strings.Join(out, ";")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1385/AC2: an own secret named foreseen is made with SecretForModule, and a pair credential the Input names would-make is
|
|
||||||
// made as planFor chose; both exist after the send's composition and a second composition names neither.
|
|
||||||
func TestPerformMakesTheOwnSecretsAndThePairCredentialsTheCompositionNamed(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
register(t, open, aWebModule())
|
|
||||||
register(t, open, aConsumerOfTheBucket("reports"))
|
|
||||||
for _, m := range []string{"web", "reports"} {
|
|
||||||
if _, err := assign(ctx, open, "laptop", m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
before, err := madeIn(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
declared, err := composeSend(ctx, open, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("the send is refused: %v", err)
|
|
||||||
}
|
|
||||||
if len(declared.foreseen) != 0 {
|
|
||||||
t.Fatalf("the declaration still names %v foreseen", declared.foreseen)
|
|
||||||
}
|
|
||||||
after, err := madeIn(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
madeNow := func(have, was []string) []string {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for _, s := range was {
|
|
||||||
seen[s] = true
|
|
||||||
}
|
|
||||||
var out []string
|
|
||||||
for _, s := range have {
|
|
||||||
if !seen[s] {
|
|
||||||
out = append(out, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
if got := madeNow(after.OwnSecrets, before.OwnSecrets); len(got) != 1 || !strings.Contains(got[0], "web/admin") {
|
|
||||||
t.Errorf("the own secrets made are %v, want laptop web/admin", got)
|
|
||||||
}
|
|
||||||
if got := madeNow(after.Pairs, before.Pairs); len(got) != 1 || !strings.Contains(got[0], "reports") {
|
|
||||||
t.Errorf("the pair credentials made are %v, want the one of reports", got)
|
|
||||||
}
|
|
||||||
// And the credential is the one a second composition reads as held.
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(in.wouldMake) != 0 {
|
|
||||||
t.Errorf("after the send the next send would still make %v", in.wouldMake)
|
|
||||||
}
|
|
||||||
if _, err := inv.SecretsFrom(ctx, "anchor"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1385/AC3 (the lead's answer 1): with the vault unreachable and no box of the credential kept, the send carries the
|
|
||||||
// controller's own seal for it (ADR 0396), says so as a condition, and keeps no box.
|
|
||||||
func TestAVaultThatIsSilentSealsTheSendByTheControllerWhereNoBoxIsKept(t *testing.T) {
|
|
||||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
|
||||||
vault.silent = errors.New("no responders")
|
|
||||||
declared, err := composeSend(t.Context(), open, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("the send is refused: %v", err)
|
|
||||||
}
|
|
||||||
f := clientsBroker(declared.Resources)
|
|
||||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
|
||||||
t.Fatalf("the core credential is composed as %v: the controller's own seal, until the vault holds it", f)
|
|
||||||
}
|
|
||||||
if len(declared.foreseen) != 0 {
|
|
||||||
t.Fatalf("the declaration still names %v foreseen", declared.foreseen)
|
|
||||||
}
|
|
||||||
if c, found := vaultSilence(t, "laptop"); !found || !strings.Contains(c.Summary, "sealed by the controller until the vault holds it") {
|
|
||||||
t.Fatalf("no condition says why: %+v", c)
|
|
||||||
}
|
|
||||||
kept, err := open.inventory.KeptCoreBoxes(t.Context(), "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(kept["mesh-provisioner-client"]) != 0 {
|
|
||||||
t.Fatalf("a box was kept for a credential the vault never boxed: %v", kept)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1385/AC6: perform is idempotent: a second send's composition over the stores it left makes and records nothing.
|
|
||||||
func TestAFinishedSendMakesNothingTheSecondTime(t *testing.T) {
|
|
||||||
open, _ := aMeshWithSecrets(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aWebModule())
|
|
||||||
if _, err := assign(ctx, open, "laptop", "web"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := composeSend(ctx, open, "laptop"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
before := tableState(t, open.inventory)
|
|
||||||
if _, err := composeSend(ctx, open, "laptop"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for k, v := range tableState(t, open.inventory) {
|
|
||||||
if before[k] != v {
|
|
||||||
t.Errorf("a second send of the same machine moved %s", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Review round 1, finding 1: a machine that rejoined with a new sealing key keeps its made rows sealed to the old key. A
|
|
||||||
// send re-makes them for the new key, as SecretForModule did on main, and the file naming the secret is not refused.
|
|
||||||
func TestASendReMakesAnOwnSecretSealedToAKeyTheMachineNoLongerHas(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
register(t, open, aBot("bot", false))
|
|
||||||
if _, err := assign(ctx, open, "laptop", "bot"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := composeSend(ctx, open, "laptop"); err != nil {
|
|
||||||
t.Fatalf("the first send is refused: %v", err)
|
|
||||||
}
|
|
||||||
if err := inv.RecordSealingKey(ctx, nodeID(t, open, "laptop"), aPublicKey(t)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := composeSend(ctx, open, "laptop"); err != nil {
|
|
||||||
t.Fatalf("a machine that rejoined cannot be sent to: %v", err)
|
|
||||||
}
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
n, _ := nodeOf(in.Input, "laptop")
|
|
||||||
found := false
|
|
||||||
for _, r := range in.OwnSecrets {
|
|
||||||
if r.Node == "laptop" && r.Module == "bot" && r.Name == "token" {
|
|
||||||
found = true
|
|
||||||
if r.SealedTo != n.SealingKey {
|
|
||||||
t.Fatalf("the own secret is still sealed to a key the machine no longer has")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("the own secret has no row")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,506 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/validate"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The controller's composition on mesh-sdk's (novox/hq ADR 0453 clause 3, design 47 R4b, issues 1386 to 1388): every
|
|
||||||
// caller that planned or composed a node reads the Input once for its operation (compose_input.go), asks the SDK to
|
|
||||||
// compose over it, and, when it sends, performs what the composition names foreseen (compose_perform.go) and composes again.
|
|
||||||
|
|
||||||
// inputCacheKey carries an operation's Input in its context, so the planning of every node of one verb reads the stores once.
|
|
||||||
type inputCacheKey struct{}
|
|
||||||
|
|
||||||
type inputCache struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
by map[string]*meshInput
|
|
||||||
}
|
|
||||||
|
|
||||||
// withInputCache is ctx for an operation: the Input read for it is kept until a send changes what it read. A context that
|
|
||||||
// carries none reads the stores for each composition, which is correct and slow.
|
|
||||||
func withInputCache(ctx context.Context) context.Context {
|
|
||||||
if _, ok := ctx.Value(inputCacheKey{}).(*inputCache); ok {
|
|
||||||
return ctx
|
|
||||||
}
|
|
||||||
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// freshInput is ctx for one composition whose Input is read afresh and kept only until the composition makes something: a
|
|
||||||
// send that is composed after another one made and recorded things (a push's machines, one after the other).
|
|
||||||
func freshInput(ctx context.Context) context.Context {
|
|
||||||
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs
|
|
||||||
// (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send.
|
|
||||||
func keptSignature(ctx context.Context) string {
|
|
||||||
skip, on := keptExcept(ctx)
|
|
||||||
var names []string
|
|
||||||
for m := range skip {
|
|
||||||
names = append(names, m)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
every, _ := ctx.Value(keepEveryBuildKey{}).(string)
|
|
||||||
settings, _ := ctx.Value(keepForSettingsKey{}).(string)
|
|
||||||
return fmt.Sprintf("kept=%v;except=%v;every=%s;settings=%s", on, names, every, settings)
|
|
||||||
}
|
|
||||||
|
|
||||||
// inputFor is the Input for ctx's operation.
|
|
||||||
func inputFor(ctx context.Context, open *stores) (*meshInput, error) {
|
|
||||||
cache, ok := ctx.Value(inputCacheKey{}).(*inputCache)
|
|
||||||
if !ok {
|
|
||||||
return readComposeInput(ctx, open)
|
|
||||||
}
|
|
||||||
sig := keptSignature(ctx)
|
|
||||||
cache.mu.Lock()
|
|
||||||
defer cache.mu.Unlock()
|
|
||||||
if in := cache.by[sig]; in != nil {
|
|
||||||
return in, nil
|
|
||||||
}
|
|
||||||
in, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
cache.by[sig] = in
|
|
||||||
return in, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgetInput drops what the operation read, because a send made or recorded something it read.
|
|
||||||
func forgetInput(ctx context.Context) {
|
|
||||||
if cache, ok := ctx.Value(inputCacheKey{}).(*inputCache); ok {
|
|
||||||
cache.mu.Lock()
|
|
||||||
cache.by = map[string]*meshInput{}
|
|
||||||
cache.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveNode works out everything a node should run, from what was assigned to it (planFor on the SDK): over the Input,
|
|
||||||
// writing nothing. A failure to compose the node's own modules is marked notResolvable, as planFor marked it; every other
|
|
||||||
// failure is returned as it is.
|
|
||||||
func resolveNode(ctx context.Context, open *stores, node string) (catalogue.Resolution, catalogue.SettingsBy, error) {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Resolution{}, nil, err
|
|
||||||
}
|
|
||||||
if err := in.refused[node]; err != nil {
|
|
||||||
return catalogue.Resolution{}, nil, err
|
|
||||||
}
|
|
||||||
resolved, err := compose.Resolve(in.Input, node)
|
|
||||||
if err != nil {
|
|
||||||
if compose.IsUnresolvable(err) {
|
|
||||||
return catalogue.Resolution{}, nil, notResolvable{err}
|
|
||||||
}
|
|
||||||
return catalogue.Resolution{}, nil, err
|
|
||||||
}
|
|
||||||
logUnheld(node, resolved.Plan.Unheld)
|
|
||||||
return resolved.Plan, resolved.Settings, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// generatorsOf is what every node's declaration is composed with, over the whole mesh (generators on the SDK).
|
|
||||||
func generatorsOf(ctx context.Context, open *stores) (map[string]catalogue.Generator, error) {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return compose.Generators(in.Input)
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderingOf is what a machine is rendered with as a question reads it (nothing made), with the private network: what the
|
|
||||||
// filter's rules, the roster and the like are derived from.
|
|
||||||
func renderingOf(ctx context.Context, open *stores, node string) (catalogue.Rendering, error) {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Rendering{}, err
|
|
||||||
}
|
|
||||||
rendered, err := compose.Render(in.Input, node, compose.Reading)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Rendering{}, markUnresolvable(err)
|
|
||||||
}
|
|
||||||
with := rendered.Rendering
|
|
||||||
if with.Generators, err = compose.Generators(in.Input); err != nil {
|
|
||||||
return catalogue.Rendering{}, err
|
|
||||||
}
|
|
||||||
return with, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// whoResolvesOn is the machines whose resolution answers a requirement, and why the others did not (whoResolves over the
|
|
||||||
// Input): by what a module provides, not by its name. A machine whose set does not compose is in refused, said; one the mesh
|
|
||||||
// could not be asked about is an error, never a refusal (novox/hq 04-ISSUES/152).
|
|
||||||
func whoResolvesOn(ctx context.Context, open *stores, requirement string) (map[string]bool, map[string]string, error) {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
on := map[string]bool{}
|
|
||||||
refused := map[string]string{}
|
|
||||||
for _, n := range in.sortedNodes() {
|
|
||||||
if err := in.refused[n.Name]; err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("whether %s answers %q cannot be read: %w", n.Name, requirement, err)
|
|
||||||
}
|
|
||||||
resolved, err := compose.Resolve(in.Input, n.Name)
|
|
||||||
switch {
|
|
||||||
case compose.IsUnresolvable(err):
|
|
||||||
refused[n.Name] = err.Error()
|
|
||||||
continue
|
|
||||||
case err != nil:
|
|
||||||
return nil, nil, fmt.Errorf("whether %s answers %q cannot be read: %w", n.Name, requirement, err)
|
|
||||||
}
|
|
||||||
for _, m := range resolved.Plan.Modules {
|
|
||||||
for _, offered := range m.Offers() {
|
|
||||||
if offered == requirement {
|
|
||||||
on[n.Name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return on, refused, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sortedNodes is the Input's nodes by name, the order the controller reads them in.
|
|
||||||
func (in *meshInput) sortedNodes() []compose.Node {
|
|
||||||
out := append([]compose.Node(nil), in.Nodes...)
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// composeQuestion composes a node's declaration as a question asks it: nothing is made, and what a send would make is a
|
|
||||||
// stand-in (Foreseeing) or left out (Reading). It also answers what the node-engine's validator refuses of it. The lines the
|
|
||||||
// composition found are said where the controller says its own (standard error, or the writer an act gave).
|
|
||||||
//
|
|
||||||
// withOrder says the declaration carries the order it was last sent under (composedAndValidated's, which a comparison with the
|
|
||||||
// last send needs); without it the order is none, as declarationWith left it for the sender to stamp.
|
|
||||||
func composeQuestion(ctx context.Context, open *stores, node string, mode compose.Mode, withOrder bool) (sendable, []string, error) {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
if err := in.refused[node]; err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
d, problems, err := compose.ComposeIn(in.Input, node, mode)
|
|
||||||
if err != nil {
|
|
||||||
return sendable{}, nil, markUnresolvable(err)
|
|
||||||
}
|
|
||||||
if err := refusedBus(in, node); err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
sayFound(ctx, d)
|
|
||||||
out := sendableOf(d)
|
|
||||||
out.fileReferences = readsFileReferences(in.Input, node)
|
|
||||||
if !withOrder {
|
|
||||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
|
||||||
}
|
|
||||||
return out, problems, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedBus is why the node cannot be composed because it runs the bus and the bus's user list cannot be: no user has a
|
|
||||||
// credential, so the list would refuse every connection in the mesh (busUserList). The controller's own composition refused that
|
|
||||||
// machine with those words; no other machine is refused for it. nil when nothing refuses the node.
|
|
||||||
func refusedBus(in *meshInput, node string) error {
|
|
||||||
if in.busRefused == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
resolved, err := compose.Resolve(in.Input, node)
|
|
||||||
if err != nil {
|
|
||||||
return nil // it does not compose for its own reasons, which are said
|
|
||||||
}
|
|
||||||
for _, m := range resolved.Plan.Modules {
|
|
||||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
|
||||||
return in.busRefused
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// markUnresolvable is an error of a node whose own set does not compose, as planFor marked it.
|
|
||||||
func markUnresolvable(err error) error {
|
|
||||||
if compose.IsUnresolvable(err) {
|
|
||||||
return notResolvable{err}
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// composeSend composes a node's declaration to send it, as the Allocating composition did: composed over what is read,
|
|
||||||
// then what it named foreseen is made (perform), the stores are read again and it is composed again. A declaration that
|
|
||||||
// still names anything foreseen is refused, said, and never sent.
|
|
||||||
func composeSend(ctx context.Context, open *stores, node string) (sendable, error) {
|
|
||||||
declared, _, err := composeSendValidated(ctx, open, node)
|
|
||||||
if err != nil {
|
|
||||||
return declared, err
|
|
||||||
}
|
|
||||||
// Never a declaration naming an image the store does not hold (novox/hq issue 1742): that module is left out.
|
|
||||||
return withoutMissingImages(ctx, open, node, declared), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// composeSendValidated is composeSend and what the node-engine's validator refuses of the declaration it composed, for the
|
|
||||||
// merge gate's two passes (composedAndValidated in its Allocating mode).
|
|
||||||
func composeSendValidated(ctx context.Context, open *stores, node string) (sendable, []string, error) {
|
|
||||||
// The core modules whose credentials the controller sealed itself because the vault was silent and no box was kept
|
|
||||||
// (ADR 0396): the composition stands in a vault box for each, and a send carries the controller's own seal instead.
|
|
||||||
byController, err := prepareSend(ctx, open, node)
|
|
||||||
if err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
for attempt := 0; ; attempt++ {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
if err := in.refused[node]; err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
d, problems, err := compose.ComposeIn(in.Input, node, compose.Foreseeing)
|
|
||||||
if err != nil {
|
|
||||||
return sendable{}, nil, markUnresolvable(err)
|
|
||||||
}
|
|
||||||
if err := refusedBus(in, node); err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
if len(byController) > 0 {
|
|
||||||
if d, problems, err = sealedByTheController(in, node, d, byController); err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
left := stillForeseen(d)
|
|
||||||
resolved, rerr := compose.Resolve(in.Input, node)
|
|
||||||
if rerr != nil {
|
|
||||||
return sendable{}, nil, markUnresolvable(rerr)
|
|
||||||
}
|
|
||||||
if len(resolved.Foreseen) > 0 {
|
|
||||||
left = append(left, resolved.Foreseen...)
|
|
||||||
}
|
|
||||||
if len(left) == 0 && len(in.wouldMake) == 0 {
|
|
||||||
sayFound(ctx, d)
|
|
||||||
out := sendableOf(d)
|
|
||||||
out.fileReferences = readsFileReferences(in.Input, node)
|
|
||||||
// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
|
|
||||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
|
||||||
here, _ := nodeOf(in.Input, node)
|
|
||||||
out.Builds = buildsOf(in.Input, here, resolved.Plan, d.LeftOutWhy)
|
|
||||||
out.Bindings = boundToData(resolved.Plan, d.LeftOutWhy)
|
|
||||||
if len(d.Filled) > 0 {
|
|
||||||
if out.plans, err = handedPlans(in, node, resolved); err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out, problems, nil
|
|
||||||
}
|
|
||||||
if attempt > 0 {
|
|
||||||
return sendable{}, nil, fmt.Errorf("%s was composed twice and still names what only a send makes (%s): nothing is sent "+
|
|
||||||
"that stands in for a value", node, strings.Join(left, "; "))
|
|
||||||
}
|
|
||||||
if _, err := perform(ctx, open, in, node, d); err != nil {
|
|
||||||
return sendable{}, nil, err
|
|
||||||
}
|
|
||||||
// What was made is in the stores now: the next read is a fresh one.
|
|
||||||
forgetInput(ctx)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// restOfTheMesh is what every other machine holds and offers (theRestOfTheMesh over the Input), for a question: the lines it
|
|
||||||
// finds are said where the controller says its own, and a seat it would pass on is passed by the next send (perform), not by
|
|
||||||
// the question. shelf, when given, replaces the Input's catalogue: a caller composing a change's shelf.
|
|
||||||
func restOfTheMesh(ctx context.Context, open *stores, shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {
|
|
||||||
in, err := inputFor(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.World{}, err
|
|
||||||
}
|
|
||||||
input := in.Input
|
|
||||||
if shelf != nil {
|
|
||||||
input.Catalogue = shelf
|
|
||||||
}
|
|
||||||
world, passes, left, err := compose.TheRestOfTheMesh(input, exclude)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.World{}, err
|
|
||||||
}
|
|
||||||
var w io.Writer = os.Stderr
|
|
||||||
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
|
|
||||||
w = lw
|
|
||||||
}
|
|
||||||
for _, l := range left {
|
|
||||||
fmt.Fprintln(w, l.String())
|
|
||||||
}
|
|
||||||
for _, line := range passes.Said {
|
|
||||||
fmt.Fprintln(w, line)
|
|
||||||
}
|
|
||||||
return world, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// restOfTheMeshOf is restOfTheMesh for a caller that holds the inventory alone: the other stores the Input reads are opened for
|
|
||||||
// the call and let go of again.
|
|
||||||
func restOfTheMeshOf(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {
|
|
||||||
open := &stores{inventory: inv}
|
|
||||||
defer func() {
|
|
||||||
if open.licences != nil {
|
|
||||||
open.licences.Close()
|
|
||||||
}
|
|
||||||
if open.identity != nil {
|
|
||||||
open.identity.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return restOfTheMesh(ctx, open, shelf, exclude)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sayFound says the lines a composition found while composing, once each: a machine left out of the rest of the mesh, a seat
|
|
||||||
// that two claimants could take, the bus's users left out for want of a credential.
|
|
||||||
func sayFound(ctx context.Context, d compose.Declared) {
|
|
||||||
if len(d.Said) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var w io.Writer = os.Stderr
|
|
||||||
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
|
|
||||||
w = lw
|
|
||||||
}
|
|
||||||
for _, line := range d.Said {
|
|
||||||
fmt.Fprintln(w, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendableOf is a composed declaration as the controller's sends know it.
|
|
||||||
func sendableOf(d compose.Declared) sendable {
|
|
||||||
out := sendable{Resources: compactKept(d.Resources), Sequence: d.Sequence, Epoch: d.Epoch, Generation: d.Generation,
|
|
||||||
ReadsModules: d.ReadsModules, moduleOf: d.ModuleOf, buildIDs: d.BuildIDs, modules: d.Modules,
|
|
||||||
localNetwork: d.LocalNetwork, Received: d.Received, Mesh: d.Mesh, BusUsers: d.BusUsers,
|
|
||||||
LeftOut: d.LeftOut, leftOutWhy: d.LeftOutWhy, withheld: d.Withheld, unbound: d.Unbound, foreseen: d.Foreseen,
|
|
||||||
unplaced: d.Unplaced, handedOnly: d.HandedOnly, Filled: d.Filled, Provisioner: d.Provisioner,
|
|
||||||
requiredBy: d.RequiredBy}
|
|
||||||
if d.Adoption != nil {
|
|
||||||
out.Adoption = &adoptionEnvelope{Taken: d.Adoption.Taken, Untaken: d.Adoption.Untaken}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildsOf is the build of each module a declaration for the node carries, as a send records it (novox/hq issue 259): the
|
|
||||||
// module's current build for each module in it, the build it was last sent for one left out of it, and a module kept at the
|
|
||||||
// build the machine runs carried at that one (ADR 0245). Never nil.
|
|
||||||
func buildsOf(in compose.Input, node compose.Node, plan catalogue.Resolution, leftOut map[string]string) map[string]string {
|
|
||||||
before := node.SentBuilds
|
|
||||||
if !node.SentKnown {
|
|
||||||
before = nil
|
|
||||||
}
|
|
||||||
out := map[string]string{}
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
if _, left := leftOut[m.Module]; left {
|
|
||||||
if was, kept := before[m.Module]; kept {
|
|
||||||
out[m.Module] = was
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out[m.Module] = in.CurrentBuilds[m.Module]
|
|
||||||
}
|
|
||||||
for m, was := range node.KeptBuilds {
|
|
||||||
if _, carried := out[m]; carried {
|
|
||||||
out[m] = was
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// handedPlans is the plan of each module handed to the provisioner, which the assignments mirror keeps with the send (design
|
|
||||||
// 57 P6): the composition's declaration does not carry it, so it is composed from the same rendering again.
|
|
||||||
func handedPlans(in *meshInput, node string, resolved compose.Resolved) (map[string]catalogue.Plan, error) {
|
|
||||||
rendered, err := compose.Render(in.Input, node, compose.Foreseeing)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
gens, err := compose.Generators(in.Input)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
with := rendered.Rendering
|
|
||||||
with.Generators = gens
|
|
||||||
composed, err := resolved.Plan.Compose(with)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return composed.Plans, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sealedByTheController is the declaration with the controller's own seal in place of the vault's box for each core module the
|
|
||||||
// vault did not answer for and no box was kept for (core_box.go, ADR 0396): the file carries "sealed", as the controller's
|
|
||||||
// composition sent it, until the vault holds the credential. What the composition stood in for the box is replaced and no longer
|
|
||||||
// wanted; the validator is run again over the body.
|
|
||||||
func sealedByTheController(in *meshInput, node string, d compose.Declared, modules []string) (compose.Declared, []string, error) {
|
|
||||||
rendered, err := compose.Render(in.Input, node, compose.Foreseeing)
|
|
||||||
if err != nil {
|
|
||||||
return d, nil, err
|
|
||||||
}
|
|
||||||
stood, _ := compactBox(catalogue.ForeseenVaultBox)
|
|
||||||
var wanted []compose.CoreBoxWanted
|
|
||||||
var foreseen []string
|
|
||||||
for _, w := range d.CoreBoxesWanted {
|
|
||||||
if !slices.Contains(modules, w.Module) {
|
|
||||||
wanted = append(wanted, w)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
sealed := rendered.Rendering.Needed[w.Module][w.Name]
|
|
||||||
if sealed == "" {
|
|
||||||
return d, nil, fmt.Errorf("%s/%s on %s has no seal of the controller's to carry until the vault holds it", w.Module, w.Name, node)
|
|
||||||
}
|
|
||||||
for _, r := range d.Resources {
|
|
||||||
box, isBox := r[catalogue.VaultSealedField]
|
|
||||||
// By the resource's id alone: its path is the manifest's with the machine's words filled in, and so is not the
|
|
||||||
// manifest's own.
|
|
||||||
if !isBox || r["id"] != w.Module+"."+catalogue.NeedID(w.Name) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if had, err := compactBox(box); err != nil || had != stood {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
delete(r, catalogue.VaultSealedField)
|
|
||||||
r["sealed"] = sealed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A stand-in box that is left is said by its resource, never sent and never guessed at.
|
|
||||||
for _, r := range d.Resources {
|
|
||||||
if box, has := r[catalogue.VaultSealedField]; has {
|
|
||||||
if had, err := compactBox(box); err == nil && had == stood {
|
|
||||||
if id, _ := r["id"].(string); slices.ContainsFunc(modules, func(m string) bool { return strings.HasPrefix(id, m+".") }) {
|
|
||||||
return d, nil, fmt.Errorf("%s on %s still carries a stand-in for the vault's box, and the controller has "+
|
|
||||||
"no seal of its own for it", id, node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
d.CoreBoxesWanted = wanted
|
|
||||||
for _, f := range d.Foreseen {
|
|
||||||
module, _, _ := strings.Cut(f, "/")
|
|
||||||
if !slices.Contains(modules, module) {
|
|
||||||
foreseen = append(foreseen, f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
d.Foreseen = foreseen
|
|
||||||
body, err := d.Body()
|
|
||||||
if err != nil {
|
|
||||||
return d, nil, err
|
|
||||||
}
|
|
||||||
return d, validate.Declaration(body), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// compactBox is a value as the compact JSON it marshals to, so a box is compared by what it says and not by the type that
|
|
||||||
// holds it.
|
|
||||||
func compactBox(v any) (string, error) {
|
|
||||||
raw, err := json.Marshal(v)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := json.Compact(&out, raw); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return out.String(), nil
|
|
||||||
}
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A declaration as the SDK composes it is the one the controller sends, byte for byte, whatever it carries: the body is the
|
|
||||||
// envelope's only marshaller on either side (design 47 R4b, issue 1386).
|
|
||||||
func TestASendableIsTheComposedDeclarationByteForByte(t *testing.T) {
|
|
||||||
d := compose.Declared{
|
|
||||||
Resources: []map[string]any{{"id": "files.config", "type": "file"}, {"id": "mesh.hosts", "type": "file"}},
|
|
||||||
Sequence: 7, Epoch: 3, Generation: 9, ReadsModules: true,
|
|
||||||
ModuleOf: map[string]string{"files.config": "files"}, BuildIDs: map[string]string{"files": "b-1"},
|
|
||||||
Adoption: &compose.Adoption{Taken: []string{"files"}, Untaken: map[string][]string{"nats": {"nats.server"}}},
|
|
||||||
Filled: []string{"/etc/files/filled.json"}, Provisioner: "mesh-provisioner",
|
|
||||||
LeftOut: []string{"cache"}, LeftOutWhy: map[string]string{"cache": "a setting cannot compose"},
|
|
||||||
}
|
|
||||||
want, err := d.Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, err := sendableOf(d).Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if string(got) != string(want) {
|
|
||||||
t.Fatalf("the controller would send\n%s\nand the composition made\n%s", got, want)
|
|
||||||
}
|
|
||||||
if empty, err := sendableOf(compose.Declared{}).Body(); err != nil || !strings.Contains(string(empty), `"owns_nothing":true`) {
|
|
||||||
t.Fatalf("an empty declaration is not deliberate: %s %v", empty, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1386: what a send records as carried: the current build of each module it carries, the build a module left out of it was last
|
|
||||||
// sent (when that is known), and a module kept at the build the machine runs at that one. (The controller's carriedOn recorded
|
|
||||||
// the same; mesh-sdk's node_test TestAModuleLeftOutKeepsTheBuildItWasLastSent holds the left-out case.)
|
|
||||||
func TestWhatASendRecordsAsCarriedIsWhatTheControllerRecorded(t *testing.T) {
|
|
||||||
plan := catalogue.Resolution{Modules: []catalogue.Manifest{{Module: "postgres"}, {Module: "files"}, {Module: "cache"}}}
|
|
||||||
leftOut := map[string]string{"cache": "a setting cannot compose"}
|
|
||||||
for name, c := range map[string]struct {
|
|
||||||
sent map[string]string
|
|
||||||
known bool
|
|
||||||
kept map[string]string
|
|
||||||
want map[string]string
|
|
||||||
}{
|
|
||||||
"nothing sent": {want: map[string]string{"postgres": "p2", "files": "f2"}},
|
|
||||||
"sent and kept": {sent: map[string]string{"postgres": "p1", "cache": "c1", "files": "f1"}, known: true,
|
|
||||||
kept: map[string]string{"postgres": "p1"}, want: map[string]string{"postgres": "p1", "files": "f2", "cache": "c1"}},
|
|
||||||
"sent but not known": {sent: map[string]string{"cache": "c1"}, want: map[string]string{"postgres": "p2", "files": "f2"}},
|
|
||||||
"kept a module not in": {known: true, kept: map[string]string{"mail": "m1"}, want: map[string]string{"postgres": "p2", "files": "f2"}},
|
|
||||||
} {
|
|
||||||
in := compose.Input{CurrentBuilds: map[string]string{"postgres": "p2", "files": "f2", "cache": "c2"}}
|
|
||||||
node := compose.Node{SentBuilds: c.sent, SentKnown: c.known, KeptBuilds: c.kept}
|
|
||||||
if got := buildsOf(in, node, plan, leftOut); !reflect.DeepEqual(got, c.want) {
|
|
||||||
t.Errorf("%s: a send records %v, want %v", name, got, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1385: a declaration that stands in for a value anywhere is still foreseen, in a list the composition names or in the body
|
|
||||||
// alone (a pair credential's stand-in is in no list); one that stands in for nothing is not. It failed first, before the body
|
|
||||||
// was scanned.
|
|
||||||
func TestADeclarationThatStandsInForAValueIsStillForeseen(t *testing.T) {
|
|
||||||
clean := compose.Declared{Resources: []map[string]any{{"id": "files.config", "type": "file", "content": "ok"}}}
|
|
||||||
if left := stillForeseen(clean); len(left) != 0 {
|
|
||||||
t.Fatalf("a declaration that stands in for nothing is foreseen: %v", left)
|
|
||||||
}
|
|
||||||
for name, d := range map[string]compose.Declared{
|
|
||||||
"an own secret": {StoodIn: []string{"files/admin"}},
|
|
||||||
"a port": {ForeseenPorts: []string{"files:80"}},
|
|
||||||
"a core box": {CoreBoxesWanted: []compose.CoreBoxWanted{{Node: "anchor", Module: "mesh-vault", Name: "root"}}},
|
|
||||||
"a pass of a seat": {Passes: compose.Passes{Passes: []catalogue.SeatPass{{Seat: catalogue.Seat{Name: "mesh-store"}}}}},
|
|
||||||
"a pair credential": {Resources: []map[string]any{{"id": "files.needs-db", "sealed": catalogue.ForeseenSealed}}},
|
|
||||||
"the certificate": {Resources: []map[string]any{{"id": "proxy.cert", "content": compose.ForeseenCertificate}}},
|
|
||||||
"the bus's list": {Resources: []map[string]any{{"id": "nats.users", "content": compose.ForeseenBusUsers}}},
|
|
||||||
"the vault's box": {Resources: []map[string]any{{"id": "x", "box": catalogue.ForeseenVaultBox}}},
|
|
||||||
} {
|
|
||||||
if left := stillForeseen(d); len(left) == 0 {
|
|
||||||
t.Errorf("%s: a declaration that stands in for it is not foreseen", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1386: an Input read for a person's push is never the one read for a send that keeps recorded builds, and the cache is
|
|
||||||
// emptied by what a send made.
|
|
||||||
func TestAnInputIsCachedPerWhatItKeepsAndForgottenAfterASend(t *testing.T) {
|
|
||||||
ctx := withInputCache(context.Background())
|
|
||||||
plain := keptSignature(ctx)
|
|
||||||
kept := keptSignature(keepingRecorded(ctx))
|
|
||||||
bus := keptSignature(keepingRecorded(ctx, "nats"))
|
|
||||||
every := keptSignature(keepingEveryBuild(ctx, "anchor"))
|
|
||||||
for name, sig := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} {
|
|
||||||
for other, o := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} {
|
|
||||||
if name != other && sig == o {
|
|
||||||
t.Errorf("%s and %s read the same Input", name, other)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cache := ctx.Value(inputCacheKey{}).(*inputCache)
|
|
||||||
cache.by[plain] = &meshInput{}
|
|
||||||
forgetInput(ctx)
|
|
||||||
if len(cache.by) != 0 {
|
|
||||||
t.Fatal("what a send made left the Input it read in the cache")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,76 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The looks of the open conditions — how often each was observed, when last, its newest evidence — are
|
|
||||||
// kept in the controller's store, not the conditions bucket (novox/hq issue 420, ADR 0289 §1): the bucket
|
|
||||||
// is written when what a condition says changes, never for a look that saw it as it was.
|
|
||||||
|
|
||||||
// storeLooks keeps the conditions' looks in the inventory.
|
|
||||||
type storeLooks struct{ inv *inventory.Inventory }
|
|
||||||
|
|
||||||
func (s storeLooks) KeepLooks(ctx context.Context, key string, l conditions.Looks) error {
|
|
||||||
evidence, err := json.Marshal(l.Evidence)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return s.inv.KeepConditionLooks(ctx, inventory.ConditionLooks{Key: key, Observations: l.Observations,
|
|
||||||
LastObserved: l.LastObserved, Evidence: evidence})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s storeLooks) Looked(ctx context.Context, key string) (conditions.Looks, bool, error) {
|
|
||||||
row, kept, err := s.inv.ConditionLooksOf(ctx, key)
|
|
||||||
if err != nil || !kept {
|
|
||||||
return conditions.Looks{}, kept, err
|
|
||||||
}
|
|
||||||
l, err := looksOf(row)
|
|
||||||
return l, err == nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s storeLooks) AllLooks(ctx context.Context) (map[string]conditions.Looks, error) {
|
|
||||||
rows, err := s.inv.AllConditionLooks(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out := make(map[string]conditions.Looks, len(rows))
|
|
||||||
for key, row := range rows {
|
|
||||||
l, err := looksOf(row)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out[key] = l
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s storeLooks) ForgetLooks(ctx context.Context, key string) error {
|
|
||||||
return s.inv.ForgetConditionLooks(ctx, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func looksOf(row inventory.ConditionLooks) (conditions.Looks, error) {
|
|
||||||
l := conditions.Looks{Observations: row.Observations, LastObserved: row.LastObserved.UTC()}
|
|
||||||
if len(row.Evidence) > 0 {
|
|
||||||
if err := json.Unmarshal(row.Evidence, &l.Evidence); err != nil {
|
|
||||||
return conditions.Looks{}, fmt.Errorf("the evidence kept for the condition %s cannot be read: %w", row.Key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return l, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// looksHere is where a process that is not the serving controller reads the conditions' looks: the store,
|
|
||||||
// opened for it, closed by the function returned. A store that cannot be opened is said on every condition
|
|
||||||
// read through it, never read as none.
|
|
||||||
func looksHere(ctx context.Context) (conditions.LookStore, func()) {
|
|
||||||
inv, err := openInventory(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return conditions.NoLooks{Err: err}, func() {}
|
|
||||||
}
|
|
||||||
return storeLooks{inv}, inv.Close
|
|
||||||
}
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Looks at an unchanged condition write the bucket once, and `conditions` still says how often and when
|
|
||||||
// it was seen** (novox/hq issue 420): the looks are in the controller's store, and the verb, run as a
|
|
||||||
// process of its own, reads them from there. On 2026-10-10 one open condition had rewritten the bucket 7 003
|
|
||||||
// times in two days.
|
|
||||||
func TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds(t *testing.T) {
|
|
||||||
inv := inventory.ForTest(t)
|
|
||||||
bus := testbus.Start(t)
|
|
||||||
serving, err := broker.Dial(bus.ClientURL())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer serving.Close()
|
|
||||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx := t.Context()
|
|
||||||
store, history, err := conditions.OnTheBus(ctx, serving.Conn())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: storeLooks{inv}})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
look := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor",
|
|
||||||
Severity: conditions.Warning, Summary: "anchor has not been heard from", Source: "S1"}
|
|
||||||
for range 4 {
|
|
||||||
if _, err := k.Observe(ctx, look); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
api, err := jetstream.New(serving.Conn())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
stream, err := api.Stream(ctx, "KV_"+broker.ConditionsBucket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
info, err := stream.Info(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if info.State.LastSeq != 1 {
|
|
||||||
t.Fatalf("four looks at one unchanged condition wrote the bucket %d times, want 1", info.State.LastSeq)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The verb as a process other than the serving controller: it reads the bucket and the store.
|
|
||||||
keptBefore := conditionsFrom
|
|
||||||
conditionsFrom = nil
|
|
||||||
defer func() { conditionsFrom = keptBefore }()
|
|
||||||
t.Setenv(broker.NATSVar, bus.ClientURL())
|
|
||||||
t.Setenv(broker.CertificateVar, "")
|
|
||||||
var shown bytes.Buffer
|
|
||||||
if err := showCondition(ctx, []string{"machine.anchor.silent", "--json"}, &shown); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var c conditions.Condition
|
|
||||||
if err := json.Unmarshal(shown.Bytes(), &c); err != nil {
|
|
||||||
t.Fatalf("%v: %s", err, shown.String())
|
|
||||||
}
|
|
||||||
if c.Observations != 4 || len(c.Evidence) != 4 || c.LastObserved.IsZero() || c.LooksUnread != "" ||
|
|
||||||
time.Since(c.LastObserved) > time.Minute {
|
|
||||||
t.Fatalf("conditions show reads %d observation(s), %d evidence line(s), last %s (%q)", c.Observations,
|
|
||||||
len(c.Evidence), c.LastObserved, c.LooksUnread)
|
|
||||||
}
|
|
||||||
listed, err := openConditions(ctx)
|
|
||||||
if err != nil || len(listed) != 1 || listed[0].Observations != 4 {
|
|
||||||
t.Fatalf("conditions lists %+v: %v", listed, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,491 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
|
||||||
//
|
|
||||||
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
|
||||||
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
|
||||||
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
|
||||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its
|
|
||||||
// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go,
|
|
||||||
// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
|
||||||
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
|
||||||
// while, with a reason, and that is recorded as a hand act.
|
|
||||||
|
|
||||||
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
|
||||||
var conditionsFrom *conditions.Keeper
|
|
||||||
|
|
||||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with
|
|
||||||
// the conditions' looks kept in looks (condition_looks.go).
|
|
||||||
func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) {
|
|
||||||
store, history, err := conditions.OnTheBus(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
js, err := conn.JetStream()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks,
|
|
||||||
Teller: link.OverNATS{Conn: conn, JS: js},
|
|
||||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
|
||||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
|
||||||
// does nothing).
|
|
||||||
Changed: func() { statusFrom.nudge(); askerFrom.nudge() },
|
|
||||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
|
||||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
|
||||||
// it was given before it returns.
|
|
||||||
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
|
||||||
if conditionsFrom != nil {
|
|
||||||
return f(conditionsFrom)
|
|
||||||
}
|
|
||||||
looks, closeLooks := looksHere(ctx)
|
|
||||||
defer closeLooks()
|
|
||||||
return onTheBus(func(conn *nats.Conn) error {
|
|
||||||
k, err := keeperOn(ctx, conn, looks)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
k.Close(flushing)
|
|
||||||
}()
|
|
||||||
return f(k)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
|
||||||
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
|
||||||
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
|
||||||
if conditionsFrom != nil {
|
|
||||||
return conditionsFrom.Open(ctx)
|
|
||||||
}
|
|
||||||
if _, err := broker.BusAddress(); err != nil {
|
|
||||||
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
|
||||||
}
|
|
||||||
var out []conditions.Condition
|
|
||||||
looks, closeLooks := looksHere(ctx)
|
|
||||||
defer closeLooks()
|
|
||||||
err := onTheBus(func(conn *nats.Conn) error {
|
|
||||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
out, err = conditions.ReadWith(reading, store, looks)
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// conditionsUsage is how the verb is typed.
|
|
||||||
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
|
||||||
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
|
||||||
"conditions history [--days N] [--key K] [--json]"
|
|
||||||
|
|
||||||
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
|
||||||
func conditionsCommand(ctx context.Context, args []string) error {
|
|
||||||
sub := "list"
|
|
||||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
||||||
sub, args = args[0], args[1:]
|
|
||||||
}
|
|
||||||
switch sub {
|
|
||||||
case "list":
|
|
||||||
return listConditions(ctx, args, os.Stdout)
|
|
||||||
case "show":
|
|
||||||
return showCondition(ctx, args, os.Stdout)
|
|
||||||
case "silence":
|
|
||||||
return silenceCondition(ctx, args)
|
|
||||||
case "history":
|
|
||||||
return conditionHistory(ctx, args, os.Stdout)
|
|
||||||
}
|
|
||||||
return errors.New(conditionsUsage)
|
|
||||||
}
|
|
||||||
|
|
||||||
func listConditions(ctx context.Context, args []string, w io.Writer) error {
|
|
||||||
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
|
||||||
usageTo(set, w)
|
|
||||||
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
|
||||||
severity := set.String("severity", "", "only urgent, or only warning")
|
|
||||||
machine := set.String("machine", "", "only those about this machine")
|
|
||||||
asJSON := set.Bool("json", false, "as data")
|
|
||||||
if rest, err := parseAround(set, args); err != nil {
|
|
||||||
return err
|
|
||||||
} else if len(rest) > 0 {
|
|
||||||
return errors.New(conditionsUsage)
|
|
||||||
}
|
|
||||||
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
|
||||||
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
|
||||||
}
|
|
||||||
open, err := openConditions(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var out []conditions.Condition
|
|
||||||
for _, c := range open {
|
|
||||||
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
|
||||||
(*machine == "" || concerns(c, *machine)) {
|
|
||||||
out = append(out, c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if *asJSON {
|
|
||||||
return printJSONTo(w, map[string]any{"conditions": inBrief(out), "open": len(open), "counted": counted(out),
|
|
||||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; " +
|
|
||||||
"each with its newest evidence — `conditions key=<key>` gives one whole"})
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
if len(open) == 0 {
|
|
||||||
fmt.Fprintln(w, "no open conditions")
|
|
||||||
} else {
|
|
||||||
fmt.Fprintf(w, "none of the %d open condition(s) is about that\n", len(open))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, line := range conditionLines(out, time.Now()) {
|
|
||||||
fmt.Fprintln(w, line)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// briefEvidence is how many observations a condition carries in a list of them: its newest. The store
|
|
||||||
// keeps ten, and a list of hundreds of conditions with ten each was the larger half of an answer more
|
|
||||||
// than the bus carries in one message (novox/hq issue 314). One condition whole is `conditions key=`.
|
|
||||||
const briefEvidence = 1
|
|
||||||
|
|
||||||
// conditionInBrief is a condition as a list carries it: its newest evidence, with how much more there is
|
|
||||||
// and where to read it. Everything else of it, so a reader of the list — the operator's channel among
|
|
||||||
// them — reads the same condition it always did.
|
|
||||||
type conditionInBrief struct {
|
|
||||||
conditions.Condition
|
|
||||||
// EvidenceKept is how many observations the condition keeps, when the list shows fewer.
|
|
||||||
EvidenceKept int `json:"evidence-kept,omitempty"`
|
|
||||||
More string `json:"more,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// inBrief is a list of conditions in brief; never null.
|
|
||||||
func inBrief(list []conditions.Condition) []conditionInBrief {
|
|
||||||
out := make([]conditionInBrief, 0, len(list))
|
|
||||||
for _, c := range list {
|
|
||||||
b := conditionInBrief{Condition: c}
|
|
||||||
if len(c.Evidence) > briefEvidence {
|
|
||||||
b.EvidenceKept = len(c.Evidence)
|
|
||||||
b.Evidence = c.Evidence[:briefEvidence:briefEvidence]
|
|
||||||
b.More = c.Show()
|
|
||||||
}
|
|
||||||
out = append(out, b)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// counted is how many of a list each source raised of each kind — `D14 stalled: 689` — so a list of
|
|
||||||
// hundreds says its shape before its lines.
|
|
||||||
func counted(list []conditions.Condition) map[string]int {
|
|
||||||
out := map[string]int{}
|
|
||||||
for _, c := range list {
|
|
||||||
out[c.Source+" "+c.Kind]++
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// concerns says whether a condition is about a machine: it names it, or its key does.
|
|
||||||
func concerns(c conditions.Condition, machine string) bool {
|
|
||||||
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
for _, part := range strings.Split(c.Key, ".") {
|
|
||||||
if part == machine {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
|
||||||
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
|
||||||
var out []string
|
|
||||||
for _, c := range list {
|
|
||||||
times := ""
|
|
||||||
if c.Count > 1 {
|
|
||||||
times = fmt.Sprintf(", raised %d times", c.Count)
|
|
||||||
}
|
|
||||||
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
|
||||||
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
|
||||||
if c.SilencedAt(now) {
|
|
||||||
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
|
||||||
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func showCondition(ctx context.Context, args []string, w io.Writer) error {
|
|
||||||
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
|
||||||
usageTo(set, w)
|
|
||||||
asJSON := set.Bool("json", false, "as data")
|
|
||||||
rest, err := parseAround(set, args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(rest) != 1 {
|
|
||||||
return errors.New("conditions show <key>")
|
|
||||||
}
|
|
||||||
key := rest[0]
|
|
||||||
var c conditions.Condition
|
|
||||||
var found bool
|
|
||||||
if conditionsFrom != nil {
|
|
||||||
c, found, err = conditionsFrom.Get(ctx, key)
|
|
||||||
} else {
|
|
||||||
looks, closeLooks := looksHere(ctx)
|
|
||||||
defer closeLooks()
|
|
||||||
err = onTheBus(func(conn *nats.Conn) error {
|
|
||||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
c, found, err = conditions.ReadOneWith(ctx, store, looks, key)
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
|
||||||
"history --key %s` what became of it", key, key)
|
|
||||||
}
|
|
||||||
if *asJSON {
|
|
||||||
return printJSONTo(w, c)
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
fmt.Fprintf(w, "%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
|
||||||
fmt.Fprintf(w, " kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
|
||||||
if c.Subject.Machine != "" {
|
|
||||||
fmt.Fprintf(w, ", on %s", c.Subject.Machine)
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, "\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
|
||||||
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
|
||||||
now.Sub(c.LastObserved).Round(time.Second))
|
|
||||||
if c.Count > 1 {
|
|
||||||
fmt.Fprintf(w, " raised %d times, each within ten minutes of clearing\n", c.Count)
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, " resolved by %s\n", resolverWords(c.Resolver))
|
|
||||||
if c.Silenced != nil {
|
|
||||||
fmt.Fprintf(w, " silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
|
||||||
c.Silenced.By, c.Silenced.Why)
|
|
||||||
}
|
|
||||||
if len(c.Tried) > 0 {
|
|
||||||
fmt.Fprintln(w, "\n tried:")
|
|
||||||
for _, t := range c.Tried {
|
|
||||||
fmt.Fprintf(w, " %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Fprintln(w, "\n evidence, newest first:")
|
|
||||||
for _, e := range c.Evidence {
|
|
||||||
fmt.Fprintf(w, " %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolverWords(r string) string {
|
|
||||||
switch r {
|
|
||||||
case conditions.ResolverSelf:
|
|
||||||
return "itself: it clears when observation says it is resolved"
|
|
||||||
case conditions.ResolverOperator:
|
|
||||||
return "the operator: nothing in the mesh will repair it"
|
|
||||||
case conditions.ResolverAgent:
|
|
||||||
return "an agent"
|
|
||||||
}
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
|
||||||
// who and why before it is done, its cause the condition's kind unless one is given.
|
|
||||||
func silenceCondition(ctx context.Context, args []string) error {
|
|
||||||
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
|
||||||
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
|
||||||
acts := addHandActFlags(set)
|
|
||||||
rest, err := parseAround(set, args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(rest) != 1 {
|
|
||||||
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
|
||||||
}
|
|
||||||
key := rest[0]
|
|
||||||
if err := acts.require("conditions silence"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
d, err := parseFor(*forFlag)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if d > conditions.MaxSilence {
|
|
||||||
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
|
||||||
}
|
|
||||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
|
||||||
c, found, err := k.Get(ctx, key)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(*acts.cause) == "" {
|
|
||||||
*acts.cause = c.Kind
|
|
||||||
}
|
|
||||||
*acts.condition = key
|
|
||||||
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
|
||||||
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
|
||||||
"`status` still says it; it clears when observation says it is resolved\n",
|
|
||||||
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseFor reads a duration, days included.
|
|
||||||
func parseFor(s string) (time.Duration, error) {
|
|
||||||
s = strings.TrimSpace(s)
|
|
||||||
if s == "" {
|
|
||||||
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
|
||||||
}
|
|
||||||
if days, ok := strings.CutSuffix(s, "d"); ok {
|
|
||||||
n, err := strconv.Atoi(days)
|
|
||||||
if err != nil || n <= 0 {
|
|
||||||
return 0, fmt.Errorf("%q is not a number of days", s)
|
|
||||||
}
|
|
||||||
return time.Duration(n) * 24 * time.Hour, nil
|
|
||||||
}
|
|
||||||
d, err := time.ParseDuration(s)
|
|
||||||
if err != nil || d <= 0 {
|
|
||||||
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
|
||||||
}
|
|
||||||
return d, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func conditionHistory(ctx context.Context, args []string, w io.Writer) error {
|
|
||||||
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
|
||||||
usageTo(set, w)
|
|
||||||
days := set.Int("days", 7, "how many days back, at most 90")
|
|
||||||
key := set.String("key", "", "only this condition")
|
|
||||||
asJSON := set.Bool("json", false, "as data")
|
|
||||||
if rest, err := parseAround(set, args); err != nil {
|
|
||||||
return err
|
|
||||||
} else if len(rest) > 0 {
|
|
||||||
return errors.New("conditions history [--days N] [--key K] [--json]")
|
|
||||||
}
|
|
||||||
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
|
||||||
var events []conditions.Event
|
|
||||||
read := func(h conditions.History) error {
|
|
||||||
var err error
|
|
||||||
events, err = h.Since(ctx, since)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var err error
|
|
||||||
if conditionsFrom != nil {
|
|
||||||
events, err = conditionsFrom.HistorySince(ctx, since)
|
|
||||||
} else {
|
|
||||||
err = onTheBus(func(conn *nats.Conn) error {
|
|
||||||
_, history, err := conditions.OnTheBus(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return read(history)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var out []conditions.Event
|
|
||||||
for _, e := range events {
|
|
||||||
if *key == "" || e.Key == *key {
|
|
||||||
out = append(out, e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if *asJSON {
|
|
||||||
if out == nil {
|
|
||||||
out = []conditions.Event{}
|
|
||||||
}
|
|
||||||
return printJSONTo(w, map[string]any{"history": out, "days": *days})
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
fmt.Fprintf(w, "nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, e := range out {
|
|
||||||
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
|
||||||
switch e.Change {
|
|
||||||
case conditions.ChangeRaised, conditions.ChangeReopened:
|
|
||||||
line += " — " + e.Summary
|
|
||||||
case conditions.ChangeSeverity:
|
|
||||||
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
|
||||||
case conditions.ChangeResolver:
|
|
||||||
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
|
||||||
default:
|
|
||||||
if e.Why != "" {
|
|
||||||
line += " — " + e.Why
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Fprintln(w, line)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
|
||||||
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
|
||||||
// is not "none open".
|
|
||||||
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
|
||||||
if unread != "" {
|
|
||||||
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(list) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
urgent := 0
|
|
||||||
for _, c := range list {
|
|
||||||
if c.Severity == conditions.Urgent {
|
|
||||||
urgent++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
|
||||||
for _, line := range conditionLines(list, now) {
|
|
||||||
fmt.Println(line)
|
|
||||||
}
|
|
||||||
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
|
||||||
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
// orHealer is who tried, as an attempt names it.
|
|
||||||
func orHealer(by string) string {
|
|
||||||
if by == "" {
|
|
||||||
return "a healer"
|
|
||||||
}
|
|
||||||
return by
|
|
||||||
}
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) {
|
|
||||||
var found []string
|
|
||||||
for i := 0; i < 689; i++ {
|
|
||||||
found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i))
|
|
||||||
}
|
|
||||||
run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found},
|
|
||||||
{ID: "D1", Verdict: "passed"}}}
|
|
||||||
answer, err := verdictAnswer(run, time.Now(), "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
brief := answer["run"].(doctorRun).Probes[0]
|
|
||||||
if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") {
|
|
||||||
t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More)
|
|
||||||
}
|
|
||||||
if len(run.Probes[0].Found) != 689 {
|
|
||||||
t.Fatal("the overview cut the run it was made from")
|
|
||||||
}
|
|
||||||
whole, err := verdictAnswer(run, time.Now(), "d14")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" {
|
|
||||||
t.Fatalf("doctor probe=D14 answered %+v", p)
|
|
||||||
}
|
|
||||||
if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil {
|
|
||||||
t.Fatal("a probe the run does not have was answered")
|
|
||||||
}
|
|
||||||
if !strings.Contains(doctorText(answer), "probe=D14` gives them all") {
|
|
||||||
t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user