diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f4fe65b4..00000000 --- a/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq -# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how -# `make image` broke once before (issue 146). -ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c -# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go -# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it. -# Genesis builds this file and raises it as the container the process replaces on the first push -# (mesh-host internal/bootstrap, novox/hq issue 223). -# -# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a -# machine where no mesh exists yet, and run before there is anything to check it against. So it -# holds the program and nothing else — no shell, no package manager, no libc, nothing with a CVE -# feed of its own. What a person has to audit before trusting a first node is one binary. -# -# There are no CA certificates in here on purpose. Nothing it does today makes an outbound TLS -# connection to a public name: it reaches PostgreSQL on the machine it was raised on, and the -# broker is verified against a fingerprint pinned in a token rather than against a public root -# (novox/hq ADR 0004). Adding them "just in case" would put a trust store in the one image whose -# whole argument is that it contains nothing to reason about. - -FROM ${GO_BASE} AS build -WORKDIR /src - -# **No dependency is committed** (issue 1305): go.sum names each one's version and checksum, and the build -# resolves them with -mod=mod. Dependencies first, so a change to the source does not re-copy them. -COPY go.mod go.sum ./ - -# `COPY . .` brings `.git` along, so Go is told not to stamp the commit: the version is said by -X, and a -# rebuild of an unchanged program is the same bytes (novox/hq ADR 0267 rule 6, issue 363). -COPY . . -ARG VERSION=development -RUN CGO_ENABLED=0 GOFLAGS=-mod=mod go build -trimpath -buildvcs=false \ - -ldflags "-s -w -X main.version=${VERSION}" \ - -o /mesh-controller ./cmd/mesh-controller - -FROM scratch -COPY --from=build /mesh-controller /mesh-controller - -# Numeric because there is no /etc/passwd to look a name up in. Nothing here needs to be root: -# it opens outbound connections and writes nothing to its own filesystem. -USER 65534:65534 - -ENTRYPOINT ["/mesh-controller"] diff --git a/Makefile b/Makefile deleted file mode 100644 index 9636d6e0..00000000 --- a/Makefile +++ /dev/null @@ -1,112 +0,0 @@ -# novox/hq ADR 0006 — the control plane, in Go. -# -# The image the bundle pins holds the program and nothing else, so the build is static and the -# container is built FROM scratch. That is not a size optimisation: this image is fetched by -# digest and run on a machine where no mesh exists to check anything, and everything in it is -# something a person would have to audit. - -VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo development) -LDFLAGS := -s -w -X main.version=$(VERSION) - -# Where `make check` raises PostgreSQL. A high port and a throwaway container: nothing here -# touches a database anybody else is using. Override PG_PORT if this one is taken -- the first -# port chosen was already serving something that had been up for six days. -PG_PORT ?= 55532 -PG_CONTAINER ?= mesh-controller-check -PG_IMAGE ?= postgres:17-alpine -export MESH_TEST_POSTGRES ?= postgres://postgres:check@127.0.0.1:$(PG_PORT)/postgres?sslmode=disable - -.PHONY: build image check test vet fmt postgres postgres-stop clean - -build: - CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o build/mesh-controller ./cmd/mesh-controller - -# Tagged 'development' as well as by version, because the lab places images by name and a -# scenario naming a version would have to be edited on every build. The version tag is what a -# real bundle pins. -IMAGE ?= mesh-controller:$(VERSION) -DEV_TAG ?= mesh-controller:development - -# The Go base the image is built on. -# -# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go -# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >= -# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody -# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146, -# what it cost). -# -# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds -# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab — -# still needs a Go base. The digest is the one the manifest declared until then. -GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c - -image: - @test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE="; exit 1; } - docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) . - @echo - @docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' - -# The provisioner ships as an image too, because it is the thing that makes a sealed credential -# true on a machine -- and the mesh cannot, having discarded the plaintext. -PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION) -PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development - -provisioner-image: - docker build --build-arg GO_BASE=$(GO_BASE) -f examples/postgres-provisioner/Dockerfile \ - -t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) . - @echo - @docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' - -# The object store's provisioner, for the same reason: a bucket and a policy are not files, and -# the mesh cannot make them -- it discarded the credential it would have to use. -OBJECTSTORE_IMAGE ?= mesh-provision-objectstore:$(VERSION) -OBJECTSTORE_DEV_TAG ?= mesh-provision-objectstore:development - -objectstore-image: - docker build --build-arg GO_BASE=$(GO_BASE) -f examples/objectstore-provisioner/Dockerfile \ - -t $(OBJECTSTORE_IMAGE) -t $(OBJECTSTORE_DEV_TAG) . - @echo - @docker image inspect $(OBJECTSTORE_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' - -# The cache's provisioner, for the same reason as the database's: an ACL user is not a file, -# and the mesh cannot make one -- it discarded the credential it would have to use. -REDIS_PROVISIONER_IMAGE ?= mesh-provision-redis:$(VERSION) -REDIS_PROVISIONER_DEV_TAG ?= mesh-provision-redis:development - -redis-provisioner-image: - docker build --build-arg GO_BASE=$(GO_BASE) -f examples/redis-provisioner/Dockerfile \ - -t $(REDIS_PROVISIONER_IMAGE) -t $(REDIS_PROVISIONER_DEV_TAG) . - @echo - @docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' - -# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests -# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the -# store's tests would skip rather than fail anyway. -test: - go test -timeout 15m ./... - -vet: - go vet ./... - -# A quick step for a developer: the checker runs the build module's own formatting and vet (ADR 0501 §5, D7). -fmt: - @unformatted=$$(gofmt -l cmd internal examples) ; \ - if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi - -postgres: - @docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true - @docker run -d --name $(PG_CONTAINER) -e POSTGRES_PASSWORD=check \ - -p 127.0.0.1:$(PG_PORT):5432 $(PG_IMAGE) >/dev/null - @printf 'waiting for postgres' - @for i in $$(seq 1 60) ; do \ - if docker exec $(PG_CONTAINER) pg_isready -U postgres >/dev/null 2>&1 ; then \ - echo ' — ready' ; exit 0 ; fi ; \ - printf '.' ; sleep 1 ; \ - done ; \ - echo ' — never came up' ; docker logs $(PG_CONTAINER) | tail -20 ; exit 1 - -postgres-stop: - @docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true - -clean: - rm -rf build/ diff --git a/README.md b/README.md index 56695985..a1de1323 100644 --- a/README.md +++ b/README.md @@ -1,218 +1,7 @@ # mesh-controller -**Tier 2 of Novox Mesh — the control plane.** Everything that needs to know about more than one -node. +The controller ended under issue 1126. Its jobs moved to the parts that ADR 0590 and ADR 0593 name: the indexer, the +assigner, the provisioner, the grants part and the node-engine. Its last module, route-proxy, moved to mesh-networking. -That is the whole test, and it draws the line the host cannot: the host applies and does not -decide, *because deciding needs knowledge one machine does not have*. Which nodes should run the -store, which peers belong in an overlay, whether a node has been unreachable for a week — nobody -on a single machine can answer any of them. - -The reasoning lives in [novox/hq](https://git.novox.be/novox/hq); this repository carries no -argument that is not settled there. - -## What it is not - -- **Not the thing that changes machines.** It decides; the host applies. It never reaches into a - node except through the host, over the link, in a bounded vocabulary. -- **Not a database.** There is no mesh database. Each context owns its own store and nothing - outside a context touches it — including nodes, which hold no credential to any of them. -- **Not privileged.** It has no more access to a machine than a declaration can express. - -## What exists today - -**One context of seven, and one of the things it will do.** - -| | | -|---|---| -| `inventory` | node records and enrolment tokens — **built, as far as identity** | -| `identity` | the control plane's signing key, and the keys nodes are known by — **built** | -| `config`, `connectivity`, `provisioning`, `delivery`, `observability` | not built | -| the interface every surface speaks to | not built; its shape is not decided | - -``` -mesh-controller migrate bring each context's schema up to date -mesh-controller node add create a node record -mesh-controller node list the nodes this mesh knows about -mesh-controller token issue --node a one-time right to join, for an existing record -mesh-controller token issue --new create the record and issue for it -mesh-controller identity show this control plane's signing key -mesh-controller broker show where the broker is, and what to expect there -mesh-controller version what this binary is -``` - -`migrate` is **step 3 of the foundation bootstrap** — the step the first node cannot get past, run -against a database raised moments earlier from the bundle the host carries. - -### Tokens, and what they are missing - -A token is **a one-time right to join, issued for a node record** — which is where re-enrolment is -decided, since what an identity binds to is settled when the token is made rather than when it is -presented. - -What is built: the secret is 256 bits from the system's random source, shown once, and **stored -only as a hash**, so a copy of this database is not a set of working credentials. It is usable -exactly once and only before it expires, and both are read from the row rather than from a status -something would have had to write. Issuing again for the same node invalidates the outstanding -one — two live tokens are two machines able to join as the same node. - -Redemption is a single statement that both finds a live token and spends it, so eight concurrent -attempts on one secret produce exactly one winner. There is a test that runs them. - -**A token now carries three of its four parts**, and is one line of base64 a person can copy. The -signing key is real: an Ed25519 key this control plane generates once and keeps, whose public half -travels in every token. A node believes a declaration because it carries a signature that key made -— and pinning only the broker would not do, because it would make the control plane's authority -transitive, so a compromised broker could forge declarations, and since the host applies whatever -the link delivers that is the whole machine. - -**All four parts are built.** Given `MESH_BROKER_ADDRESS` and `MESH_BROKER_CERTIFICATE`, a token -carries everything ADR 0004 requires. Without them it carries two, and `token issue` prints it -**and names what is missing** rather than producing something that looks complete and cannot be -used. - -**The fingerprint is derived from the certificate, never configured.** A configured one can drift -from the certificate it describes, and a drifted pin is worse than none: every node issued a token -during the drift refuses to connect, and the failure looks like an attack. It is computed over the -DER bytes — what a client actually sees on the wire — so the same certificate re-wrapped with -different line endings still produces the same pin. - -### Two contexts, and the rule between them is real - -`identity` is the second context and it exists partly to test a claim this repository had made and -never checked: that a context reaches only its own store. It holds `MESH_STORE_IDENTITY`; -`inventory` holds `MESH_STORE_INVENTORY`; there is no setting that reaches both and no way to ask -for one. Run `migrate` with only one and it stops, naming the grant it does not have. - -A token needs a node record from one and a signing key from the other. Neither reads the other's -store — the process holding both grants asks each for its part. - -### How a node is known - -**The node generates a keypair; the mesh records the public half.** The same principle as SSH, and -the same rule `08-connectivity` already applies to the overlay keys — which is where this was -settled all along, though it took being asked directly to notice. - -Only the public half is ever stored, and that is the property worth having: **a copy of this -database grants nothing.** It is a list of who to believe, not a set of credentials, which is what -makes *compromise of a node is compromise of that node* literally true. - -Exactly one key is live per node. Re-enrolment revokes the one it replaced, in the same -transaction — two live identities for one node record is the stolen-laptop case, with the replaced -machine still believed. The database enforces it as well as the code, and there is a test running -six concurrent enrolments that fails when the constraint is removed. - -**Not the machine's SSH host key**, though that was the obvious economy. Host keys are regenerated -by reinstalls and image clones, which would silently un-enrol a node; their lifecycle belongs to -sshd rather than the mesh; and a partial host has no SSH daemon at all. - -## Reaching a store - -**A context is granted only what it exclusively owns.** No shared writes, no read-only role on -another context's store, and no connection string that reaches more than one. - -That is a rule about credentials, so it is built as one. There is no mesh-wide connection setting -and no way to ask for one: - -``` -MESH_STORE_INVENTORY=postgres://…/inventory -``` - -A process granted `inventory` holds that variable and no other. Reaching another context's store -is not a matter of restraint — it has no address for it and no credential to present. And it is -how the rule is *checked*: what a context can reach is visible in the declaration that runs it, -as the list of variables it was given. - -Each context's database is named after the context. There is deliberately no database named for -the mesh as a whole. - -## Migrations - -Numbered, embedded in the binary, applied in order, each in a transaction with the row recording -it. The applying is four lines; the rest is refusals, and the refusals are the point: - -| it stops when | because | -|---|---| -| a migration that ran has since been edited | the database holds the old version, the repository holds the new one, and nothing holds the difference | -| a migration is numbered below one that already ran | usually two branches taking the same next number — applying it now runs the schema in an order nobody tested | -| two migrations share a number | order is the entire guarantee, and two files with one number have none | -| a file in the directory is not a valid migration name | a misnamed migration would otherwise never run and nothing would say so | -| a migration is empty | it records that something happened and changes nothing, which cannot be told from a mistake | - -All of them stop rather than warn. At the moment any of them is true, nobody knows what the -database contains, and there is no correct guess about a schema. - -Running it again does nothing. Two copies running at once take an advisory lock, so a restart -during a slow migration does not become two runners racing. - -## Building - -``` -make build the binary -make image the container image -make check gofmt, vet, and every test against a real PostgreSQL -``` - -`make check` raises a throwaway PostgreSQL in a container and takes it down afterwards, including -when the tests fail. Without one the tests that need a database **skip and say so** rather than -passing quietly — `make test` is the honest subset, not the gate. - -Tests run against a real database rather than a fake because what is being tested *is* the -database's behaviour: that DDL is transactional, that an advisory lock serialises, that a -checksum mismatch is caught against a record PostgreSQL actually kept. A fake would assert that -the fake behaves as expected. - -Every test here has been confirmed to fail when the behaviour it defends is removed. Two did not, -when first written, and both are now commented with what they were missing. - -## Checks that cross into the host's repository - -Two things are agreed between this repository and `novox/mesh-host`, and each is a separate struct -on each side. A field renamed on one of them fails **silently** — the crossing succeeds and -something is simply absent — so both are checked by handing one side's real output to the other's -real parser. Neither runs by default; each skips with a reason, because a repository that fails -without its neighbour checked out is a repository nobody can build. - -**What this mesh sends, read by the host that receives it:** - -``` -mesh-controller: ./build/mesh-controller plan --json > /tmp/d.json -mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v -``` - -**What a node says when it joins, read by this mesh:** - -``` -mesh-host: MESH_ENROL_OUT=/tmp/enrol.json go test ./internal/link/ -mesh-controller: MESH_ENROL=/tmp/enrol.json make check -``` - -The second does more than compare shapes: it seals something to the key that arrived and opens it -with the private half the host kept. A key that is present, correctly named and simply *wrong* -passes every check that only looks at the message. - -## The image - -**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go -bundle, `controller`, which the host on the controller's machine unpacks and runs as the process -`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what -still runs a container of the controller: genesis, which raises the first controller from it and -installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the -mesh's own build any more — `make image` builds it. - -`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package -manager, no libc, no CA certificates. - -Not a size optimisation. The bundle a host carries pins this image by digest, and it is fetched -and run on a machine where no mesh exists to check anything and a person is expected to have read -the bundle and believed it. Everything in the image is something that person would have to audit. - -## Where the reasoning lives - -| | | -|---|---| -| what the control plane is | novox/hq ADR 0006 | -| what it takes to run one, and why Go | novox/hq ADR 0006 | -| a context owns its store, exclusively | novox/hq ADR 0008 | -| schema changes are numbered migrations | novox/hq ADR 0013 | -| a test defends a decision | novox/hq ADR 0017 | +This repository holds no code and is archived once the delivery of the group that emptied it (release sdk1) is done. +Its history stays here for reading. diff --git a/cmd/mesh-controller/accept_outside.go b/cmd/mesh-controller/accept_outside.go deleted file mode 100644 index dd83125e..00000000 --- a/cmd/mesh-controller/accept_outside.go +++ /dev/null @@ -1,149 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "strings" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A provision whose credential only a person outside the mesh can give (novox/hq ADR 0337, issue 164, issue 165): -// given once for its provider, at the controller's terminal, and sealed in one act to every consumer that binds the -// provision from that provider now. The value is never read back, printed or kept. - -// outsideOffer is the provider module's own name when `secret accept ` names a provision that -// module offers with a credential issued outside the mesh, and "" when it names anything else (an own secret, or a -// pair with --provider). -func outsideOffer(ctx context.Context, module, name, provider string) (string, error) { - if provider != "" { - return "", nil - } - open, err := openStores(ctx) - if err != nil { - return "", err - } - defer open.Close() - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - if m, known := shelf[module]; known && m.IssuedOutsideOf(name) { - return module, nil - } - return "", nil -} - -// offeredOutsideBy is a module that offers the provision with a credential issued outside the mesh, or "". -func offeredOutsideBy(ctx context.Context, provision string) (string, error) { - open, err := openStores(ctx) - if err != nil { - return "", err - } - defer open.Close() - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - module, _ := catalogue.OffersIssuedOutside(shelf, provision) - return module, nil -} - -// consumersFrom is every consumer that binds a provision from one provider now, found by resolving every machine as -// its next send would, and the machines that could not be resolved, with why. -func consumersFrom(ctx context.Context, open *stores, provider, providerModule, provision string) ( - []inventory.PairEnd, []string, error) { - // One reading of the mesh for every machine this looks at. - ctx = withInputCache(ctx) - nodes, err := open.inventory.Nodes(ctx) - if err != nil { - return nil, nil, err - } - var out []inventory.PairEnd - var unresolved []string - for _, n := range nodes { - plan, _, err := resolveNode(ctx, open, n.Name) - if err != nil { - unresolved = append(unresolved, fmt.Sprintf("%s: %s", n.Name, oneLine(err.Error()))) - continue - } - out = append(out, bindsFrom(plan, n.Name, provider, providerModule, provision)...) - } - return out, unresolved, nil -} - -// bindsFrom is every consumer on one machine whose resolution answers the provision from that provider. Pure. -func bindsFrom(plan catalogue.Resolution, machine, provider, providerModule, provision string) []inventory.PairEnd { - var out []inventory.PairEnd - for _, need := range plan.Needs { - if need.ByRecord || need.Name != provision || need.From != provider { - continue - } - if need.Module != "" && need.Module != providerModule { - continue - } - out = append(out, inventory.PairEnd{Consumer: machine, ConsumerModule: need.For, Local: need.Local}) - } - sort.Slice(out, func(i, j int) bool { - return out[i].ConsumerModule+"\x00"+out[i].Local < out[j].ConsumerModule+"\x00"+out[j].Local - }) - return out -} - -// acceptForProvision gives a provision's credential issued outside the mesh, once for its provider (ADR 0337 §3). -func acceptForProvision(ctx context.Context, provider, providerModule, provision, value string) error { - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - consumers, unresolved, err := consumersFrom(ctx, open, provider, providerModule, provision) - if err != nil { - return err - } - accepted, err := open.inventory.AcceptSecretForProvision(ctx, provision, provider, providerModule, value, consumers) - value = "" - if err != nil { - return err - } - fmt.Print(acceptedSaid(provider, providerModule, provision, accepted, unresolved)) - if len(accepted.Sealed) == 0 { - return nil - } - // Composed into each consumer's composition (novox/hq ADR 0539 clause 5): a new revision its want then names. - if _, err := composeModules(ctx, open, accepted.Machines); err != nil { - return fmt.Errorf("%s is sealed and its compositions on %s could not be written: %w", provision, - strings.Join(accepted.Machines, ", "), err) - } - fmt.Printf(" composed for %s; each consumer starts again on it once its want names the new composition\n", - strings.Join(accepted.Machines, ", ")) - return nil -} - -// acceptedSaid is what one accept for a provider did, in words that never carry the value. Pure. -func acceptedSaid(provider, providerModule, provision string, a inventory.ProvisionAccepted, unresolved []string) string { - var b strings.Builder - if len(a.Sealed) == 0 { - fmt.Fprintf(&b, "%s from %s on %s: no consumer binds it now, so nothing was sealed; a consumer that binds "+ - "later waits for it, and this line given again seals it to every consumer then\n", provision, - providerModule, provider) - } else { - var names []string - for _, c := range a.Sealed { - names = append(names, fmt.Sprintf("%s on %s%s", c.ConsumerModule, c.Consumer, asLocal(c.Local))) - } - fmt.Fprintf(&b, "%s from %s on %s is given, sealed to %s and to %s\n", provision, providerModule, provider, - strings.Join(names, ", "), provider) - fmt.Fprintf(&b, " the mesh cannot read it back, never makes one of its own for it, and will not rotate it\n") - } - for _, c := range a.Unsealable { - fmt.Fprintf(&b, " not sealed to %s on %s: no sealing key on that machine; give it again once it has one\n", - c.ConsumerModule, c.Consumer) - } - for _, u := range unresolved { - fmt.Fprintf(&b, " not known whether a consumer on %s binds it, which did not resolve; give it again once it does\n", u) - } - return b.String() -} diff --git a/cmd/mesh-controller/acting.go b/cmd/mesh-controller/acting.go deleted file mode 100644 index ebf60696..00000000 --- a/cmd/mesh-controller/acting.go +++ /dev/null @@ -1,414 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "os" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1). -// -// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a -// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may -// not happen. Three ways a process stands to the lease: -// -// - **The serving controller** takes it before it does anything else — before it asserts the bus's -// objects, which are the controller's to write — waiting while another holds it, and renews it. -// Another holder seen, or the key's age passed with no renewal made, is the lease lost: the gate -// closes at once and the process exits, so its service manager restarts it as a candidate (serve, -// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the -// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal -// made opens the gate again. -// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose -// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the -// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like -// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover -// between makes it stale and refused like any other. **When nobody holds the lease, the command -// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for -// it, as it would for another controller. -// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is -// refused nothing: there is nothing to order against, and nothing it does reaches a machine. -// -// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's -// user list is older than this build and does not grant the bucket yet — and that sees no other holder -// serves without one, as every controller did before the lease: declarations carry no epoch, which no -// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the -// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act -// instead would be a controller that can never send the user list that lets it act. - -// actor is this process's standing to the lease. -type actor struct { - mu sync.Mutex - // held is the lease this process holds: the serving controller's, or a command's own. - held *lease.Lease - // unleased is why a serving controller acts without the lease; empty while it holds it or is not - // serving. - unleased string - // serving is a serving controller, which never borrows another's epoch. - serving bool - // kv is the lease bucket, for a command to read the holder's epoch from. - kv jetstream.KeyValue - close func() - // noBus is a process with no bus configured. - noBus bool - // reset is when the lease bucket was found raised again from nothing and its revisions moved past - // the highest epoch issued, and what was said of it; zero when it was not (S12). - reset time.Time - resetSaid string - // renewEvery and poll, when set, are how often the serving controller renews the lease it holds and - // looks again at one another holds, in place of the lease's own: a test's, so two controllers hand the - // lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given - // the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it. - renewEvery, poll time.Duration - // releasing is held for the whole of a release, so a second caller — the stop signal's and serve's own - // deferred one — waits until the lease is given back and its epoch recorded, never past it to the store's - // close (novox/hq issue 906). - releasing sync.Mutex - // keeper and inv are where a takeover is said (sayTakeover): set once the serving controller's condition - // store opens; nil before, and in a command. - keeper *conditions.Keeper - inv *inventory.Inventory -} - -// theLease is this process's standing to the lease. -var theLease = &actor{} - -// instance names this process among controller instances: its machine, its process and when it -// started. The lease's holder and every call this process keeps carry it. -var instance = func() string { - host, _ := os.Hostname() - return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339)) -}() - -// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen. -func (a *actor) epoch(ctx context.Context) (uint64, error) { - a.mu.Lock() - held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus - a.mu.Unlock() - switch { - case held != nil: - return held.Epoch() - case serving && unleased != "": - return 0, nil - case serving: - return 0, lease.ErrNotHeld - case noBus: - return 0, nil - } - return a.forACommand(ctx) -} - -// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one. -func (a *actor) forACommand(ctx context.Context) (uint64, error) { - a.mu.Lock() - defer a.mu.Unlock() - if a.held != nil { - return a.held.Epoch() - } - if a.kv == nil { - address, err := broker.BusAddress() - if err != nil { - // No bus: this process reaches no machine, and has nothing to order against. - a.noBus = true - return 0, nil - } - js, err := broker.Dial(address) - if err != nil { - return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+ - "read and nothing is done: %w", err) - } - api, err := jetstream.New(js.Conn()) - if err != nil { - js.Close() - return 0, err - } - reading, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - if err := broker.EnsureLeaseBucket(reading, api); err != nil { - js.Close() - return 0, err - } - kv, err := api.KeyValue(reading, broker.LeaseBucket) - if err != nil { - js.Close() - return 0, err - } - a.kv, a.close = kv, js.Close - if _, found, err := lease.Current(reading, kv); err == nil && !found { - // Nobody: this command takes it for as long as it runs. - l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance), - Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }}) - if err != nil { - return 0, err - } - epoch, err := l.TryTake(reading) - if err != nil { - return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err) - } - keeping, stop := context.WithCancel(context.Background()) - go l.Keep(keeping) - a.held = l - closeBus := a.close - a.close = func() { - stop() - l.Release(context.Background()) - closeBus() - } - return epoch, nil - } - } - reading, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - holder, found, err := lease.Current(reading, a.kv) - if err != nil { - return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err) - } - if !found { - return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " + - "more is done under an epoch nobody holds — run the command again") - } - return holder.Epoch, nil -} - -// release gives back what this process holds, at its end — or at the stop signal, before the rest of the -// process is taken down (serveUnderTheLease). A second call waits for the first to finish. -func (a *actor) release() { - a.releasing.Lock() - defer a.releasing.Unlock() - a.mu.Lock() - closing := a.close - a.close = nil - a.mu.Unlock() - if closing != nil { - closing() - } -} - -// holderOf is this process as the lease's holder. -func holderOf(instance string) lease.Holder { - host, _ := os.Hostname() - build := runningBuild() - if build == "" { - build = version - } - return lease.Holder{Instance: instance, Host: host, Build: build} -} - -// RunningBuildVar is where the declaration tells this process which build it is (module.json, the -// controller process's env): the version its bundle is delivered as, `${version}` composed by the -// catalogue from the bundle's digest. Empty for a process placed by hand. -const RunningBuildVar = "MESH_CONTROLLER_VERSION" - -// runningBuild is the version of the build this process is, or empty when the declaration did not say. -func runningBuild() string { - return strings.TrimSpace(os.Getenv(RunningBuildVar)) -} - -// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and -// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it. -func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) { - a.mu.Lock() - a.serving = true - a.mu.Unlock() - // Its own connection, held as long as the lease, and named so (novox/hq issue 327). - js, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease")) - if err != nil { - return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+ - "lease: %w", broker.BareAddress(address), err) - } - api, err := jetstream.New(js.Conn()) - if err != nil { - js.Close() - return nil, err - } - asserting, cancel := context.WithTimeout(ctx, 10*time.Second) - err = broker.EnsureLeaseBucket(asserting, api) - cancel() - if err != nil { - js.Close() - return nil, err - } - say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) } - l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance), - RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) { - a.mu.Lock() - defer a.mu.Unlock() - a.reset = time.Now() - a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+ - "raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+ - "no machine refuses the next epoch", was, floor, floor) - }}) - if err != nil { - js.Close() - return nil, err - } - gone := make(chan struct{}) - epoch, err := l.Take(ctx) - switch { - case ctx.Err() != nil: - js.Close() - return nil, ctx.Err() - case err != nil && !errors.Is(err, lease.ErrUnwritable): - // Whether another controller acts cannot be told: this one does not act, and exits to try again. - js.Close() - return nil, err - case err != nil: - // Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above). - a.mu.Lock() - a.unleased = err.Error() - a.mu.Unlock() - say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+ - "every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery) - go a.takeWhenGranted(ctx, l, inv, gone) - default: - a.took(ctx, l, inv, epoch, gone) - } - a.mu.Lock() - a.close = func() { - // Given back first, then recorded: an epoch is recorded released only when its key is gone, and - // whatever the gate says by then — a shutdown longer than half the key's age closed it, and the epoch - // was left open for the next holder to find "expired" (novox/hq issue 906, 2026-10-26). - if given := l.Release(context.Background()); given != 0 { - ending, cancel := context.WithTimeout(context.Background(), 5*time.Second) - if err := inv.EndEpoch(ending, given, inventory.EpochReleased); err != nil { - say("how epoch %d ended could not be recorded: %v", given, err) - } - cancel() - } - js.Close() - } - a.mu.Unlock() - // **The stop signal gives the lease back first** (novox/hq issue 906): the moment ctx ends, before the rest - // of the process is taken down, the gate closes and the key goes, so the next controller takes the lease - // at once and the epoch is recorded released. Waiting for the process's own end left the key unrenewed for - // as long as the shutdown took — ten seconds on 2026-10-26, past the gate's half-age. - go func() { - <-ctx.Done() - if a.holds(l) { - say("stopping: the controller lease is given back first") - } - a.release() - }() - return gone, nil -} - -// holds is whether this process holds l now. -func (a *actor) holds(l *lease.Lease) bool { - a.mu.Lock() - defer a.mu.Unlock() - return a.held == l -} - -// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept. -func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) { - a.mu.Lock() - a.held, a.unleased = l, "" - a.mu.Unlock() - h := holderOf(instance) - recording, cancel := context.WithTimeout(ctx, 10*time.Second) - expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host, - Build: h.Build, Taken: time.Now()}, l.GivenBack()) - cancel() - if err != nil { - fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err) - } - if given := l.GivenBack(); given != 0 { - fmt.Printf("the lease was given back by the controller of epoch %d: taken at once, at epoch %d\n", given, epoch) - } - for _, e := range expired { - fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+ - "taken over at epoch %d\n", e.Epoch, e.Instance, epoch) - } - a.sayTakeover(ctx, epoch) - go l.Keep(ctx) - go func() { - <-l.Lost() - if ctx.Err() == nil { - why := l.LostWhy() - ending, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = inv.EndEpoch(ending, epoch, inventory.EpochLost) - cancel() - fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+ - "be started again as a candidate\n", epoch, why) - } - close(gone) - }() -} - -// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this -// controller if another took it meanwhile: two serving at once is what the lease is for. -func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) { - tick := time.NewTicker(lease.RenewEvery) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - } - epoch, err := l.TryTake(ctx) - if errors.Is(err, lease.ErrTaken) { - fmt.Printf("another controller took the lease while this one served without it: %v — this one "+ - "stops and exits\n", err) - close(gone) - return - } - if err != nil { - // Still not written, or not readable this time: unleased, said by S12, tried again. - a.mu.Lock() - a.unleased = err.Error() - a.mu.Unlock() - continue - } - a.took(ctx, l, inv, epoch, gone) - return - } -} - -// standing is what `status` and the self-check say of this process and the lease. -type standing struct { - Epoch uint64 - Held bool - Renewed time.Time - Unleased string - // Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it. - Reset time.Time - ResetSaid string -} - -func (a *actor) standing() standing { - a.mu.Lock() - held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid - a.mu.Unlock() - st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid} - if held == nil { - return st - } - epoch, err := held.Epoch() - st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed() - return st -} - -// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory). -func init() { - link.ActingGate = func(ctx context.Context) error { - _, err := theLease.epoch(ctx) - if err != nil { - return fmt.Errorf("this controller may not send: %w", err) - } - return nil - } -} diff --git a/cmd/mesh-controller/acts.go b/cmd/mesh-controller/acts.go deleted file mode 100644 index 354e83cc..00000000 --- a/cmd/mesh-controller/acts.go +++ /dev/null @@ -1,479 +0,0 @@ -package main - -import ( - "context" - "fmt" - "github.com/novox/mesh-controller/internal/broker" - "io" - "slices" - "sort" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The things the mesh can be asked to do, separated from how it was asked. -// -// **A surface is an adapter with no decisions in it** (novox/hq ADR 0035). The command line and -// the command API call the same functions here, so an assignment refused at one is refused at the -// other for the same reason and in the same words. The moment a surface can accept something -// another would reject, the mesh has two answers to one question and people learn which to trust. -// -// Each returns what happened as text a person can read and a caller can pass on. Neither surface -// composes its own explanation, because two explanations of one refusal drift. - -// assign puts a module on a node, and says at once what in the mesh no longer works out. -// -// The assignment is kept even when the node does not resolve: it is what a person meant, and -// assignment is not an ordering. A consumer assigned before its provider does not resolve for as -// long as it takes to assign the provider, and refusing the first half of a pair would make the -// order somebody types two commands in part of the mesh's rules. -// -// **What is checked is the mesh, not the one machine** — because that is what the assignment -// changes. novox/hq 04-ISSUES/017 names the shape: an action can succeed into a state its own -// verify rejects, and it happens when the action's test is not the test the verify uses. Here the -// action tested one node and the verify is resolution over all of them, so an assignment could be -// reported as fine while it took a provision away from every other machine — a module offering a -// mesh-scoped provision stops offering it the moment its own node stops resolving, and every -// consumer elsewhere is then told *nothing in this mesh provides it*, with a remedy that names a -// module already assigned. That was found on a four-node raise and read as a version bump breaking -// provider recognition; it was neither the version nor the provider. -// -// It costs a resolution per machine. Assignment is a person typing a command, and being told which -// machines this just blocked is worth more than the milliseconds. -func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) { - return assignWith(ctx, open, node, assignOptions{}, modules...) -} - -// assignWith is assign asked with its options: build, for a module known and not built (novox/hq issue 325). -func assignWith(ctx context.Context, open *stores, node string, opts assignOptions, modules ...string) (string, error) { - if len(modules) == 0 { - return "", fmt.Errorf("assign %s names no module", node) - } - if err := refusedMovingTheController(modules); err != nil { - return "", err - } - // Every line of the answer comes from the one resolution after the write (novox/hq issue 237): - // what the reads before it would say about a machine left out is not the answer's. - ctx = leftOutSaidBy(ctx, io.Discard) - // **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue - // 325): a build in flight keeps the assignment pending, a module known and not built is said with how to - // build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act - // are judged together (ADR 0207), so an act naming one not registered is not made in part. - if missing, err := notInCatalogue(ctx, open, modules); err != nil { - return "", err - } else if len(missing) > 0 { - if len(modules) == 1 { - return notRegistered(ctx, open, node, modules[0], opts) - } - var lines []string - for _, m := range missing { - where, err := whereIs(ctx, open.inventory, m, time.Now()) - if err != nil { - return "", err - } - lines = append(lines, where.said) - } - return "", fmt.Errorf("%w: nothing was assigned, since modules assigned together are judged together "+ - "(ADR 0207) and %s not registered:\n %s\n assign them together once each is registered, or each "+ - "alone to keep it pending on its build", inventory.ErrNoSuchModule, strings.Join(missing, ", "), - strings.Join(lines, "\n ")) - } - // **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else - // an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose - // resources are applied through a seat nothing on the node holds is refused, because that order - // — the service manager, the package manager and the runtime before anything that installs, - // runs or contains — is the mesh's to keep. Several modules in one act are judged together, so - // holders that depend on each other go on in one command. - shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules) - if err != nil { - return "", err - } - // **Before the new assignment can unsettle a seat somebody holds only by being alone** - // (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the - // assignment resolves against a record rather than against a coincidence. - settled, err := recordDerivedHolders(ctx, open) - if err != nil { - return "", err - } - var lines []string - var added []string - for _, module := range modules { - fresh, err := open.inventory.Assign(ctx, node, module) - if err != nil { - return strings.Join(lines, "\n"), err - } - if !fresh { - // Nothing changed, and saying "is assigned" would read as an action. One node runs one - // of each — the module's name is the assignment's identity (novox/hq ADR 0115). - lines = append(lines, fmt.Sprintf( - "%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module)) - continue - } - added = append(added, module) - lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module)) - } - if len(added) == 0 { - return strings.Join(lines, "\n"), nil - } - answer := strings.Join(lines, "\n") - for _, line := range settled { - answer += "\n " + line - } - // Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its - // credential exists delivers a process that cannot authenticate and crash-loops until somebody - // runs a second verb and a second push. Issued here when the module speaks on the bus and has - // no credential yet; kept when it has one, so re-assigning rotates nothing. - var issued []string - for _, module := range added { - if line := issueOnAssign(ctx, open, node, module); line != "" { - issued = append(issued, line) - } - } - // **The one resolution the rest of the answer is said from** (novox/hq issue 237), taken after - // the write: what this act changed about this node's unmet seat dependencies, and nothing else - // (novox/hq ADR 0207) — a dependency of a module just assigned, or one this assignment met; the - // rest of the node's list, and every other node's, is `status`'s — and whether the node resolves. - after := resolvedAfter(ctx, open, shelf, node) - for _, line := range unheldChangeTo(shelf, node, before, after.modules, after.unheld) { - answer += "\n " + line - } - for _, line := range issued { - answer += "\n " + line - } - plan, err := after.plan, after.err - if err != nil { - // Kept, and still refused. Both halves are the answer, and the rest of the mesh is still - // worth reporting: this machine's refusal is rarely the only consequence. - return answer + blockedElsewhere(ctx, open, node), err - } - // Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose - // capability the machine lacks is on the wrong machine, and the assignment records what a person - // meant while this line says it will not run until it moves. The rest of the node still pushes. - isAdded := map[string]bool{} - for _, m := range added { - isAdded[m] = true - } - for _, u := range plan.Unhostable { - if !isAdded[u.Module] { - continue - } - for _, c := range u.Missing { - answer += "\n but " + catalogue.WrongMachine(u.Module, c, node) - } - } - return answer + composedAfterAct(ctx, open, node) + blockedElsewhere(ctx, open, node), nil -} - -// composedAfterAct composes the node an assignment or its end changed (novox/hq ADR 0539 clause 5): what the -// answer says of it. The assigner then writes the node's wants by its rules; nothing is sent. -func composedAfterAct(ctx context.Context, open *stores, node string) string { - wrote, err := composeModules(ctx, open, []string{node}) - if err != nil { - return fmt.Sprintf("\n but its compositions on %s could not all be written: %v", node, firstLine(err.Error())) - } - if len(wrote[node]) == 0 { - return fmt.Sprintf("\n %s's compositions were already as composed now", node) - } - return fmt.Sprintf("\n composed for %s (%d composition(s) written): its wants follow by the assigner's rules", node, - len(wrote[node])) -} - -// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the -// catalogue and the node's assignments it was judged against. Modules already assigned are not new -// and are not judged again. -func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) ( - map[string]catalogue.Manifest, []string, error) { - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return nil, nil, err - } - assigned, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, nil, err - } - already := map[string]bool{} - for _, a := range assigned { - already[a] = true - } - var adding []string - for _, m := range modules { - if !already[m] { - adding = append(adding, m) - } - } - // The lines AssignRefusal says beside an assignment it lets through are said by unheldChange - // with everything else this act changed, so they are not said twice. - if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil { - return nil, nil, err - } - // Two modules declaring one path or unit, or one package present and absent, is refused before anything - // is recorded (novox/hq ADR 0210, ADR 0303, 04-ISSUES/235): kept, the node would not resolve until one came - // off again. Two modules declaring one package present is not a collision (ADR 0303, issue 628). - if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil { - return nil, nil, err - } - return shelf, assigned, nil -} - -// unassign takes modules off a node. What they leave behind is the host's business: a directory -// holding anything the mesh did not put there is kept (novox/hq ADR 0030). -// -// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a -// module off one machine is the ordinary way to stop providing something to another, and nothing -// about the command's own output would ever have said so. -// -// **Refused when it takes away the last holder of a seat a module left on the node depends on** -// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several -// modules in one act are judged together, so a holder and its dependents come off in one command. -func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) { - if err := refusedMovingTheController(modules); err != nil { - return "", err - } - if len(modules) == 0 { - return "", fmt.Errorf("unassign %s names no module", node) - } - // One resolution after the write, as assign's answer (novox/hq issue 237). - ctx = leftOutSaidBy(ctx, io.Discard) - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - assigned, err := open.inventory.Assigned(ctx, node) - if err != nil { - return "", err - } - // Every one checked before any is taken off, so a refusal leaves the node as it was. - runs := map[string]bool{} - for _, a := range assigned { - runs[a] = true - } - // A module only pending on the node (novox/hq issue 325) is withdrawn, when it is the act's one module: - // that is the undo of an assignment kept while its build runs. - if len(modules) == 1 && !runs[modules[0]] { - said, withdrawn, err := withdrawPending(ctx, open.inventory, node, modules[0]) - if err != nil { - return "", err - } - if withdrawn { - return said, nil - } - } - for _, module := range modules { - if !runs[module] { - return "", fmt.Errorf("%s is not assigned to %s", module, node) - } - } - if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil { - return "", err - } - for _, module := range modules { - if err := open.inventory.Unassign(ctx, node, module); err != nil { - return "", err - } - } - answer := fmt.Sprintf("%s no longer runs %s: its want is written removed by the assigner", - node, strings.Join(modules, ", ")) - after := resolvedAfter(ctx, open, shelf, node) - for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) { - answer += "\n " + line - } - // Whether the node itself still resolves, from the same resolution: before, only the rest of the - // mesh's reads said it, on standard error and once per other machine. - if after.err != nil { - if unresolvable(after.err) { - answer += fmt.Sprintf("\n but %s does not resolve now, so nothing will be sent to it:", node) - } else { - answer += fmt.Sprintf("\n but whether %s still resolves could not be read:", node) - } - for _, line := range strings.Split(strings.TrimRight(after.err.Error(), "\n"), "\n") { - answer += "\n " + strings.TrimSpace(line) - } - } - // Its bus accounts on the node go with it once it no longer resolves there (a module another requires keeps running - // and keeps them), so an assignment again issues them afresh (issue 1945). The node's own provisioner account stays. - if after.err == nil { - for _, module := range modules { - if slices.Contains(after.modules, module) { - continue - } - forgot, err := open.inventory.ForgetBusAccountsOf(ctx, node, module, false) - switch { - case err != nil: - answer += fmt.Sprintf("\n its bus accounts on %s are not forgotten: %v", node, err) - case len(forgot) > 0: - answer += fmt.Sprintf("\n its bus accounts on %s forgotten: %s", node, strings.Join(forgot, ", ")) - } - } - } - // What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233). - for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) { - answer += "\n " + line - } - return answer + composedAfterAct(ctx, open, node) + blockedElsewhere(ctx, open, node), nil -} - -// afterAct is the one resolution of a node an act's answer is said from, taken after its write -// (novox/hq issue 237): the plan or why there is none, the modules it has on the node, and its unmet -// seat dependencies. -type afterAct struct { - plan catalogue.Resolution - err error - modules []string - unheld []catalogue.Unheld -} - -// resolvedAfter resolves the node as `plan` does, once, after the act's write. When the node resolves, -// its modules and unmet seat dependencies are the resolution's — the closure, so a holder pulled in by -// a requirement counts as `plan` counts it. When it does not, they are judged over the assignments read -// after the write, which can only find more unmet than the closure would: a seat said held there is -// held in the closure too. -func resolvedAfter(ctx context.Context, open *stores, shelf map[string]catalogue.Manifest, node string) afterAct { - plan, _, err := resolveNode(ctx, open, node) - if err == nil { - out := afterAct{plan: plan, unheld: plan.Unheld} - for _, m := range plan.Modules { - out.modules = append(out.modules, m.Module) - } - return out - } - out := afterAct{err: err} - assigned, readErr := open.inventory.Assigned(ctx, node) - if readErr != nil { - return out - } - out.modules, out.unheld = assigned, unheldOver(shelf, node, assigned) - return out -} - -// splitModules is a surface's one `module` field as the modules it names: several, comma-separated, -// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the -// command API and the controller seat's verbs as they do from the command line. -func splitModules(field string) []string { - var out []string - for _, m := range strings.Split(field, ",") { - if m = strings.TrimSpace(m); m != "" { - out = append(out, m) - } - } - return out -} - -// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand. -// -// **The state, not the cause.** Saying "this assignment broke laptop" would mean resolving the -// whole mesh twice and would still be a guess about which of several changes did it; saying -// "laptop cannot be worked out, and here is what it says" is true, is what somebody has to fix, -// and cannot mislead. The machine that was just changed is left out because its own refusal is -// already the answer beside this one. -// -// Nothing here can fail the act it reports on. A mesh that cannot be read is worth saying and is -// not a reason to claim the assignment did not happen — it did. -func blockedElsewhere(ctx context.Context, open *stores, except string) string { - // One reading of the mesh for every machine this looks at. - ctx = withInputCache(ctx) - nodes, err := open.inventory.Nodes(ctx) - if err != nil { - return "\n\nThe rest of the mesh could not be checked: " + err.Error() - } - blocked := map[string]string{} - for _, n := range nodes { - if n.Name == except { - continue - } - if _, _, err := resolveNode(ctx, open, n.Name); err != nil { - blocked[n.Name] = err.Error() - } - } - if len(blocked) == 0 { - return "" - } - names := make([]string, 0, len(blocked)) - for name := range blocked { - names = append(names, name) - } - sort.Strings(names) - - var out strings.Builder - fmt.Fprintf(&out, "\n\nAND %d other machine(s) cannot be worked out as things stand, so "+ - "nothing will be sent to them:\n", len(names)) - for _, name := range names { - fmt.Fprintf(&out, " %s\n", name) - for _, line := range strings.Split(strings.TrimRight(blocked[name], "\n"), "\n") { - fmt.Fprintf(&out, " %s\n", strings.TrimSpace(line)) - } - } - out.WriteString("\nThis may or may not be what just changed — it is what is true now.") - return out.String() -} - -// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and -// says what it did in one line. Nothing for a module that declares no broker secret and runs no -// provisioner client; nothing for one whose user is already minted (a credential is rotated on -// purpose, never by re-assigning); and when the bus cannot be reached from here, the line names the -// verb and the push that would refuse the module until it is run — never a silent placeholder (novox/hq issue 203). -func issueOnAssign(ctx context.Context, open *stores, node, module string) string { - // Mint only what no row names: the assignments' watch hears the same write and may issue at once (ADR 0543). - ctx = onlyIfAbsent(ctx) - inv := open.inventory - shelf, err := inv.Catalogue(ctx) - if err != nil { - return "" - } - m, known := shelf[module] - if !known || mayIssue(m) != nil { - return "" - } - user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username() - _, reads := m.OwnSecrets["broker"] - if !reads && !runsTheProvisionerClient(m) { - // A module whose bus accounts are its processes' alone has none of its own: only the processes that - // are not issued yet are issued (issue 1226). - return processAccountsOnAssign(ctx, inv, m, node) - } - if !reads { - // A module running the provisioner client alone is minted the client's account (issue 917). - user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username() - } - if _, sealed, err := inv.BusAccountIssued(ctx, user); err != nil { - return "" - } else if sealed { // a row whose credential is not sealed to the node is not issued (issue 1945) - // Its own account stands and is kept; a process it gained is not issued yet (issue 1226). - return processAccountsOnAssign(ctx, inv, m, node) - } - busAddress, err := broker.BusAddress() - if err == nil { - err = issueOnTheNewBus(ctx, inv, m, node, busAddress) - } - if err != nil { - return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+ - "`push %s` refuses to send %s until it is", err, module, node, node, module) - } - return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node) -} - -// processAccountsOnAssign issues the process accounts of an assigned module not issued yet, and says what it did. -func processAccountsOnAssign(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) string { - line, err := issueMissingProcessAccounts(ctx, inv, m, node) - if err != nil { - return fmt.Sprintf("its process accounts are not issued (%v): `module issue %s --node %s` first — "+ - "`push %s` refuses to send %s until it is", err, m.Module, node, node, m.Module) - } - return line -} - -// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of -// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is -// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every -// change that says who may change what. -func refusedMovingTheController(modules []string) error { - verb, through := throughAVerb() - if !through || !slices.Contains(modules, controllerModule) { - return nil - } - return fmt.Errorf("%s is assigned and unassigned at the controller's terminal only (mesh-cli on the control-node), "+ - "never through a verb (this came through %q): the node it runs on is the control-node, whose operator is the "+ - "terminal (novox/hq ADR 0272). Nothing was assigned", controllerModule, verb) -} diff --git a/cmd/mesh-controller/acts_test.go b/cmd/mesh-controller/acts_test.go deleted file mode 100644 index a7c6f819..00000000 --- a/cmd/mesh-controller/acts_test.go +++ /dev/null @@ -1,135 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// What an assignment says about the machines it was not about. - -// **An assignment that blocks another machine says so.** -// -// The shape found on a four-node raise: a module offering a mesh-scoped provision stops offering it -// the moment its own node stops resolving, so an assignment to the provider's machine silently took -// a provision away from every consumer elsewhere. Those consumers were then told *nothing in this -// mesh provides it*, naming as the remedy a module that was already assigned — which read, on the -// way back, as a version bump breaking provider recognition. It was neither the version nor the -// provider: it was one machine's set of assignments, and nothing said so. -// -// novox/hq 04-ISSUES/017 names the general shape — an action succeeding into a state its own verify -// rejects, because the action's test is not the test the verify uses. `assign` tested one node; the -// verify is resolution over all of them. -func TestAnAssignmentThatBlocksAnotherMachineSaysWhichAndWhy(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - - // A provider on the hub and a consumer on the other machine, both resolving. - register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1", - Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}}) - register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1", - Requires: []string{"acme-ca"}}) - if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil { - t.Fatal(err) - } - said, err := assign(ctx, open, "laptop", "route-proxy") - if err != nil { - t.Fatalf("a mesh that should resolve did not: %v\n%s", err, said) - } - if strings.Contains(said, "cannot be worked out") { - t.Fatalf("a well mesh was reported as blocked:\n%s", said) - } - - // Now break the hub's own set, with nothing but the command a person has. - one, two := rivals() - register(t, open, one) - register(t, open, two) - if _, err := assign(ctx, open, "anchor", "rival-one"); err != nil { - t.Fatal(err) - } - said, err = assign(ctx, open, "anchor", "rival-two") - if err == nil { - t.Fatal("an assignment that makes its own node incoherent was not reported at all") - } - - // The node's own refusal is the error, as it always was. What is new is that the machines this - // just took a provision away from are named in the same breath. - if !strings.Contains(said, "laptop") { - t.Fatalf("the machine this blocked is not named:\n%s\n\n%v", said, err) - } - if !strings.Contains(said, "acme-ca") { - t.Fatalf("what laptop is now missing is not said:\n%s", said) - } - if !strings.Contains(said, "cannot be worked out as things stand") { - t.Fatalf("the report does not say what state the mesh is in:\n%s", said) - } - // And the assignment is kept: it is what a person meant, and assignment is not an ordering. - assigned, err := open.inventory.Assigned(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if !contains(assigned, "rival-two") { - t.Fatalf("the assignment was not kept: %v", assigned) - } -} - -// A consumer assigned before its provider is refused for itself and kept, because assignment is not -// an ordering — refusing the first half of a pair would make the order somebody types two commands -// in part of the mesh's rules. -func TestAConsumerAssignedBeforeItsProviderIsStillAssigned(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1", - Requires: []string{"acme-ca"}}) - - said, err := assign(ctx, open, "laptop", "route-proxy") - if err == nil { - t.Fatalf("a consumer with nothing to consume resolved:\n%s", said) - } - assigned, err := open.inventory.Assigned(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if !contains(assigned, "route-proxy") { - t.Fatalf("assignment became an ordering: %v", assigned) - } -} - -// Unassigning is the ordinary way to stop providing something to another machine, and it reports -// the same way for the same reason. -func TestUnassigningAProviderNamesWhoIsNowBlocked(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1", - Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}}) - register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1", - Requires: []string{"acme-ca"}}) - if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "route-proxy"); err != nil { - t.Fatal(err) - } - - said, err := unassign(ctx, open, "anchor", "step-ca") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(said, "laptop") || !strings.Contains(said, "acme-ca") { - t.Fatalf("taking the provider away said nothing about who was consuming it:\n%s", said) - } -} - -func contains(all []string, one string) bool { - for _, s := range all { - if s == one { - return true - } - } - return false -} diff --git a/cmd/mesh-controller/addresses_integration_test.go b/cmd/mesh-controller/addresses_integration_test.go deleted file mode 100644 index 5042cd91..00000000 --- a/cmd/mesh-controller/addresses_integration_test.go +++ /dev/null @@ -1,231 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "os" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/licences" -) - -func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aStore()) - if _, err := assign(ctx, open, "anchor", "distribution"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "distribution", - map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil { - t.Fatal(err) - } - // A module the mesh built, recorded by digest and path, running on the other machine. - if err := open.inventory.RecordBuild(ctx, inventory.Build{ - ID: "b1", Repository: "r", Module: "app", Commit: "abc", - Made: []inventory.Artifact{{Name: "server", Kind: "image", - Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}, - }); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "app", Version: "1", - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app", - "image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}}) - if _, err := assign(ctx, open, "laptop", "app"); err != nil { - t.Fatal(err) - } - // The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it - // asks where this machine reaches the store, as the docker module does. - register(t, open, catalogue.Manifest{Module: "runtime", Version: "1", - Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", - "into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}}) - if _, err := assign(ctx, open, "laptop", "runtime"); err != nil { - t.Fatal(err) - } - - on := map[string]bool{"anchor": true, "laptop": true} - node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on) - if err != nil || !found || node != "anchor" || port != "5101" { - t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err) - } - - var trust string - for _, r := range composed(t, open, "laptop").Resources { - if r["path"] == "/etc/docker/daemon.json" { - trust, _ = r["content"].(string) - } - if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest { - t.Errorf("the image the mesh built is fetched as %v", r["image"]) - } - } - if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") { - t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust) - } - - // And a replay to the catalogue says where the store is now. - announced, err := following{open: open}.Announceable(ctx) - if err != nil { - t.Fatal(err) - } - if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest { - t.Fatalf("the replay announces %+v", announced) - } -} - -func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) { - open := aMesh(t) - // Issue 1126: the placement export over this fresh mesh, before anything below changes it. - placementExportIsEachNodesPlacementFromTheStore(t, open) - ctx := t.Context() - raw, err := os.ReadFile("../../module.json") - if err != nil { - t.Fatal(err) - } - m, err := catalogue.Decode(raw) - if err != nil { - t.Fatal(err) - } - control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle, - Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}}) - if err != nil { - t.Fatal(err) - } - register(t, open, control) - register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", - Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, - Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", - "ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}}) - register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1", - Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}, - Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh}, - {Port: 5672, From: catalogue.FromMesh}}, - Guards: []int{15672}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker", - "ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}}) - // The control plane's own bus user is the installer's, seeded at genesis before the controller - // runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203). - if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller", - Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil { - t.Fatal(err) - } - // Seeded with the credential the controller already uses, so it is sealed: an account counts as issued only once - // sealed (ADR 0543 §2, PR #467), and genesis's seed is the credential in use. - if err := open.inventory.MarkBusAccountSealed(ctx, "anchor.mesh-controller"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - // The store's module is registered and given its port, and NOT assigned: the state genesis - // leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085). - if err := open.inventory.SetSettings(ctx, "anchor", "postgres", - map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq", - map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil { - t.Fatal(err) - } - - var env map[string]any - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == "mesh-controller.controller" { - env, _ = r["env"].(map[string]any) - } - } - if env == nil { - t.Fatal("the control plane's process is not in its own node's declaration") - } - for key, want := range map[string]string{ - "MESH_STORE_INVENTORY_PORT": "6852", - "MESH_STORE_IDENTITY_PORT": "6852", - "MESH_STORE_LICENCES_PORT": "6852", - "MESH_BROKER_AMQP_PORT": "5679", - // Neither given nor assigned by the mesh: the manifest's own number is NOT the answer, - // because the sealed value beside it carries the port genesis wrote (finding F3). - "MESH_BROKER_ADDRESS_PORT": "", - "MESH_BROKER_MANAGEMENT_PORT": "", - } { - if env[key] != want { - t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want) - } - } -} - -func aLoneNode(t *testing.T) *stores { - t.Helper() - inventory.ForTest(t) - licences.ForTest(t) - open, err := openStores(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(open.Close) - for _, m := range provided { - if err := open.inventory.Provide(t.Context(), m); err != nil { - t.Fatal(err) - } - } - record, err := open.inventory.AddNode(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{ - {"name": "container-runtime", "present": true}}}) - var profile map[string]any - _ = json.Unmarshal(reported, &profile) - if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - return open -} - -func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) { - open := aLoneNode(t) - ctx := t.Context() - register(t, open, aStore()) - register(t, open, catalogue.Manifest{Module: "builder", Version: "1", - Requires: []string{catalogue.ArtifactStoreProvision}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder", - "image": "registry.example/mesh-builder@" + aDigest}}}) - if err := open.inventory.RecordBuild(ctx, inventory.Build{ - ID: "b1", Repository: "r", Module: "postgres", Commit: "abc", - Made: []inventory.Artifact{{Name: "runtime", Kind: "image", - Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}, - }); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", - Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres", - "image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}}) - for _, module := range []string{"distribution", "builder", "postgres"} { - if _, err := assign(ctx, open, "anchor", module); err != nil { - t.Fatal(err) - } - } - if err := open.inventory.SetSettings(ctx, "anchor", "distribution", - map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil { - t.Fatal(err) - } - - var image any - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == "postgres.runtime" { - image = r["image"] - } - } - if image != "127.0.0.1:5100/postgres/runtime@"+aDigest { - t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image) - } -} diff --git a/cmd/mesh-controller/addresses_test.go b/cmd/mesh-controller/addresses_test.go deleted file mode 100644 index 09ed3438..00000000 --- a/cmd/mesh-controller/addresses_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "strings" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// An address is read from the node's settings where it is used, never recorded with a port -// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here. - -var aDigest = "sha256:" + strings.Repeat("e", 64) - -// aStore is a module offering the artifact store on 5000, published the long way as the -// distribution module does, so a node may be given another number for it. -func aStore() catalogue.Manifest { - return catalogue.Manifest{Module: "distribution", Version: "1", - Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}}, - Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}}, - Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry", - "ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}} -} - -// **The trust a machine writes for the store, and the address every built image is fetched -// through, say the port the node gave the store** — not the catalogue's number. - -// **The control plane's own connections say the port the node gave the store and the broker.** -// -// Composed from the control plane's own manifest against a real inventory: the store's module is -// given 6852 on this node the way genesis or an operator gives it, and the control plane's -// process is told so beside the sealed connection genesis wrote. - -// withSeatPorts is the control plane's manifest with the seat placeholders in its environment — -// added here until module.json carries them (the manifest lands one commit after the code that -// fills it, so a control plane one build behind never sees a placeholder it cannot fill). -func withSeatPorts(m catalogue.Manifest) catalogue.Manifest { - seatPorts := map[string]string{ - "MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}", - "MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}", - "MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}", - "MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}", - "MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}", - "MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}", - } - out := m - out.Resources = nil - for _, r := range m.Resources { - if r["type"] != "container" && r["type"] != "process" { - out.Resources = append(out.Resources, r) - continue - } - copied := map[string]any{} - for k, v := range r { - copied[k] = v - } - env := map[string]any{} - if had, ok := r["env"].(map[string]any); ok { - for k, v := range had { - env[k] = v - } - } - for k, v := range seatPorts { - if _, said := env[k]; !said { - env[k] = v - } - } - copied["env"] = env - out.Resources = append(out.Resources, copied) - } - return out -} - -// aLoneNode is one capable machine with nothing placed on any network — the control-node during -// genesis, before the "network" step, which is after the store, the broker, the vault and the -// catalogue have each been built and pushed (finding F2). - -// **Before the network exists, the store's own node reaches it by loopback** — never refused, -// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to -// a node on no network, and builds the catalogue on a base it must be able to pull. diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go deleted file mode 100644 index 9a95be52..00000000 --- a/cmd/mesh-controller/adopting_test.go +++ /dev/null @@ -1,660 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/nats-io/nats.go/jetstream" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" - "net/http" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/overlay" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// novox/hq ADR 0100: taking a module is its cutover; converging a node is one act, previewed, and -// refused while a found container is held; returning to adopted keeps what was taken. - -// anAdoptedAnchor is aMesh with the anchor adopted, running a served module the predecessor also -// runs and a module with only a file, and a filter module in the catalogue. -func anAdoptedAnchor(t *testing.T) (*stores, *[]string) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "hello-web", Version: "1", - Listens: []catalogue.Listening{{Port: 8080, From: catalogue.FromEverywhere}}, - Resources: []map[string]any{ - {"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hi"}, - {"id": "server", "type": "container", "name": "hello-web", "ports": []any{"8080:80"}, - "image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)}, - }}) - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{ - {"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}, - }}) - register(t, open, catalogue.Manifest{Module: "nftables", Version: "1", - Filtering: &catalogue.Filtering{Into: "/etc/nftables.conf"}, - Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service", - "state": "running", "restart-on": []any{"filtering"}}}}) - if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { - t.Fatal(err) - } - for _, m := range []string{"hello-web", "notes"} { - if _, err := assign(ctx, open, "anchor", m); err != nil { - t.Fatal(err) - } - } - onTheBusWithItsHaves(t, open) - sent := &[]string{} - saved := sendNodes - sendNodes = func(_ context.Context, _ *stores, names []string) error { - *sent = append(*sent, names...) - return nil - } - t.Cleanup(func() { sendNodes = saved }) - return open, sent -} - -// reportsHolding has the anchor report, on what it was last sent, holding what is given. -func reportsHolding(t *testing.T, open *stores, held ...link.Held) { - t.Helper() - reportsReaching(t, open, []link.Reach{ - {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}, - {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, - ContainerPort: 80}, - {Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry", - Published: true, ContainerPort: 5000}, - {Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker", - Published: true, ContainerPort: 15672}, - }, held...) -} - -// reportsReaching has the anchor report, on what it was last sent, what is reachable on it and -// holding what is given. -func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ...link.Held) { - t.Helper() - ctx := t.Context() - body, err := composed(t, open, "anchor").Body() - if err != nil { - t.Fatal(err) - } - if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{ - Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body), - Firewall: "ufw", Held: held, Reachable: reachable, - // A machine says which of its links face outside on every apply (novox/hq ADR 0140), and a - // filter is not sent to one that has not. The anchor reports one, as a real host does; this - // fixture lacked it from 2026-09-28 and nothing ran the test (issue 177). - Outward: []string{"eth0"}, - // And what filters it (ADR 0168): its front end, the runtime's own, and a chain a - // predecessor left in the runtime's user chain. - Filters: anchorFilters, - }); err != nil { - t.Fatal(err) - } -} - -// anchorFilters is what the adopted anchor says filters it: ufw's chains, the runtime's, and a -// predecessor's chain the mesh did not write. -var anchorFilters = []link.Filter{ - {Where: "table ip filter, chain ufw-reject-input", Owner: "found-firewall", Refuses: "reject"}, - {Where: "table ip filter, chain DOCKER", Owner: "runtime", Refuses: `iifname != "docker0" oifname "docker0" drop`}, - {Where: "table ip filter, chain DOCKER-USER", Owner: "other", Refuses: `iifname "eth0" tcp dport 6000 drop`}, -} - -var ( - heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container", - Target: "hello-web", Since: time.Now()} - heldFile = link.Held{ID: "notes.conf", Module: "notes", Kind: "file", - Target: "/etc/notes.conf", Since: time.Now(), Kept: "/var/lib/mesh-host/kept/abc-notes.conf"} -) - -func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) { - open, _ := anAdoptedAnchor(t) - if _, err := take(t.Context(), open, "anchor", "nftables", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAssigned) { - t.Fatalf("taking an unassigned module gave %v", err) - } - if _, err := take(t.Context(), open, "laptop", "network", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAdopted) { - t.Fatalf("taking on a converged node gave %v", err) - } -} - -func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) { - open, sent := anAdoptedAnchor(t) - _, err := converge(t.Context(), open, "anchor", false, "", "") - if err == nil || !strings.Contains(err.Error(), "has not reported") { - t.Fatalf("a node that never reported was previewed: %v", err) - } - if len(*sent) != 0 { - t.Fatal("a refused converge sent something") - } -} - -func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) { - open, sent := anAdoptedAnchor(t) - reportsHolding(t, open, heldContainer, heldFile) - _, err := converge(t.Context(), open, "anchor", true, "", "") - if err == nil || !strings.Contains(err.Error(), "take anchor hello-web once its data has moved") { - t.Fatalf("the flip was not refused while hello-web holds its found container: %v", err) - } - if n, _ := open.inventory.NodeByName(t.Context(), "anchor"); !n.Adopted || len(*sent) != 0 { - t.Fatal("a refused flip changed something") - } -} - -func TestTakingNamesWhatItReplaces(t *testing.T) { - open, _ := anAdoptedAnchor(t) - reportsHolding(t, open, heldContainer, heldFile) - ctx := t.Context() - // The machine holds something for the module, so the take acts on the preview the operator - // saw and names its digest (novox/hq ADR 0163). - preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{}) - if err != nil { - t.Fatal(err) - } - saw := takeDigestIn(t, preview) - if !strings.Contains(preview, "nothing taken; `take anchor hello-web --yes "+saw+"`") { - t.Fatalf("the preview does not say how to act on it:\n%s", preview) - } - if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 { - t.Fatal("the preview took something") - } - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil || !strings.Contains(err.Error(), "name its digest") { - t.Fatalf("--yes without the digest was not refused: %v", err) - } - said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(said, "container hello-web (hello-web.server)") || - !strings.Contains(said, "compositions of anchor are written") { - t.Fatalf("taking did not say what it replaces and what to run:\n%s", said) - } -} - -// takeDigestIn is the digest a take's preview printed. -func takeDigestIn(t *testing.T, preview string) string { - t.Helper() - for _, line := range strings.Split(preview, "\n") { - if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" { - return fields[1] - } - } - t.Fatalf("the preview printed no digest:\n%s", preview) - return "" -} - -// A take acts on the preview the operator saw, and on an account of the machine that is still the -// machine: a changed preview and a stale account refuse (novox/hq ADR 0163, rule 1). -func TestATakeIsRefusedOnAChangedPreviewOrAStaleAccount(t *testing.T) { - open, _ := anAdoptedAnchor(t) - ctx := t.Context() - reportsHolding(t, open, heldContainer, heldFile) - preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{}) - if err != nil { - t.Fatal(err) - } - saw := takeDigestIn(t, preview) - - // The machine reports again, and what it holds has changed: the found container now carries - // facts the preview never showed. - changed := heldContainer - changed.Facts = map[string]any{"image": "hello:2", "declared_image": "registry.example/hello"} - reportsHolding(t, open, changed, heldFile) - _, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}) - if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) { - t.Fatalf("a changed preview was acted on: %v", err) - } - if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 { - t.Fatal("a refused take took something") - } - - // And an account older than the flip allows. - preview, err = take(ctx, open, "anchor", "hello-web", takeOptions{}) - if err != nil { - t.Fatal(err) - } - saw = takeDigestIn(t, preview) - saved := reportFreshFor - reportFreshFor = -time.Second - defer func() { reportFreshFor = saved }() - _, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}) - if err == nil || !strings.Contains(err.Error(), "a take acts only on an account newer than") { - t.Fatalf("a stale account was acted on: %v", err) - } - reportFreshFor = saved - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}); err != nil { - t.Fatal(err) - } - // A module the machine holds nothing for has nothing to compare: --yes alone suffices. - if _, err := take(ctx, open, "anchor", "notes", takeOptions{Yes: true}); err == nil { - // notes holds a file, so this one needs the digest too. - t.Fatal("notes holds a found file and was taken without a digest") - } -} - -func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) { - open, sent := anAdoptedAnchor(t) - ctx := t.Context() - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { - t.Fatal(err) - } - *sent = nil // the take composed the node; what follows is judged from here - reportsHolding(t, open, heldFile) - - preview, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - for _, want := range []string{ - "tcp/8080 hello-web (published, container port 80)", - "declared by hello-web (from anywhere)", - "WILL CLOSE — no module assigned here declares it", - // The anchor faces inward and is on the private network: the derived filter admits ssh - // from the mesh only. - "WILL CLOSE to everything outside the private network — ssh stays open from the mesh", - "notes\n replacing the found file /etc/notes.conf (notes.conf), original kept at", - "assigns nftables", - "the found firewall (ufw) is disabled, never flushed", - // What it routes is not a listener: said not to be previewed, and to be dropped. - "not previewed: traffic the machine routes that is not a published port", - "the derived filter drops it unless a module declares it", - } { - if !strings.Contains(preview, want) { - t.Errorf("the preview does not say %q:\n%s", want, preview) - } - } - if strings.Contains(preview, "15672") { - t.Errorf("a loopback listener is in the preview:\n%s", preview) - } - // What filters the machine now, and the fate of each (novox/hq ADR 0168): the predecessor's - // chain is named as not the mesh's and left, so the reader knows before the flip. - for _, want := range []string{ - "table ip filter, chain DOCKER-USER", - "NOT THE MESH'S; left in force", - `iifname "eth0" tcp dport 6000 drop`, - "table ip filter, chain ufw-reject-input", - "the found firewall's; retired with it", - "the container runtime's own; left", - } { - if !strings.Contains(preview, want) { - t.Errorf("the preview does not say %q:\n%s", want, preview) - } - } - for _, line := range strings.Split(preview, "\n") { - if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") { - t.Errorf("an undeclared published port is not said to close: %s", line) - } - } - if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 { - t.Fatal("the preview changed something") - } - - if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil { - t.Fatal(err) - } - n, _ := open.inventory.NodeByName(ctx, "anchor") - if n.Adopted { - t.Fatal("converge --yes left the node adopted") - } - taken, _ := open.inventory.Taken(ctx, "anchor") - if !reflect.DeepEqual(taken, []string{"hello-web", overlay.Name, "nftables", "notes"}) { - t.Fatalf("the flip took %v", taken) - } - if !reflect.DeepEqual(*sent, []string{"anchor"}) { - t.Fatalf("the flip sent %v", *sent) - } - declared := composed(t, open, "anchor") - if declared.Adoption != nil { - t.Fatal("a converged node is still sent an adoption envelope") - } - if !hasID(declared.Resources, "nftables.filtering") { - t.Fatal("the converged node is not declared the mesh's filter") - } - - if _, err := adopt(ctx, open, "anchor"); err != nil { - t.Fatal(err) - } - if _, err := adopt(ctx, open, "anchor"); err == nil { - t.Fatal("adopting an adopted node was not refused") - } - again, _ := open.inventory.Taken(ctx, "anchor") - if !reflect.DeepEqual(again, taken) { - t.Fatalf("returning to adopted lost what was taken: %v", again) - } - declared = composed(t, open, "anchor") - if declared.Adoption == nil || len(declared.Adoption.Untaken) != 0 { - t.Fatalf("returned to adopted, the envelope is %+v", declared.Adoption) - } - if hasID(declared.Resources, "nftables.filtering") || hasID(declared.Resources, "nftables.load") { - t.Fatal("returned to adopted, the mesh's filter is still declared") - } -} - -func hasID(resources []map[string]any, id string) bool { - for _, r := range resources { - if r["id"] == id { - return true - } - } - return false -} - -// The command API refuses a flip exactly as the command line does, in the same words. -func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) { - open, _ := anAdoptedAnchor(t) - reportsHolding(t, open, heldContainer) - _, direct := converge(t.Context(), open, "anchor", true, "", "") - if direct == nil { - t.Fatal("the flip was not refused") - } - got := asking(t, letIn{}, "POST", "/converge", `{"node":"anchor","yes":true}`) - if got.Code != http.StatusConflict { - t.Fatalf("got %d: %s", got.Code, got.Body.String()) - } - var said map[string]any - if err := json.Unmarshal(got.Body.Bytes(), &said); err != nil { - t.Fatal(err) - } - if said["refused"] != direct.Error() { - t.Fatalf("the API said %q and the command line %q", said["refused"], direct.Error()) - } - if got := asking(t, letIn{}, "POST", "/take", `{"node":"anchor"}`); got.Code != http.StatusBadRequest { - t.Fatalf("a take naming no module got %d", got.Code) - } -} - -// novox/hq ADR 0100: the preview says what the derived filter does, rendered as it is rendered. On -// a machine that faces inward, ssh is admitted from the private network only, and a port a module -// admits from the mesh only closes to everything outside it: both are said to close. -func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) { - open, _ := anAdoptedAnchor(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "store", Version: "1", - Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}}) - if _, err := assign(ctx, open, "anchor", "store"); err != nil { - t.Fatal(err) - } - reportsReaching(t, open, []link.Reach{ - {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}, - {Protocol: "tcp", Address: "0.0.0.0", Port: 5432, By: "postgres"}, - {Protocol: "tcp", Address: "10.77.0.1", Port: 5432, By: "postgres"}, - {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, - ContainerPort: 80}, - }) - preview, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - lines := map[string]string{} - for _, line := range strings.Split(preview, "\n") { - fields := strings.Fields(line) - if len(fields) > 1 && strings.HasPrefix(fields[0], "tcp/") { - lines[fields[0]+" "+fields[1]] += line + "\n" - } - } - if got := lines["tcp/22 sshd"]; !strings.Contains(got, "WILL CLOSE to everything outside "+ - "the private network") { - t.Errorf("ssh on an inward machine is not said to close outside the mesh:\n%s", preview) - } - store := lines["tcp/5432 postgres"] - if strings.Count(store, "WILL CLOSE to everything outside the private network") != 1 || - !strings.Contains(store, "declared by store (from mesh)") { - t.Errorf("the store's narrowing is not said to close, or its mesh address is:\n%s", preview) - } - if got := lines["tcp/8080 hello-web"]; !strings.Contains(got, "declared by hello-web (from anywhere)") { - t.Errorf("a port open to everywhere is not said to stay:\n%s", preview) - } -} - -// digestIn is the digest a converge preview printed. -func digestIn(t *testing.T, preview string) string { - t.Helper() - for _, line := range strings.Split(preview, "\n") { - if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" { - return fields[1] - } - } - t.Fatalf("the preview printed no digest:\n%s", preview) - return "" -} - -// The flip acts on the preview the operator saw: it names that preview's digest, and it is refused -// when the digest is missing, when anything the preview says has changed since, or when the node's -// account of itself is too old to be the machine as it is. -func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) { - open, sent := anAdoptedAnchor(t) - ctx := t.Context() - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { - t.Fatal(err) - } - *sent = nil // the take composed the node; what follows is judged from here - reportsHolding(t, open, heldFile) - preview, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - saw := digestIn(t, preview) - if !strings.Contains(preview, "converge anchor --yes "+saw) { - t.Fatalf("the preview does not say how to act on it:\n%s", preview) - } - unchanged := func() { - t.Helper() - if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 { - t.Fatal("a refused flip changed something") - } - } - - if _, err := converge(ctx, open, "anchor", true, "", ""); err == nil || - !strings.Contains(err.Error(), "--yes "+saw) { - t.Fatalf("a flip naming no preview was not refused: %v", err) - } - unchanged() - - // Something new is reachable: the preview the operator saw is not what would happen. - reportsReaching(t, open, []link.Reach{ - {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}, - {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, - ContainerPort: 80}, - {Protocol: "tcp", Address: "0.0.0.0", Port: 6000, By: "something-new"}, - }, heldFile) - _, err = converge(ctx, open, "anchor", true, saw, "") - if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) { - t.Fatalf("a flip on a changed preview was not refused: %v", err) - } - unchanged() - - // An account older than the flip trusts is refused, whatever digest is named. - again, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - saved := reportFreshFor - reportFreshFor = time.Nanosecond - _, err = converge(ctx, open, "anchor", true, digestIn(t, again), "") - reportFreshFor = saved - if err == nil || !strings.Contains(err.Error(), "wait for its next report") { - t.Fatalf("a flip on an old account was not refused: %v", err) - } - unchanged() - - if _, err := converge(ctx, open, "anchor", true, digestIn(t, again), ""); err != nil { - t.Fatalf("the flip on the preview just seen was refused: %v", err) - } - if n, _ := open.inventory.NodeByName(ctx, "anchor"); n.Adopted { - t.Fatal("the flip did not converge the node") - } -} - -// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found, -// not only its files, and the digest changes when any of them does. -func TestThePreviewNamesEveryHeldKind(t *testing.T) { - open, _ := anAdoptedAnchor(t) - ctx := t.Context() - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { - t.Fatal(err) - } - since := time.Now() - held := []link.Held{heldFile, - {ID: "notes.data", Module: "notes", Kind: "directory", Target: "/var/lib/notes", Since: since}, - {ID: "notes.daemon", Module: "notes", Kind: "service", Target: "notes.service", Since: since}, - {ID: "notes.seed", Module: "notes", Kind: "archive", Target: "/srv/notes", Since: since}, - {ID: "notes.worker", Module: "notes", Kind: "process", Target: "notes-worker", Since: since}, - {ID: "notes.account", Module: "notes", Kind: "user", Target: "notes", Since: since}, - } - reportsHolding(t, open, held...) - preview, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - for _, want := range []string{ - "replacing the found directory /var/lib/notes (notes.data)", - "replacing the found service notes.service (notes.daemon)", - "replacing the found archive /srv/notes (notes.seed)", - "replacing the found process notes-worker (notes.worker)", - "replacing the found user notes (notes.account)", - } { - if !strings.Contains(preview, want) { - t.Errorf("the preview does not say %q:\n%s", want, preview) - } - } - reportsHolding(t, open, held[:len(held)-1]...) - fewer, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - if digestIn(t, fewer) == digestIn(t, preview) { - t.Fatal("the digest does not change with what is held") - } -} - -// novox/hq ADR 0100: the flip acts on what the node said is reachable, so an account naming nothing -// is refused. Every machine that is up answers on ssh; nothing reported means the host's collectors -// did not, and flipping would close ports the preview never named. -func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) { - open, sent := anAdoptedAnchor(t) - ctx := t.Context() - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { - t.Fatal(err) - } - *sent = nil // the take composed the node; what follows is judged from here - // Only a loopback listener: nothing off the machine, which is the same silence. - reportsReaching(t, open, []link.Reach{ - {Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"}, - }, heldFile) - preview, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(preview, "this account looks partial") { - t.Errorf("the preview does not mark a partial account:\n%s", preview) - } - _, err = converge(ctx, open, "anchor", true, digestIn(t, preview), "") - if err == nil || !strings.Contains(err.Error(), "says nothing is reachable on it") { - t.Fatalf("the flip was not refused on an account naming nothing: %v", err) - } - if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 { - t.Fatal("a refused flip changed something") - } -} - -// onTheBusWithItsHaves gives the controller a bus that holds, for the anchor, a want and a have of its module that agree: the node -// has everything wanted of it, as read from the haves (novox/hq ADR 0539 §9), so a flip is refused for what it is about and -// not because the bus could not be read. -func onTheBusWithItsHaves(t *testing.T, open *stores) { - t.Helper() - ctx := t.Context() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - servingBus.Store(js) - t.Cleanup(func() { servingBus.Store(nil) }) - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - fp := sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64) - composed := "sha256:" + strings.Repeat("b", 64) - put := func(bucket, key string, value []byte) { - kv, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket}) - if err != nil { - t.Fatal(err) - } - if _, err := kv.Put(ctx, key, value); err != nil { - t.Fatal(err) - } - } - wantKey, err := sdkdelivery.WantKey("anchor", "hello-web") - if err != nil { - t.Fatal(err) - } - haveKey, err := sdkdelivery.HaveKey("anchor", "hello-web") - if err != nil { - t.Fatal(err) - } - want, err := sdkdelivery.EncodeWant(sdkdelivery.Want{FP: fp, Composed: composed}) - if err != nil { - t.Fatal(err) - } - have, err := sdkdelivery.EncodeHave(sdkdelivery.Have{FP: fp, Composed: composed, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}) - if err != nil { - t.Fatal(err) - } - put(sdkdelivery.AssignerWantsBucket, wantKey, want) - put(sdkdelivery.HaveBucket, haveKey, have) -} - -// **When the bus cannot be read, whether the machine runs what is wanted is not known, and the flip says exactly that** (issue -// 1945): not "push", and not "has not reported". -func TestTheFlipIsRefusedWithTheBusUnreachableSayingItCannotTell(t *testing.T) { - open, sent := anAdoptedAnchor(t) - reportsHolding(t, open, heldContainer, heldFile) - servingBus.Store(nil) // this process has no bus now, and no address to dial - _, err := converge(t.Context(), open, "anchor", true, "", "") - if err == nil || !strings.Contains(err.Error(), "cannot tell whether anchor runs what is wanted: the bus is not reachable") || - strings.Contains(err.Error(), "push") { - t.Fatalf("the flip with the bus out of reach: %v", err) - } - if len(*sent) != 0 { - t.Fatal("a refused flip sent something") - } -} - -// **A take composes the node only when it is performed** (issue 1945): a preview, a take that names no digest and a take refused -// for a changed preview compose nothing; the performed take composes the machine once, and says so. -func TestATakeComposesTheNodeOnlyWhenItIsPerformed(t *testing.T) { - open, sent := anAdoptedAnchor(t) - ctx := t.Context() - reportsHolding(t, open, heldContainer, heldFile) - preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{}) - if err != nil { - t.Fatal(err) - } - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil { - t.Fatal("a take of a held module without the preview's digest was performed") - } - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: "not-the-digest"}); err == nil { - t.Fatal("a take naming another preview's digest was performed") - } - if len(*sent) != 0 { - t.Fatalf("a preview or a refused take composed %v", *sent) - } - said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: takeDigestIn(t, preview)}) - if err != nil { - t.Fatal(err) - } - if len(*sent) != 1 || (*sent)[0] != "anchor" || !strings.Contains(said, "compositions of anchor are written") { - t.Fatalf("the performed take composed %v and said:\n%s", *sent, said) - } -} diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go deleted file mode 100644 index b61a124d..00000000 --- a/cmd/mesh-controller/adoption.go +++ /dev/null @@ -1,1113 +0,0 @@ -package main - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "flag" - "fmt" - "slices" - "sort" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/overlay" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the -// mesh reports a node's state: node list, node show, status and the board. - -// showMode is the node show lines about a node's mode and what was taken on it. -func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node) error { - if !node.Adopted { - fmt.Printf(" mode converged\n") - // A converged machine holds nothing, and can still run what nobody asked for - // (novox/hq ADR 0163): what it reports as strays is said whatever its mode. - if said, err := inv.AdoptionOf(ctx, node.Name); err == nil && len(said.Strays) > 0 { - showStrays(said.Strays) - } - // And what filters it, truthfully (novox/hq ADR 0168): the mesh alone, or not. - if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil { - showFiltering(filtering, false) - } - return nil - } - fmt.Printf(" mode adopted since %s\n", - node.AdoptedSince.Local().Format(time.DateTime)) - taken, err := inv.Taken(ctx, node.Name) - if err != nil { - return err - } - if len(taken) == 0 { - fmt.Printf(" taken nothing yet\n") - } else { - fmt.Printf(" taken %s\n", strings.Join(taken, ", ")) - } - said, err := inv.AdoptionOf(ctx, node.Name) - if err != nil { - return err - } - if said.At.IsZero() { - fmt.Printf(" it has not yet said what it found\n") - return nil - } - fmt.Printf(" firewall found %s\n", orNone(said.Firewall)) - if err := showTunnel(ctx, inv, node.Name); err != nil { - return err - } - if len(said.Held) == 0 { - fmt.Printf(" holding nothing found\n") - } - for _, h := range said.Held { - // A held thing that changed is how a predecessor still writing is caught: said first. - line := fmt.Sprintf(" holds %-11s %s %s, for %s", h.Kind, h.Target, h.ID, h.Module) - if h.Changed != "" { - line += " — " + strings.ToUpper(h.Changed) + " by something other than the mesh" - } - fmt.Println(line) - if h.Kept != "" { - fmt.Printf(" %-17s original kept at %s\n", "", h.Kept) - } - for _, f := range comparisonLines(h) { - fmt.Printf(" %-17s %s\n", "", f) - } - } - showStrays(said.Strays) - if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil { - showFiltering(filtering, true) - } - fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime)) - return nil -} - -// showFiltering says what filters a machine, with owners (novox/hq ADR 0168), and for a converged -// machine the state of the firewall it was found with. A machine that has not said is not said to -// be filtered by anything. -func showFiltering(f inventory.Filtering, adopted bool) { - if len(f.Filters) == 0 && f.FoundFirewall == nil { - return - } - if fw := f.FoundFirewall; fw != nil && !adopted { - switch { - case fw.Active: - fmt.Printf(" found firewall %s is ACTIVE on this converged machine; the next apply retires it again\n", fw.Kind) - case fw.RetiredBy == "removed": - fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0180)\n", fw.Kind) - case fw.RetiredBy == inventory.FilterMesh || fw.RetiredBy == "mesh": - fmt.Printf(" found firewall %s, retired by the mesh; its configuration stays on disk\n", fw.Kind) - case fw.RetiredBy != "": - fmt.Printf(" found firewall %s, found inactive — not by the mesh\n", fw.Kind) - default: - fmt.Printf(" found firewall %s, inactive\n", fw.Kind) - } - } - if len(f.Filters) == 0 { - return - } - if f.Alone() { - fmt.Printf(" filtered by the mesh alone (%s)\n", filterSummary(f.Filters)) - return - } - fmt.Printf(" filtered by NOT the mesh alone: %d rule set(s) the mesh did not write refuse traffic here\n", len(f.Others())) - for _, x := range f.Filters { - if x.Owner == inventory.FilterOther || x.Owner == inventory.FilterFoundFirewall { - fmt.Printf(" %-17s %s — %s: %s\n", "", x.Where, x.Owner, x.Refuses) - } - } - fmt.Printf(" %-17s and its own: %s\n", "", filterSummary(f.Filters)) -} - -// filterSummary counts a machine's filters by owner: "mesh 2, runtime 3, ban 1". -func filterSummary(filters []inventory.Filter) string { - counts := map[string]int{} - for _, x := range filters { - counts[x.Owner]++ - } - var parts []string - for _, owner := range []string{inventory.FilterMesh, inventory.FilterRuntime, inventory.FilterBan, inventory.FilterFoundFirewall, inventory.FilterOther} { - if n := counts[owner]; n > 0 { - parts = append(parts, fmt.Sprintf("%s %d", owner, n)) - } - } - return strings.Join(parts, ", ") -} - -// showStrays says what a machine runs that the mesh neither wrote nor holds (ADR 0163). -func showStrays(strays []inventory.Stray) { - if len(strays) == 0 { - return - } - fmt.Printf(" strays %d container(s) the mesh neither wrote nor holds:\n", len(strays)) - for _, s := range strays { - fmt.Printf(" %-17s %s (%s)\n", "", s.Name, s.Detail) - } -} - -// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq -// ADR 0105): what it presented at enrolment, and what it last said about taking it over. -func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error { - tunnel, err := inv.TunnelOf(ctx, name) - if errors.Is(err, inventory.ErrNoTunnel) { - return nil - } - if err != nil { - return err - } - fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n", - tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers)) - carried, said, err := inv.CarriedTunnelOf(ctx, name) - if err != nil { - return err - } - switch { - case !said: - fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "") - case carried.State == inventory.CarriedTaken: - fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+ - "runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers) - case carried.State == inventory.CarriedDown: - fmt.Printf(" %-17s TUNNEL DOWN: %s is stopped and the mesh's interface is not up — the peers "+ - "reach nothing. On the machine: systemctl start %s\n", "", carried.Interface, - "wg-quick@"+carried.Interface) - default: - fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface) - } - if said && carried.Note != "" { - fmt.Printf(" %-17s %s\n", "", carried.Note) - } - if said && carried.Kept != "" { - fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept) - } - return nil -} - -func orNone(s string) string { - if s == "" { - return "none reported" - } - return s -} - -// adoptedNodes are the names of every adopted node, in the order given. -func adoptedNodes(nodes []inventory.Node) []string { - var out []string - for _, n := range nodes { - if n.Adopted { - out = append(out, n.Name) - } - } - return out -} - -// The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the -// command API alike, so a refusal is the same refusal in the same words at both (ADR 0035). - -// sendNodes composes the named machines as they should be now (novox/hq ADR 0539 clause 5): each node's want then -// names what changed. A variable so a test can see what an act would compose without a broker. -var sendNodes = composeAndSay - -// DefaultFilter is the module converging a node assigns to load the mesh's derived filter. -const DefaultFilter = "nftables" - -// take is a module's cutover on an adopted node: the operator's act, done when that module's data -// has moved. From the next push its resources converge there like any other, replacing what the -// node found and holds for it. -// -// **Previewed, and the preview is a comparison** (novox/hq ADR 0163): for every held thing the -// module would replace, what runs beside what the module declares, and the difference; the -// module's secrets on the machine and where each came from; its settings on the machine. Without -// --yes the comparison is printed and nothing changes. `--yes ` cuts over exactly what was -// previewed, the way the flip is confirmed: the preview ends with a digest of what it said, and a -// take naming an older one, or acting on an account of the machine older than the flip allows, is -// refused. A module the machine holds nothing for has nothing to compare, and `--yes` suffices. -// takeOptions is what a take was told about the differences it may pass (novox/hq ADR 0163). -type takeOptions struct { - Yes bool - // Digest is the preview's, named with --yes; required whenever the machine holds something - // for the module. - Digest string - Downgrade bool - Replace map[string]bool - // Mint names the secrets the service shall take a new value for, although the mesh minted - // one and the service already has its own (rule 2). - Mint map[string]bool -} - -func take(ctx context.Context, open *stores, node, module string, opts takeOptions) (string, error) { - inv := open.inventory - assigned, err := inv.Assigned(ctx, node) - if err != nil { - return "", err - } - if !slices.Contains(assigned, module) { - if plan, _, err := resolveNode(ctx, open, node); err == nil { - if why, runs := plan.Because[module]; runs { - return "", fmt.Errorf("%w: %s runs on %s because %s — assign it to %s to take it", - inventory.ErrNotAssigned, module, node, why, node) - } - } - } - // The comparison first (novox/hq ADR 0163): every held thing the module would replace, beside - // what the module declares, and the differences that refuse unless named. - c, err := comparisonFor(ctx, open, node, module) - if err != nil { - return "", err - } - preview, refusals, saw := comparisonOf(module, c, opts) - if len(refusals) > 0 { - return "", fmt.Errorf("taking %s on %s is refused:\n %s\n%s", module, node, - strings.Join(refusals, "\n "), preview) - } - holds := len(heldOf(c.reported, module)) > 0 - if holds { - preview += "\n preview " + saw - } - if !opts.Yes { - if !holds { - return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes` declares it as the mesh's own", node, module), nil - } - return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes %s` cuts it over as previewed", node, module, saw), nil - } - if holds { - // The take acts on the preview the operator saw, and on an account of the machine that - // is still the machine: the same two refusals the flip makes. - if age := time.Since(c.reported.At); age > reportFreshFor { - return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+ - "an account newer than %s: wait for its next report, then preview again", - node, age.Round(time.Second), reportFreshFor) - } - if opts.Digest == "" { - return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+ - "`take %s %s --yes %s`, once you have read it", module, node, node, module, saw) - } - if opts.Digest != saw { - return preview, fmt.Errorf("what taking %s on %s would replace has changed since preview %s "+ - "(it is now %s): read the preview above, and run `take %s %s --yes %s` if it is "+ - "what you want", module, node, opts.Digest, saw, node, module, saw) - } - } - if err := inv.Take(ctx, node, module); err != nil { - return "", err - } - said := fmt.Sprintf("%s is taken on %s", module, node) - if holds { - said += "; its next composition replaces what the node found and holds for it:\n" + preview - } - if err := sendNodes(ctx, open, []string{node}); err != nil { - return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err - } - return said + fmt.Sprintf("\n the compositions of %s are written; its wants then cut it over", node), nil -} - -// comparison is everything a take puts beside what the module declares: the machine's account of -// what it holds and what is reachable on it, the module's secrets on the machine, its settings -// there, and which found networks a setting keeps for each of its containers (by held id). -type comparison struct { - reported inventory.Adoption - secrets []inventory.SecretState - layers []catalogue.Layer - keeps map[string][]string - // settingsRefused is why the module's settings cannot compose with its definition, when - // they cannot — the module would be left out of the declaration (rule 6). - settingsRefused string -} - -func comparisonFor(ctx context.Context, open *stores, node, module string) (comparison, error) { - inv := open.inventory - var c comparison - var err error - if c.reported, err = inv.AdoptionOf(ctx, node); err != nil { - return c, err - } - if c.secrets, err = inv.SecretsOf(ctx, node, module); err != nil { - return c, err - } - if c.layers, err = inv.SettingsFor(ctx, node, module); err != nil { - return c, err - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - return c, err - } - if m, known := shelf[module]; known && len(c.layers) > 0 { - if err := catalogue.JudgeSettings(m, c.layers, true); err != nil { - c.settingsRefused = err.Error() - } - if kept, err := catalogue.KeptNetworks(m, c.layers, true); err == nil && len(kept) > 0 { - c.keeps = map[string][]string{} - for id, networks := range kept { - c.keeps[module+"."+id] = networks - } - } - } - return c, nil -} - -// heldOf is what a node holds for one module. -func heldOf(reported inventory.Adoption, module string) []inventory.Held { - var out []inventory.Held - for _, h := range reported.Held { - if h.Module == module { - out = append(out, h) - } - } - return out -} - -// comparisonOf is a take's preview: for every held thing of the module, what runs beside what the -// module declares; its secrets and its settings on the machine; and the refusals the differences -// earn unless the take named them (novox/hq ADR 0163): an image older than the one running, a -// declared file that differs from the found one, a secret the mesh minted for a service whose data -// was found. A narrowed port and a shared network are said and not refused. The digest is of what -// the preview says, so anything in it changing changes the digest. -func comparisonOf(module string, c comparison, opts takeOptions) (preview string, refusals []string, digest string) { - var b strings.Builder - held := heldOf(c.reported, module) - foundData := false - for _, h := range held { - if h.Kind == "container" || h.Kind == "directory" { - foundData = true - } - fmt.Fprintf(&b, " %s", heldLine(h)) - if h.Kept != "" { - fmt.Fprintf(&b, ", original kept at %s", h.Kept) - } - b.WriteString("\n") - for _, line := range comparisonLinesWith(h, c.keeps[h.ID], c.reported) { - fmt.Fprintf(&b, " %s\n", line) - } - f := factsOf(h) - if f.downgrade && !opts.Downgrade { - refusals = append(refusals, fmt.Sprintf("%s: the module's image (%s, made %s) is older than the one running (%s, made %s) — "+ - "a service that migrated its data forward may not start on it; `--downgrade` to take it anyway", - h.Target, f.declaredImage, day(f.declaredCreated), f.image, day(f.imageCreated))) - } - if f.differs && !opts.Replace[h.Target] && !opts.Replace["*"] { - refusals = append(refusals, fmt.Sprintf("%s: the module's content differs from the file found; the lines above "+ - "marked - are lost by taking it; `--replace %s` to replace it anyway, or declare the file partially", - h.Target, h.Target)) - } - } - // The module's secrets on the machine (rule 2 and 3): a service whose data was found already - // has a value for each, so one the mesh minted and nobody accepted refuses unless --mint says - // the service shall take a new one. - for _, sec := range c.secrets { - name := sec.Name - if sec.Local != "" { - name += " (" + sec.Local + ")" - } - what := "own secret" - accept := fmt.Sprintf("`secret accept %s %s`", module, sec.Name) - if !sec.Own() { - what = "secret from " + sec.Provider - accept = fmt.Sprintf("`secret accept %s %s --provider %s`", module, sec.Name, sec.Provider) - if sec.Local != "" { - accept = strings.TrimSuffix(accept, "`") + " --local " + sec.Local + "`" - } - } - switch { - case sec.Origin == inventory.OriginAccepted: - fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name) - case sec.Origin == inventory.OriginIssued: - // Taken from the provider's software (novox/hq ADR 0305): the value the running service uses. - fmt.Fprintf(&b, " %s %s: issued by the provider's software, carried in as it is\n", what, name) - case opts.Mint[sec.Name]: - fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name) - case foundData: - fmt.Fprintf(&b, " %s %s: MINTED by the mesh and not accepted — the running service already has one\n", what, name) - refusals = append(refusals, fmt.Sprintf("%s: the mesh minted a value and the service whose data was found "+ - "already uses its own; %s carries the existing value in, or `--mint %s` says the service shall take "+ - "the new one", name, accept, sec.Name)) - default: - fmt.Fprintf(&b, " %s %s: minted by the mesh\n", what, name) - } - } - // And its settings on this machine, composed against its definition (rule 1, rule 6). - for _, layer := range c.layers { - said := layer.Whole() - keys := make([]string, 0, len(said)) - for k := range said { - keys = append(keys, k) - } - sort.Strings(keys) - fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", ")) - } - if c.settingsRefused != "" { - fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the composition leaves it out: %s\n", c.settingsRefused) - } - preview = strings.TrimRight(b.String(), "\n") - sum := sha256.Sum256([]byte(preview)) - return preview, refusals, hex.EncodeToString(sum[:])[:12] -} - -// facts is a held thing's facts as the preview reads them. -type facts struct { - image, imageCreated, declaredImage, declaredCreated string - downgrade, differs bool - networks map[string][]string - mounts, ports, declaredPorts, declaredVolumes []string - difference []string -} - -func factsOf(h inventory.Held) facts { - var f facts - if h.Facts == nil { - return f - } - str := func(k string) string { s, _ := h.Facts[k].(string); return s } - list := func(k string) []string { - var out []string - if raw, ok := h.Facts[k].([]any); ok { - for _, x := range raw { - if s, ok := x.(string); ok { - out = append(out, s) - } - } - } - return out - } - f.image, f.imageCreated = str("image"), str("image_created") - f.declaredImage, f.declaredCreated = str("declared_image"), str("declared_image_created") - f.downgrade, _ = h.Facts["downgrade"].(bool) - f.differs, _ = h.Facts["differs"].(bool) - f.mounts, f.ports = list("mounts"), list("ports") - f.declaredPorts, f.declaredVolumes, f.difference = list("declared_ports"), list("declared_volumes"), list("difference") - if raw, ok := h.Facts["networks"].(map[string]any); ok { - f.networks = map[string][]string{} - for name, members := range raw { - var out []string - if ms, ok := members.([]any); ok { - for _, m := range ms { - if s, ok := m.(string); ok { - out = append(out, s) - } - } - } - f.networks[name] = out - } - } - return f -} - -// comparisonLines says a held thing's facts the way a person weighs them. -func comparisonLines(h inventory.Held) []string { - return comparisonLinesWith(h, nil, inventory.Adoption{}) -} - -// comparisonLinesWith is comparisonLines knowing which found networks this machine's setting keeps -// for the container (rule 4) and what the machine reports reachable, so a published port's reach -// is said beside the port (rule 1). -func comparisonLinesWith(h inventory.Held, keeps []string, reported inventory.Adoption) []string { - f := factsOf(h) - var out []string - if f.image != "" || f.declaredImage != "" { - line := fmt.Sprintf("runs %s", orNone(f.image)) - if f.imageCreated != "" { - line += " (made " + day(f.imageCreated) + ")" - } - line += "; the module declares " + orNone(f.declaredImage) - switch { - case f.declaredCreated != "": - line += " (made " + day(f.declaredCreated) + ")" - case f.declaredImage != "": - line += " (not on the machine yet, so its age is unknown)" - } - if f.downgrade { - line += " — DOWNGRADE" - } - out = append(out, line) - } - names := make([]string, 0, len(f.networks)) - for n := range f.networks { - names = append(names, n) - } - sort.Strings(names) - for _, n := range names { - members := f.networks[n] - if len(members) == 0 { - continue - } - if slices.Contains(keeps, n) { - out = append(out, fmt.Sprintf("on the network %s with %s — kept by this machine's setting, so they still reach it by name once taken", - n, strings.Join(members, ", "))) - continue - } - out = append(out, fmt.Sprintf("on the network %s with %s, which may reach it by name and will not once it moves to the module's own network"+ - " (`settings set %s --node ` with {%q: {: [%q]}} keeps it)", - n, strings.Join(members, ", "), h.Module, catalogue.NetworksSetting, n)) - } - for _, n := range keeps { - if _, found := f.networks[n]; !found { - out = append(out, fmt.Sprintf("keeps the network %s by this machine's setting, which the found container is not on", n)) - } - } - if len(f.ports) > 0 || len(f.declaredPorts) > 0 { - out = append(out, fmt.Sprintf("publishes %s; the module declares %s", - orNone(strings.Join(f.ports, " ")), orNone(strings.Join(f.declaredPorts, " ")))) - // How far each published port reaches now, as the machine reported it: the listener the - // runtime publishes for this container. The found firewall's and the guard's rules are - // not read; what they let through is said as what was reported reachable. - var reach []string - for _, r := range reported.Reachable { - if r.By == h.Target && r.Published { - reach = append(reach, fmt.Sprintf("%s:%d (%s, container port %d)", r.Address, r.Port, r.Protocol, r.ContainerPort)) - } - } - switch { - case len(reach) > 0: - line := "reachable now at " + strings.Join(reach, ", ") - if reported.Firewall != "" && reported.Firewall != "none" { - line += ", behind the found firewall (" + reported.Firewall + "), whose rules are not read" - } - out = append(out, line) - case len(f.ports) > 0 && len(reported.Reachable) > 0: - out = append(out, "not reported reachable on the machine") - } - } - if len(f.mounts) > 0 || len(f.declaredVolumes) > 0 { - out = append(out, fmt.Sprintf("mounts %s; the module declares %s", - orNone(strings.Join(f.mounts, " ")), orNone(strings.Join(f.declaredVolumes, " ")))) - } - if f.differs { - out = append(out, "the declared content differs from the file found (- lost, + new):") - for _, d := range f.difference { - out = append(out, " "+d) - } - } - return out -} - -// day is a timestamp as a person reads it in a preview: its date. -func day(stamp string) string { - if len(stamp) >= 10 { - return stamp[:10] - } - return stamp -} - -// reportFreshFor is how old a node's account of itself may be for the flip to act on it. A -// variable so a test can age a report without waiting. -var reportFreshFor = 15 * time.Minute - -// converge previews, and with yes makes, the flip of an adopted node to converged: every module it -// runs is taken, the filter module is assigned to load the mesh's derived filter in place of the -// guard, and the found firewall is retired — disabled, never flushed — by the host. -// -// Refused while an assigned module still holds a found container: each service is taken on its -// own, when its data has moved, never by the flip. And refused on a preview that would be stale: -// what is reachable is the node's last account, so that account must be of what it was last sent. -// -// **The flip acts on the preview the operator saw** and on nothing else. The preview ends with a -// short digest of what it said — every reachable thing and its fate, the modules the flip takes and -// the filter — and yes must name that digest: if anything the preview would say has changed since, -// the flip is refused rather than done on a preview nobody read. And it is refused on an account -// older than reportFreshFor: what was reachable then is not evidence of what is reachable now. -func converge(ctx context.Context, open *stores, node string, yes bool, digest string, - filter string) (string, error) { - inv := open.inventory - if filter == "" { - filter = DefaultFilter - } - record, err := inv.NodeByName(ctx, node) - if err != nil { - return "", err - } - if !record.Adopted { - return "", fmt.Errorf("%s is converged already; there is nothing to flip", node) - } - reports, err := lastReports(ctx, inv) - if err != nil { - return "", err - } - current, unknown := false, false - for _, r := range reports { - if r.Node == node { - current, unknown = r.Current, r.CurrentUnknown - } - } - if unknown { - return "", fmt.Errorf("cannot tell whether %s runs what is wanted: the bus is not reachable", node) - } - reported, err := inv.AdoptionOf(ctx, node) - if err != nil { - return "", err - } - if !current || reported.At.IsZero() { - return "", fmt.Errorf("%s does not run everything that is wanted of it yet, or has not reported, so what it "+ - "says is reachable may not be the machine as it is: converge once its haves match its wants and it has "+ - "reported", node) - } - - assignedWhenPreviewed, err := inv.Assigned(ctx, node) - if err != nil { - return "", err - } - plan, _, err := resolveNode(ctx, open, node) - if err != nil { - return "", err - } - runs := map[string]bool{} - for _, m := range plan.Modules { - runs[m.Module] = true - } - var holding []string - for _, h := range reported.Held { - if h.Kind == "container" && runs[h.Module] { - holding = append(holding, fmt.Sprintf(" %s holds the found container %s — take %s %s "+ - "once its data has moved", h.Module, h.Target, node, h.Module)) - } - } - if len(holding) > 0 { - sort.Strings(holding) - return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+ - "never by the flip:\n%s", node, strings.Join(holding, "\n")) - } - // And refused while a peer of the tunnel this hub took over has not enrolled (novox/hq ADR - // 0105): the flip loads the derived filter and retires the found firewall, and a machine the - // mesh has no record of is not one the filter admits — it would go dark. - if _, hubName, adopted, err := inv.AdoptedTunnel(ctx); err != nil { - return "", err - } else if adopted && hubName == node { - carried, err := inv.CarriedPeers(ctx) - if err != nil { - return "", err - } - var waiting []string - for _, c := range carried { - if c.EnrolledAs == "" { - waiting = append(waiting, fmt.Sprintf(" %s at %s", overlay.CarriedName(c.PublicKey), c.Address)) - } - } - if len(waiting) > 0 { - return "", fmt.Errorf("%s carries peers of the tunnel it took over that have not enrolled, and "+ - "converging would cut them off — enrol each first (`overlay show` says which are enrolled):\n%s", - node, strings.Join(waiting, "\n")) - } - } - - shelf, err := inv.Catalogue(ctx) - if err != nil { - return "", err - } - filterModule, known := shelf[filter] - if !known { - return "", fmt.Errorf("%w: %s — converging assigns it to load the mesh's filter; "+ - "name another with --filter", inventory.ErrNoSuchModule, filter) - } - if filterModule.Filtering == nil { - return "", fmt.Errorf("%s loads no filter of the mesh's; name a module that does with --filter", - filter) - } - - // What the filter is derived from: the rendering of this machine as a question reads it, with the private network. - with, err := renderingOf(ctx, open, node) - if err != nil { - return "", err - } - rules, err := plan.Rules(with) - if err != nil { - return "", err - } - taken, err := inv.Taken(ctx, node) - if err != nil { - return "", err - } - derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh, - outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks, local: with.LocalNetworks} - filtering, err := inv.FilteringOf(ctx, node) - if err != nil { - return "", err - } - preview, saw := previewOf(node, reported, filtering, derived, plan, taken, filter, runs[filter]) - preview += "\n\n preview " + saw - if !yes { - return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+ - "it.", node, saw), nil - } - // An account naming nothing reachable is not an account of a machine: every machine answers - // on ssh, and the host's collectors failing — `ss` refusing, or the container runtime not - // answering, which drops every published port at once — leaves exactly this. Flipping on it - // would close ports the preview never named. - if yes && countReachable(reported) == 0 { - return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+ - "up ever is: its account looks partial — whatever reads what is listening, or what "+ - "the container runtime publishes, did not answer. Fix that on the machine, wait for its next report, "+ - "then preview again", node) - } - if age := time.Since(reported.At); age > reportFreshFor { - return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+ - "only on an account newer than %s: wait for its next report, then preview again", node, - age.Round(time.Second), reportFreshFor) - } - if digest == "" { - return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+ - "`converge %s --yes %s`, once you have read it", node, node, saw) - } - if digest != saw { - return preview, fmt.Errorf("what converging %s would do has changed since preview %s "+ - "(it is now %s): read the preview above, and run `converge %s --yes %s` if it is "+ - "what you want", node, digest, saw, node, saw) - } - - // The flip. The filter first, and only kept if the node still resolves with it: a node that - // cannot be worked out would be sent nothing, and would sit with its guard and no filter. - assigned, err := inv.Assigned(ctx, node) - if err != nil { - return "", err - } - // And nothing assigned since the preview was composed: the flip takes every module the node - // runs, and one assigned in between would be taken without ever having been previewed. - if !slices.Equal(assigned, assignedWhenPreviewed) { - return preview, fmt.Errorf("what %s runs changed while this was converging (it is now %s): "+ - "the flip takes every module on the node, so read the preview again", node, - strings.Join(assigned, ", ")) - } - if !slices.Contains(assigned, filter) { - if _, err := inv.Assign(ctx, node, filter); err != nil { - return "", err - } - if _, _, err := resolveNode(ctx, open, node); err != nil { - _ = inv.Unassign(ctx, node, filter) - return "", fmt.Errorf("%s cannot run %s, so it was not converged: %w", node, filter, err) - } - } - took, err := inv.Converge(ctx, node) - if err != nil { - return "", err - } - said := preview + fmt.Sprintf("\n\n%s is converged", node) - if len(took) > 0 { - said += "; took " + strings.Join(took, ", ") - } - if err := sendNodes(ctx, open, []string{node}); err != nil { - return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err - } - return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil -} - -// previewOf is what converging a node will change, before it changes it, and a short digest of -// what it said: every reachable thing and its fate, the modules the flip takes and the filter. The -// digest is what the flip is asked to act on, so it changes whenever any of those would. -func previewOf(node string, reported inventory.Adoption, filtering inventory.Filtering, derived derivedFilter, - plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) { - var said []string - var b strings.Builder - fmt.Fprintf(&b, "converging %s\n", node) - fmt.Fprintf(&b, "\n reachable on the machine now, as it reported at %s:\n", - reported.At.Local().Format(time.DateTime)) - for _, r := range reported.Reachable { - if loopback(r.Address) { - continue - } - what := fmt.Sprintf("%s/%d", r.Protocol, r.Port) - if r.By != "" { - what += " " + r.By - } - if r.Published { - what += fmt.Sprintf(" (published, container port %d)", r.ContainerPort) - } - fate := derived.fate(r) - fmt.Fprintf(&b, " %-44s %s\n", what, fate) - said = append(said, fmt.Sprintf("reach %s %s %s", r.Address, what, fate)) - } - if countReachable(reported) == 0 { - // Said as what it is: no machine that is up is reachable on nothing, so this is an - // account that did not come back, not a machine with nothing on it. - b.WriteString(" nothing reported — this account looks partial, and the flip is " + - "refused on it\n") - said = append(said, "reach nothing reported") - } - // What the machine routes for others is not a listener and not a published port, so nothing - // above can show it; the derived filter's forward chain drops it all the same. - b.WriteString(" not previewed: traffic the machine routes that is not a published port " + - "(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " + - "declares it\n") - // Which links the filter constrains, said rather than left to the sentence above (novox/hq ADR - // 0140). Everything arriving anywhere else is this machine's own guest and keeps working — which - // is what a reader most wants to know, because the previous shape of this filter cut a machine's - // guests off at the flip without saying so, and that is how this was found. - if len(derived.outwardLinks) > 0 { - b.WriteString(fmt.Sprintf(" it filters what arrives on: %s, and on the private network "+ - "— everything its own guests send keeps working\n", - strings.Join(derived.outwardLinks, ", "))) - } else { - b.WriteString(" it has reported no link facing outside, so no filter can be composed " + - "for it — the flip is refused until it reports one\n") - } - - isTaken := map[string]bool{} - for _, m := range taken { - isTaken[m] = true - } - var takes []string - for _, m := range plan.Modules { - if !isTaken[m.Module] { - takes = append(takes, m.Module) - } - } - if !filterAssigned && !isTaken[filter] { - takes = append(takes, filter) - } - sort.Strings(takes) - b.WriteString("\n the flip takes:\n") - if len(takes) == 0 { - b.WriteString(" nothing — every module is taken already\n") - } - for _, m := range takes { - fmt.Fprintf(&b, " %s\n", m) - said = append(said, "take "+m) - // Every kind it holds — a directory, a service, an archive, a process, a user as well as a - // file (novox/hq ADR 0103) — each said, and each part of what the flip is asked to act on. - for _, h := range reported.Held { - if h.Module != m { - continue - } - fmt.Fprintf(&b, " replacing the found %s", heldLine(h)) - if h.Kept != "" { - fmt.Fprintf(&b, ", original kept at %s", h.Kept) - } - b.WriteString("\n") - said = append(said, "replace "+m+" "+heldLine(h)+" "+h.Kept) - } - } - if !filterAssigned { - fmt.Fprintf(&b, "\n and assigns %s, which loads the mesh's filter in place of its guard\n", filter) - } - fw := reported.Firewall - if fw == "" || fw == "none" { - b.WriteString(" no firewall was found on the machine; the mesh's filter is its first\n") - } else { - fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw) - } - said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw)) - // What filters the machine now, and the fate of each (novox/hq ADR 0168): the found firewall - // retired, the runtime's own and bans left, and what the mesh did not write left and named — - // so the reader knows before the flip that the machine will not be filtered by the mesh alone. - if len(filtering.Filters) > 0 { - b.WriteString("\n what filters the machine now, and what the flip does to each:\n") - for _, x := range filtering.Filters { - fate := "left: " + x.Owner + "'s" - switch x.Owner { - case inventory.FilterMesh: - fate = "the mesh's guard; replaced by its filter" - case inventory.FilterFoundFirewall: - fate = "the found firewall's; retired with it" - case inventory.FilterRuntime: - fate = "the container runtime's own; left" - case inventory.FilterBan: - fate = "a ban list; left" - case inventory.FilterOther: - fate = "NOT THE MESH'S; left in force — the machine is not filtered by the mesh alone until you remove it" - } - fmt.Fprintf(&b, " %-50s %s\n", x.Where, fate) - fmt.Fprintf(&b, " %-50s %s\n", "", x.Refuses) - said = append(said, "filter "+x.Owner+" "+x.Where) - } - } - // Sorted: the same account, reported in another order, is the same preview. - sort.Strings(said) - sum := sha256.Sum256([]byte(strings.Join(said, "\n"))) - return strings.TrimRight(b.String(), "\n"), hex.EncodeToString(sum[:])[:12] -} - -// derivedFilter is what the filter the flip loads is rendered from, as AsNftables renders it. -type derivedFilter struct { - rules []catalogue.Rule - foundation []int - // mesh is every address on the private network; outward says the machine faces outside. - mesh []string - outward bool - // outwardLinks is the links this machine reported as facing outside it (novox/hq ADR 0140). - // The filter constrains what arrives on them; everything arriving elsewhere is this machine's - // own guest and is not filtered. - outwardLinks []string - // local is the machine's local networks, which a port from `local-network` stays open to (novox/hq - // ADR 0340). - local []string -} - -// closesOutside is what a narrowing from everywhere to the private network is called: it closes. -const closesOutside = "WILL CLOSE to everything outside the private network" - -// fate is what the derived filter does to one reachable thing: which module declares it and from -// where, or that it will close — wholly, or to everything outside the private network. Rendered -// exactly as AsNftables admits it, ssh included. -func (d derivedFilter) fate(r inventory.Reach) string { - // Bound to an address on the private network, it was never reachable from outside it, so - // admitting it from the mesh narrows nothing. - onMesh := slices.Contains(d.mesh, strings.Trim(r.Address, "[]")) - if r.Protocol == "tcp" && r.Port == catalogue.SSHPort { - // From everywhere only when the machine faces outward or the mesh has no addresses to - // narrow it to; otherwise from the private network only. - if d.outward || len(d.mesh) == 0 || onMesh { - return "stays open — ssh is never closed" - } - return closesOutside + " — ssh stays open from the mesh, never closed there" - } - for _, port := range d.foundation { - if r.Protocol == "tcp" && r.Port == port { - return "stays open — the mesh's own, from anywhere" - } - } - // This machine's own guests ask it for an address and for names, and those two arrive here - // (novox/hq ADR 0140). Admitted by the link they arrive on, so a listener bound anywhere but an - // outward link keeps answering them. - if (r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53) { - return "stays open — this machine's own guests asking it for an address and for names" - } - for _, rule := range d.rules { - if rule.Port != r.Port || rule.Protocol != r.Protocol { - continue - } - by := strings.Join(rule.Because, ", ") - switch rule.From { - case catalogue.FromMachine: - return fmt.Sprintf("WILL CLOSE to the network — declared by %s for this machine only", by) - case catalogue.FromMesh: - if len(d.mesh) == 0 { - return fmt.Sprintf("WILL CLOSE — declared by %s from the mesh, and this node "+ - "knows no mesh addresses", by) - } - if !onMesh { - return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by) - } - case catalogue.FromLocalNetwork: - // The private network and this machine's local networks (novox/hq ADR 0340): it closes to - // everything else outside, and says which local networks stay. - if !onMesh { - if len(d.local) == 0 { - return fmt.Sprintf("%s — declared by %s from the local network, and this machine "+ - "reported none", closesOutside, by) - } - return fmt.Sprintf("WILL CLOSE to everything outside the private network and %s — declared "+ - "by %s from the local network", strings.Join(d.local, ", "), by) - } - } - return fmt.Sprintf("declared by %s (from %s)", by, rule.From) - } - return "WILL CLOSE — no module assigned here declares it" -} - -// countReachable is how much of a node's account of itself names something off the machine. -// Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it, -// so a report of loopback alone says nothing about what the filter would close. -func countReachable(reported inventory.Adoption) int { - n := 0 - for _, r := range reported.Reachable { - if !loopback(r.Address) { - n++ - } - } - return n -} - -// heldLine is one thing a node holds as found, as take and the converge preview both say it. -func heldLine(h inventory.Held) string { - return fmt.Sprintf("%s %s (%s)", h.Kind, h.Target, h.ID) -} - -// loopback is an address nothing off the machine reaches. -func loopback(address string) bool { - a := strings.Trim(address, "[]") - return strings.HasPrefix(a, "127.") || a == "::1" || a == "localhost" -} - -// adopt returns a converged node to adopted: the mesh's filter is unloaded, the guard restored, -// the found firewall enabled again and the openings converged through it once more. What was -// taken stays taken. -func adopt(ctx context.Context, open *stores, node string) (string, error) { - inv := open.inventory - record, err := inv.NodeByName(ctx, node) - if err != nil { - return "", err - } - if record.Adopted { - return "", fmt.Errorf("%s is adopted already", node) - } - if err := inv.SetAdopted(ctx, node, true); err != nil { - return "", err - } - said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node) - if err := sendNodes(ctx, open, []string{node}); err != nil { - return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err - } - return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil -} - -// takeCommand, convergeCommand and adoptCommand are the command line's adapters to the acts above. -func takeCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("take", flag.ContinueOnError) - yes := set.Bool("yes", false, "cut over as previewed, naming the digest the preview printed after it; "+ - "without it the comparison is printed and nothing is taken") - downgrade := set.Bool("downgrade", false, "take it although the module's image is older than the one running") - var replace, mint stringList - set.Var(&replace, "replace", "a found file's path whose content the module may replace although it differs (repeatable; * for every one)") - set.Var(&mint, "mint", "a secret the service shall take the mesh's minted value for, although it already has its own (repeatable)") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) < 2 || len(positionals) > 3 || (len(positionals) == 3 && !*yes) { - return errors.New("take [--yes ] [--downgrade] [--replace ]... [--mint ]...") - } - opts := takeOptions{Yes: *yes, Downgrade: *downgrade, Replace: map[string]bool{}, Mint: map[string]bool{}} - if len(positionals) == 3 { - opts.Digest = positionals[2] - } - for _, r := range replace { - opts.Replace[r] = true - } - for _, m := range mint { - opts.Mint[m] = true - } - return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, positionals[0], positionals[1], opts) }) -} - -// stringList is a repeatable flag. -type stringList []string - -func (l *stringList) String() string { return strings.Join(*l, ",") } -func (l *stringList) Set(v string) error { *l = append(*l, v); return nil } - -func convergeCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("converge", flag.ContinueOnError) - yes := set.String("yes", "", "do it, naming the digest the preview printed; without it, only "+ - "the preview") - filter := set.String("filter", DefaultFilter, "the module that loads the mesh's filter") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) != 1 { - return errors.New("converge [--yes ] [--filter nftables]") - } - return runAct(ctx, func(open *stores) (string, error) { - return converge(ctx, open, positionals[0], *yes != "", *yes, *filter) - }) -} - -func adoptCommand(ctx context.Context, args []string) error { - if len(args) != 1 { - return errors.New("adopt ") - } - return runAct(ctx, func(open *stores) (string, error) { return adopt(ctx, open, args[0]) }) -} - -func runAct(ctx context.Context, act func(*stores) (string, error)) error { - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - said, err := act(open) - if said != "" { - fmt.Println(said) - } - if err != nil && said != "" { - fmt.Println() - } - return err -} diff --git a/cmd/mesh-controller/advisories_counted_test.go b/cmd/mesh-controller/advisories_counted_test.go deleted file mode 100644 index 0a999eb8..00000000 --- a/cmd/mesh-controller/advisories_counted_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller -// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered -// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last -// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and -// went looking for a missing grant. The condition counts what the bus said, with when it last said it, -// never how often the controller looked. -func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) { - refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC) - now := refused.Add(10 * time.Second) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return now }}) - t.Cleanup(func() { k.Close(t.Context()) }) - - // The words the live condition bus.controller.refused carried on 2026-10-10. - said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " + - `Publish to "mesh.seat.mesh-delivery.tool.times"` - advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said, - First: refused, Last: refused, Count: 1} - look := func() conditions.Condition { - t.Helper() - f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}} - if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil { - t.Fatal(err) - } - c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused") - if err != nil || !found { - t.Fatalf("bus.controller.refused: found %v, %v", found, err) - } - return c - } - - var c conditions.Condition - for range 15 { - c = look() - now = now.Add(30 * time.Second) - } - if c.Observations != 1 || len(c.Evidence) != 1 { - t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v", - c.Observations, len(c.Evidence), c.Evidence) - } - if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) { - t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused) - } - - // The bus refuses it three times more between two looks: the condition counts four refusals, says - // the newest, and adds one line of evidence for the look that saw them. - again := now.Add(-5 * time.Second) - advisory.Last, advisory.Count = again, 4 - c = look() - if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) { - t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence) - } - if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") { - t.Fatalf("summary %q", c.Summary) - } - now = now.Add(30 * time.Second) - if c = look(); c.Observations != 4 || len(c.Evidence) != 2 { - t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence) - } -} - -// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only -// what reads a record): a machine silent for three looks was observed three times, and says so. -func TestAConditionOfThePresentCountsItsLooks(t *testing.T) { - now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return now }}) - t.Cleanup(func() { k.Close(t.Context()) }) - var c conditions.Condition - for range 3 { - var err error - if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", - Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from", - Source: "S1"}); err != nil { - t.Fatal(err) - } - now = now.Add(30 * time.Second) - } - if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) { - t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved) - } -} - -// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from -// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory -// without a token names the same key; read beside it, each look added an observation and a line of -// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The -// dead-letter row alone says that key, and counts the messages given up on, never the looks. -func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) { - gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC) - now := gaveUp.Add(20 * time.Second) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return now }}) - t.Cleanup(func() { k.Close(t.Context()) }) - - const key = "bus.EVENTS.media_sonarr.max-deliveries" - held := map[string]int{"EVENTS.media_sonarr": 1} - newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp} - advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS", - Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it", - First: gaveUp, Last: gaveUp, Count: 1} - look := func() conditions.Condition { - t.Helper() - f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}, - deadLetters: held, deadLettersNewest: newest} - if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil { - t.Fatal(err) - } - c, found, err := conditions.ReadOne(t.Context(), store, key) - if err != nil || !found { - t.Fatalf("%s: found %v, %v", key, found, err) - } - if !strings.Contains(c.Summary, "dead-letters verb") { - t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary) - } - return c - } - - var c conditions.Condition - for range 5 { - c = look() - now = now.Add(30 * time.Second) - } - if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) { - t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+ - "with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence) - } - - // The consumer gives up on a second message between two looks: two given up on, one more line. - again := now.Add(-10 * time.Second) - held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again - c = look() - if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) { - t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations, - c.LastObserved, c.Evidence) - } - now = now.Add(30 * time.Second) - if c = look(); c.Observations != 2 || len(c.Evidence) != 2 { - t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence) - } -} - -// The advisory still says the max-deliveries it alone knows: a message given up on that could not be -// kept, under a key of its own (issue 330), which issue 440's change leaves as it was. -func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) { - at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC) - f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, - ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept, - Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}} - said := watchAdvisories(f) - if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" { - t.Fatalf("%+v", said) - } -} - -// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS -// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing. -func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) { - at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC) - f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, - ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", - Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}} - if said := watchAdvisories(f); len(said) != 0 { - t.Fatalf("said %+v", said) - } -} diff --git a/cmd/mesh-controller/agent_account.go b/cmd/mesh-controller/agent_account.go deleted file mode 100644 index 90c1a7e3..00000000 --- a/cmd/mesh-controller/agent_account.go +++ /dev/null @@ -1,253 +0,0 @@ -package main - -// The account agents run as (novox/hq ADR 0266). -// -// On the control node every agent session ran as the operator's account, which may become root without a -// password — so any agent there could become root without a person, and ADR 0259 §8 (an answer from the -// operator's phone authorises an act) rests on that being false where the router and its channels run. The -// decision: a node may name an account its agents run as, of their own and without sudo; the operator's -// account keeps its sudo. -// -// - **Named at the controller's terminal only** (`node agent-account`): not a verb, not a setting, so no -// agent can name itself another account. Empty is a real state: agents run as the operator there. -// - **Composed** as `${machine:agent-account}`, `${machine:agent-home}` and `${machine:agent-root}` for the -// agent's module, which declares the account with `root: never`, and as MESH_AGENT_ACCOUNT and -// MESH_AGENT_HOME for its tools (catalogue/machine_into_files.go, runtime.go). -// - **Judged by the machine itself.** The node-engine reads, on every look, whether an account declared -// `root: never` can become root without a person — uid 0, a group that grants root, a sudo rule, a -// secret of the mesh it may read — and says it as the declaring module's account verdict, marked -// Root "never". agentConfined reads that verdict; the self-check (probe DA) raises -// `agent-can-become-root` while it does not hold, and `node show` says it. -// -// A verdict not given is never a pass: an engine older than the judging, an account not yet declared, a -// statement that says nothing of it — each is "not judged", and fails. - -import ( - "context" - "fmt" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// kindAgentCanBecomeRoot is the condition raised while a machine's agent account can become root without -// a person, or is not judged (ADR 0266). -const kindAgentCanBecomeRoot = "agent-can-become-root" - -// agentAccountProbe is the self-check's probe of it. -const agentAccountProbe = "DA" - -// agentConfined says whether the agents of a machine that names an agent account are confined: the -// machine's newest statement holds a healthy account verdict, judged for root, on that account. named is -// false for a machine that names none — agents run as the operator account there, which this does not -// judge. why is said either way, in the mesh's words; err is a store that could not be read. -// -// The one judgement: `node show`, the self-check, and ADR 0259's router honouring a verified sender read -// it here. -// now is the judging clock, threaded so a caller judging several things at one instant judges them all at it. -func agentConfined(ctx context.Context, inv *inventory.Inventory, node string, now time.Time) (named, confined bool, why string, err error) { - n, err := inv.NodeByName(ctx, node) - if err != nil { - return false, false, "", err - } - if n.AgentAccount == "" { - return false, false, fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)", - node, orNoneKnown(n.Account)), nil - } - h, had, err := inv.HealthOf(ctx, node) - if err != nil { - return true, false, "", err - } - confined, why = judgedConfined(n.AgentAccount, h, had, now) - return true, confined, why, nil -} - -// verdictFreshFor is how old the statement holding the verdict may be, by this controller's clock. A -// node-engine states its health on every change and at least every five minutes (mesh-host's sayAnyway), so -// three statements missed is a node-engine stopped, or a machine away. **A stale verdict is not a pass**: an -// agent that stopped the node-engine must not leave "cannot become root" standing from before. -const verdictFreshFor = 15 * time.Minute - -// judgedConfined is the judgement over one statement, without the store, at now. -func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Time) (bool, string) { - if !had { - return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+ - "nothing of what it runs", agent) - } - if age := now.Sub(h.HeardAt); age > verdictFreshFor { - return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement was heard at "+ - "%s, more than %d minutes ago, and a verdict that old is not a verdict on now", agent, - h.HeardAt.Local().Format("2006-01-02 15:04"), int(verdictFreshFor.Minutes())) - } - if h.Contract < link.RootContract { - return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+ - "the judging of an account's root (its statement's contract is %d, the judging is %d)", - agent, h.Contract, link.RootContract) - } - var verdicts []inventory.ResourceHealth - for _, r := range h.Resources { - if r.Kind == link.KindAccount && r.Target == agent && r.Root == link.RootNever { - verdicts = append(verdicts, r) - } - } - if len(verdicts) == 0 { - return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+ - "verdict on it — no module there declares it never to become root, or the declaration naming it "+ - "has not been applied", agent) - } - sort.Slice(verdicts, func(i, j int) bool { - return verdicts[i].Module+verdicts[i].Resource < verdicts[j].Module+verdicts[j].Resource - }) - for _, v := range verdicts { - switch v.State { - case link.StateHealthy: - case link.StateUnhealthy: - // The engine's own words, which start with link.ReasonRoot when it found a way to root. - return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent, - orNoneKnown(v.Reason), v.Module, v.Resource) - default: - return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent, - orNoneKnown(v.Reason), v.Module, v.Resource, v.State) - } - } - return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent, - h.SaidAt.Local().Format("2006-01-02 15:04")) -} - -// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid -// search before that is itself the urgent condition: the longest a search is expected to take (link.RootSearchQuiet, -// the engine's own value; novox/hq issue 361 — the search runs to completion, with no bound of its own), counted -// from when this controller first saw it waiting, never from the engine's start. Quiet raises nothing; it never -// makes the agent account confined, which only a healthy verdict from a complete, fresh search does. -const searchQuietFor = link.RootSearchQuiet - -// The kinds of an agent account's verdict, for the quiet a search earns. -const ( - verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown - verdictPending // waiting for the search, and otherwise healthy - verdictComplete // judged: healthy, or a way to root found -) - -// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it -// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when -// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at -// every restart of the engine. -func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int { - if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract { - return verdictOther - } - pending, any := false, false - for _, r := range h.Resources { - if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever { - continue - } - any = true - switch { - case r.State == link.StateHealthy: - case r.State == link.StateUnhealthy: - return verdictComplete - case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending): - pending = true - default: - return verdictOther - } - } - switch { - case !any: - return verdictOther - case pending: - return verdictPending - } - return verdictComplete -} - -// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid -// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a -// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began. -func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth, - had bool, now time.Time) (bool, error) { - switch rootVerdictKind(agent, h, had, now) { - case verdictComplete: - return false, inv.RootSearchJudged(ctx, node) - case verdictPending: - since, err := inv.RootSearchPending(ctx, node, now) - if err != nil { - return false, err - } - return now.Sub(since) <= searchQuietFor, nil - } - return false, nil -} - -// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's -// newest statement, unable to become root without a person (ADR 0266). -func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - inv := d.open.inventory - nodes, err := inv.Nodes(ctx) - if err != nil { - return nil, err - } - var out []conditions.Observation - for _, n := range nodes { - if n.AgentAccount == "" { - continue - } - h, had, err := inv.HealthOf(ctx, n.Name) - if err != nil { - return nil, err - } - now := time.Now() - quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now) - if err != nil { - return nil, err - } - confined, why := judgedConfined(n.AgentAccount, h, had, now) - if confined { - continue - } - // Not judged yet only because the node-engine's search runs and no complete, fresh one judges: not the - // urgent condition after every restart. The agent is still not confined — ADR 0259's router reads - // agentConfined, not this — and `node show` still says not judged. Loud again once the search fails, - // waits past searchQuietFor, or the statement goes stale. - if quiet { - continue - } - out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root", - Machine: n.Name, Severity: conditions.Urgent, - Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and "+ - "no answer from a channel authorises an act there (ADR 0259 §8)", n.Name, why), - Said: why}) - } - return sortedFound(out), nil -} - -// agentAccountLines is what `node show` says of the account agents run as. -func agentAccountLines(ctx context.Context, inv *inventory.Inventory, n inventory.Node) []string { - if n.AgentAccount == "" { - return []string{fmt.Sprintf(" agents run as the operator account (%s); no agent account is named", - orNoneKnown(n.Account))} - } - _, confined, why, err := agentConfined(ctx, inv, n.Name, time.Now()) - if err != nil { - return []string{fmt.Sprintf(" agents run as %s (home %s); whether it can become root could NOT be read: %v", - n.AgentAccount, n.AgentHome(), err)} - } - verdict := "CAN become root, or is not judged: " + why - if confined { - verdict = why - } - return []string{fmt.Sprintf(" agents run as %s (home %s)", n.AgentAccount, n.AgentHome()), - " " + verdict} -} - -// orNoneKnown is a value, or that none is known. -func orNoneKnown(s string) string { - if strings.TrimSpace(s) == "" { - return "none known" - } - return s -} diff --git a/cmd/mesh-controller/agent_account_integration_test.go b/cmd/mesh-controller/agent_account_integration_test.go deleted file mode 100644 index ece0c1f5..00000000 --- a/cmd/mesh-controller/agent_account_integration_test.go +++ /dev/null @@ -1,161 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - snapshot "git.novox.be/novox/mesh-sdk/go/facts" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-host/rootsearch" -) - -func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.NodeByName(ctx, n); err != nil { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAccount(ctx, n, "ops", ""); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { - t.Fatal(err) - } - d := &doctor{open: open} - found, err := probeAgentAccounts(ctx, d) - if err != nil { - t.Fatal(err) - } - found = onlyMachine(found, "anchor") - if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent || - !strings.Contains(found[0].Said, "not judged") { - t.Fatalf("a named agent account with no verdict: %+v", found) - } - w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"}) - if w.Headline == "" || w.Needs == "" || w.Resolved == "" { - t.Errorf("the condition has no plain words: %+v", w) - } - - healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", - Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"} - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, - SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil { - t.Fatal(err) - } - if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 { - t.Fatalf("a judged agent account still fails: %+v %v", found, err) - } - if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined { - t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err) - } - if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named || - !strings.Contains(why, "operator account") { - t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err) - } -} - -func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) { - open, _ := aMeshWithSecrets(t) - ctx := t.Context() - if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil { - t.Fatal(err) - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - body, _ := f.Encode() - if strings.Contains(string(body), "warden") { - t.Error("the agent account's name is in the snapshot") - } - m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor")) - if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount { - t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account) - } -} - -func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { - if searchQuietFor != rootsearch.Quiet { - t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet) - } - // A daily search that finishes within the quiet never leaves the account unjudged between two of them. - if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet { - t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)", - rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet) - } - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.NodeByName(ctx, "anchor"); err != nil { - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { - t.Fatal(err) - } - d := &doctor{open: open} - say := func(state, reason string) []conditions.Observation { - t.Helper() - // The engine's since is always now: it was just restarted. - v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", - Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever, - Account: "agent", Since: time.Now()} - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, - SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil { - t.Fatal(err) - } - found, err := probeAgentAccounts(ctx, d) - if err != nil { - t.Fatal(err) - } - return onlyMachine(found, "anchor") - } - running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet" - healthy := func() { - t.Helper() - if found := say(link.StateHealthy, ""); len(found) != 0 { - t.Fatalf("a healthy verdict raised: %+v", found) - } - } - if found := say(link.StateUnknown, running); len(found) != 0 { - t.Fatalf("a search first seen now was raised: %+v", found) - } - if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") { - t.Fatalf("an account whose search runs was read as confined: %q", why) - } - // The engine restarted again and again, each statement's own since fresh: the controller's clock runs on. - if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { - t.Fatal(err) - } - healthy() // a complete verdict forgets when the waiting began … - if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { - t.Fatal(err) // … and this restart loop began past the bound - } - if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent { - t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found) - } - healthy() - incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " + - "19:23: timeout); it is tried again later" - if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent { - t.Fatalf("a search that did not finish was not urgent: %+v", found) - } - if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 || - found[0].Severity != conditions.Urgent { - t.Fatalf("a way to root found while the search runs was not urgent: %+v", found) - } -} diff --git a/cmd/mesh-controller/agent_account_test.go b/cmd/mesh-controller/agent_account_test.go deleted file mode 100644 index c8db9085..00000000 --- a/cmd/mesh-controller/agent_account_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The agent account's judgement (novox/hq ADR 0266): confined only on a healthy account verdict judged for -// root, on the very account; every verdict not given — no statement, an older engine, no verdict on it, a -// verdict of unknown — is "not judged" and fails, never a pass. -func TestAnAgentAccountIsConfinedOnlyOnAHealthyVerdictJudgedForRoot(t *testing.T) { - at := time.Date(2026, 10, 8, 19, 21, 0, 0, time.UTC) - verdict := func(target, root, state, reason string) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", - Kind: link.KindAccount, Target: target, State: state, Reason: reason, Root: root, Account: target} - } - statement := func(contract int, rs ...inventory.ResourceHealth) inventory.NodeHealth { - return inventory.NodeHealth{Node: "anchor", Contract: contract, SaidAt: at, HeardAt: at, Resources: rs} - } - for _, c := range []struct { - name string - h inventory.NodeHealth - had bool - confined bool - says string - }{ - {"judged and unable", statement(link.RootContract, verdict("agent", link.RootNever, link.StateHealthy, "")), - true, true, "cannot become root without a person"}, - {"judged and able", statement(link.RootContract, verdict("agent", link.RootNever, link.StateUnhealthy, - link.ReasonRoot+": in the group docker, which grants root")), true, false, "in the group docker"}, - {"a verdict of unknown", statement(link.RootContract, verdict("agent", link.RootNever, link.StateUnknown, - "sudo could not be read")), true, false, "not judged"}, - {"no statement", inventory.NodeHealth{}, false, false, "not judged"}, - {"an older engine", statement(link.ReadinessContract, verdict("agent", "", link.StateHealthy, "")), - true, false, "older than the judging"}, - {"a verdict on groups only", statement(link.RootContract, verdict("agent", "", link.StateHealthy, "")), - true, false, "no verdict on it"}, - {"a verdict on another account", statement(link.RootContract, verdict("ops", link.RootNever, link.StateHealthy, "")), - true, false, "no verdict on it"}, - } { - confined, why := judgedConfined("agent", c.h, c.had, at.Add(time.Minute)) - if confined != c.confined || !strings.Contains(why, c.says) { - t.Errorf("%s: confined %v, %q; want %v saying %q", c.name, confined, why, c.confined, c.says) - } - } - // A verdict heard longer ago than the bound is no verdict: an agent that stopped the node-engine must not - // leave "healthy" standing. - fresh := statement(link.RootContract, verdict("agent", link.RootNever, link.StateHealthy, "")) - if ok, _ := judgedConfined("agent", fresh, true, at.Add(verdictFreshFor)); !ok { - t.Error("a verdict exactly at the bound is still one") - } - if ok, why := judgedConfined("agent", fresh, true, at.Add(verdictFreshFor+time.Second)); ok || - !strings.Contains(why, "not judged") { - t.Errorf("a stale healthy verdict passed: %q", why) - } -} - -// DA raises an urgent condition, with plain words, on a machine whose agent account is not judged unable to -// become root; a machine that names none is not its to judge. - -func TestTheAgentRootWordsArePlain(t *testing.T) { - w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"}) - if why, ok := conditions.PlainWords(w, "anchor"); !ok { - t.Fatalf("not plain: %s: %+v", why, w) - } -} - -func onlyMachine(obs []conditions.Observation, machine string) []conditions.Observation { - var out []conditions.Observation - for _, o := range obs { - if o.Machine == machine { - out = append(out, o) - } - } - return out -} - -// The snapshot a merge check composes from carries the agent account as a pseudonym (novox/hq ADR 0266), -// so a change is judged against machines that name one, and the name never leaves. - -// No verb runs a `node` command that sets something: through the generic `command` verb, `node account`, -// `node agent-account` and every other `node` subcommand but list and show are refused, naming the terminal. -func TestNoVerbSetsANodesAccounts(t *testing.T) { - for _, line := range []string{ - "node agent-account novox --clear", - "node agent-account novox ops", - "node account novox agent", - "node account novox", - "node add intruder", - "node public-domain novox --clear", - "node", - "node frobnicate", - } { - argv, err := argvFor("command", map[string]any{"command": line}) - if err == nil || !strings.Contains(err.Error(), "controller's terminal") || - !strings.Contains(err.Error(), "ADR 0266") { - t.Errorf("%q ran as %v (%v); want a refusal naming the terminal", line, argv, err) - } - } - for _, line := range []string{"node show novox", "node list --json", "status --json"} { - if _, err := argvFor("command", map[string]any{"command": line}); err != nil { - t.Errorf("%q, a read, was refused: %v", line, err) - } - } - if err := terminalOnly([]string{"node", "account", "a", "b"}); err == nil { - t.Error("the refusal is not only the command verb's") - } -} - -// Naming the agent account is the controller's terminal's alone: the `node` verb only shows. -func TestTheNodeVerbOnlyShows(t *testing.T) { - argv, err := argvFor("node", map[string]any{"node": "anchor"}) - if err != nil || strings.Join(argv, " ") != "node show anchor" { - t.Fatalf("the node verb runs %v (%v)", argv, err) - } - for _, v := range catalogue.ControllerVerbs { - if strings.Contains(v.Name, "agent") { - t.Errorf("a verb %q may name the agent account", v.Name) - } - } -} - -// Until a complete search for setuid programs judges — none since the node-engine's state was kept, or the last -// older than the engine lets one judge — the agent account is not judged, and the search runs to its end with no -// bound (novox/hq issue 361). DA does not raise that as urgent within searchQuietFor, counted from when this -// controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine -// restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not -// judged; a search that failed, or a way to root found, is urgent at once. diff --git a/cmd/mesh-controller/api.go b/cmd/mesh-controller/api.go deleted file mode 100644 index e89d2d3f..00000000 --- a/cmd/mesh-controller/api.go +++ /dev/null @@ -1,186 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "net/http" - "strings" - "time" -) - -// The command API: what the mesh can be asked to do, over a network. -// -// **An adapter and nothing else** (novox/hq ADR 0035). Every route here calls the same function -// the command line calls, so a refusal is the same refusal in the same words. Nothing is decided -// in this file — the moment it validates something the command line does not, the mesh has two -// answers to one question. -// -// **It refuses everything unless it was told how to know who is asking.** The board is published -// on a public name, and this is what stands behind it: an unauthenticated command surface reachable -// from the internet is authority over the mesh handed to whoever finds it. So there is no -// permissive default and no flag that removes the check — a mesh that has not been told how to -// authenticate serves nothing, loudly. -// -// The intended authenticator is an OAuth2 provider (ADR 0035), which is an ordinary module. Until -// one is configured this refuses, which is the correct behaviour rather than a placeholder: a -// surface that worked without authentication would be one somebody left running. -func apiCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("api", flag.ContinueOnError) - listen := set.String("listen", "127.0.0.1:8081", "where to serve it") - issuer := set.String("issuer", "", - "the OAuth2 issuer whose tokens this accepts; without it, nothing is served") - if _, err := parseAround(set, args); err != nil { - return err - } - if strings.TrimSpace(*issuer) == "" { - // Refused at start rather than per request, so it is discovered by whoever ran it rather - // than by whoever finds it. - return errors.New( - "--issuer is not set, and this serves commands rather than pages: it will not run " + - "without being told whose tokens to believe. An OAuth2 provider is an ordinary " + - "module (novox/hq ADR 0035)") - } - - server := &http.Server{ - Addr: *listen, - ReadHeaderTimeout: 10 * time.Second, - Handler: commands(mustAuthenticate(*issuer)), - } - fmt.Printf("the command API is on http://%s\n", *listen) - fmt.Printf(" it accepts tokens from %s and refuses everything else\n", *issuer) - fmt.Printf(" every route calls what the command line calls\n") - - go func() { - <-ctx.Done() - closing, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _ = server.Shutdown(closing) - }() - if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - return err - } - return nil -} - -// Authenticator says whether a request may act, and as whom. -// -// An interface so the check can be driven by a test without an identity provider, and so the one -// real implementation is the only thing that has to be right. -type Authenticator interface { - // Who returns the subject a request is acting as, or an error naming why it may not. - Who(r *http.Request) (string, error) -} - -// mustAuthenticate is the real one: a bearer token from the configured issuer. -// -// **Not yet verifying the signature**, and it says so rather than pretending. Verification needs -// the issuer's keys, which needs an identity provider to exist — so this refuses every request -// until that is built, which is the same answer as having no API at all and is honest about why. -func mustAuthenticate(issuer string) Authenticator { return notYet{issuer: issuer} } - -type notYet struct{ issuer string } - -func (n notYet) Who(*http.Request) (string, error) { - return "", fmt.Errorf( - "this mesh has no way to verify a token from %s yet: the identity provider is a module "+ - "and none is running. Use the command line, which authenticates through nothing "+ - "because it is already behind the machine's own login", n.issuer) -} - -// commands is the routing, separate so a test can drive it without a listener. -func commands(who Authenticator) http.Handler { - mux := http.NewServeMux() - - mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) { - return assign(ctx, open, in.Node, splitModules(in.Module)...) - })) - mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) { - return unassign(ctx, open, in.Node, splitModules(in.Module)...) - })) - // Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`. - mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) { - return take(ctx, open, in.Node, in.Module, takeOptions{Yes: in.Yes, Digest: in.Digest}) - })) - mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) { - return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter) - })) - mux.HandleFunc("POST /adopt", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) { - return adopt(ctx, open, in.Node) - })) - - // Anything else is said plainly, because a command surface answering 404 to a verb somebody - // expected is indistinguishable from one that is down. - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - refuse(w, http.StatusNotFound, fmt.Errorf( - "%s %s is not something this mesh can be asked; it accepts POST /assign, "+ - "POST /unassign, POST /take, POST /converge and POST /adopt", r.Method, r.URL.Path)) - }) - return mux -} - -type request struct { - Node string `json:"node"` - Module string `json:"module"` - // Yes, Digest and Filter are converge's and take's: do it rather than preview it, the digest - // of the preview it acts on, and (converge) which module loads the mesh's filter. - Yes bool `json:"yes,omitempty"` - Digest string `json:"digest,omitempty"` - Filter string `json:"filter,omitempty"` -} - -// acting is the shape every route shares: authenticate, read, act, answer. -func acting( - who Authenticator, - needsModule bool, - do func(context.Context, *stores, request) (string, error), -) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if _, err := who.Who(r); err != nil { - refuse(w, http.StatusUnauthorized, err) - return - } - var in request - if err := json.NewDecoder(r.Body).Decode(&in); err != nil { - refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err)) - return - } - if in.Node == "" || (needsModule && in.Module == "") { - if needsModule { - refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`)) - } else { - refuse(w, http.StatusBadRequest, errors.New(`"node" is needed`)) - } - return - } - - open, err := openStores(r.Context()) - if err != nil { - refuse(w, http.StatusServiceUnavailable, err) - return - } - defer open.Close() - - said, err := do(r.Context(), open, in) - if err != nil { - // **The refusal the command line would have given, unchanged.** Carrying `said` with - // it matters: an assignment that was kept and still does not resolve is two facts, - // and dropping either makes the answer wrong. - answer(w, http.StatusConflict, map[string]any{"said": said, "refused": err.Error()}) - return - } - answer(w, http.StatusOK, map[string]any{"said": said}) - } -} - -func answer(w http.ResponseWriter, status int, body map[string]any) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(body) -} - -func refuse(w http.ResponseWriter, status int, err error) { - answer(w, status, map[string]any{"refused": err.Error()}) -} diff --git a/cmd/mesh-controller/api_test.go b/cmd/mesh-controller/api_test.go deleted file mode 100644 index 9cc5d99a..00000000 --- a/cmd/mesh-controller/api_test.go +++ /dev/null @@ -1,79 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -type letIn struct{} - -func (letIn) Who(*http.Request) (string, error) { return "somebody", nil } - -func asking(t *testing.T, who Authenticator, method, path, body string) *httptest.ResponseRecorder { - t.Helper() - recorded := httptest.NewRecorder() - commands(who).ServeHTTP(recorded, httptest.NewRequest(method, path, strings.NewReader(body))) - return recorded -} - -// **Nothing is served to somebody the mesh cannot identify**, and that is the default rather than -// a setting. The board this stands behind is published on a public name (novox/hq 11-a-board), so -// an unauthenticated command surface is authority over the mesh handed to whoever finds it. -func TestAnUnauthenticatedRequestIsRefused(t *testing.T) { - got := asking(t, mustAuthenticate("https://identity.example/realms/mesh"), - "POST", "/assign", `{"node":"anchor","module":"umami"}`) - if got.Code != http.StatusUnauthorized { - t.Fatalf("an unidentified caller got %d", got.Code) - } - // And it says what to do instead, because the answer is not "give up". - if !strings.Contains(got.Body.String(), "command line") { - t.Errorf("the refusal does not say what still works: %s", got.Body.String()) - } -} - -// The API refuses to start at all without being told whose tokens to believe. -// -// At start rather than per request, so it is found by whoever ran it rather than by whoever -// finds it. -func TestTheApiWillNotRunWithoutAnIssuer(t *testing.T) { - err := apiCommand(context.Background(), []string{}) - if err == nil { - t.Fatal("it served commands without being told who may give them") - } - if !strings.Contains(err.Error(), "issuer") { - t.Errorf("the refusal does not name what is missing: %v", err) - } -} - -// A request missing what it acts on is refused before anything is opened. -func TestARequestThatNamesNothingIsRefused(t *testing.T) { - // **The exact refusal, not merely "not accepted".** Asserting non-200 passes even when the - // request got as far as opening a store and failing there, which proves nothing about whether - // anything was checked — that is what the first version of this test did. - for _, body := range []string{`{}`, `{"node":"anchor"}`, `{"module":"umami"}`, `not json`} { - got := asking(t, letIn{}, "POST", "/assign", body) - if got.Code != http.StatusBadRequest { - t.Errorf("%s got %d, and a request naming nothing is a bad request", body, got.Code) - } - } -} - -// A verb nobody implemented is said plainly. A command surface answering 404 to something -// somebody expected is indistinguishable from one that is down. -func TestAnUnknownRouteSaysWhatIsAccepted(t *testing.T) { - got := asking(t, letIn{}, "POST", "/rotate", `{}`) - if got.Code != http.StatusNotFound { - t.Fatalf("got %d", got.Code) - } - var said map[string]any - _ = json.Unmarshal(got.Body.Bytes(), &said) - if !strings.Contains(said["refused"].(string), "/assign") { - t.Errorf("it does not say what it does accept: %v", said) - } -} diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go deleted file mode 100644 index 5f76ef80..00000000 --- a/cmd/mesh-controller/apply_failing.go +++ /dev/null @@ -1,164 +0,0 @@ -package main - -import ( - "fmt" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part, -// for longer than one apply interval (novox/hq issue 233). -const kindApplyFailing = "apply-failing" - -// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its -// ReconcileEvery. A failure still there after one more of them is not one the next apply clears. -const applyEvery = 5 * time.Minute - -// packageManagerKey is how a node-engine names a package manager that does not answer among what failed -// (mesh-host's probe.go): "the package manager (pacman)". -const packageManagerKey = "the package manager (" - -// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only -// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every -// declaration for an hour and a half because its package manager did not answer, and the mesh's only word -// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply -// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine -// says a re-apply only when how it went changed, so the same failure is one report however long it lasts. -// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while -// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost. -func watchApplyFailing(f *signalFacts) []conditions.Observation { - heard := map[string]bool{} - for _, m := range f.machines { - if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) { - heard[m.name] = true - } - } - var out []conditions.Observation - said := map[string]bool{} - for _, d := range f.doing { - if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { - continue - } - out = append(out, applyFailing(d, f.now)) - said[d.Node] = true - } - // **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's - // newest word, judged by the same bound — the client says a failure once, and fills again on every change — - // on a machine whose node-engine applied what it was sent. - failed := map[string][]inventory.ProvReport{} - for _, r := range f.provFailed { - if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] { - continue - } - failed[r.Node] = append(failed[r.Node], r) - } - for _, node := range sortedNames(failed) { - out = append(out, provisioningFailing(node, failed[node], f.now)) - } - return out -} - -// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to -// place. -func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation { - sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module }) - var names, words []string - since := failed[0].At - for _, r := range failed { - names = append(names, r.Module) - words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason))) - if r.At.Before(since) { - since = r.At - } - } - o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, - Severity: conditions.Warning, Said: strings.Join(words, "; "), - Resolved: m + " applies what it is sent again"} - o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m, - strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery)) - o.Headline = m + " is not applying its update" - o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+ - "details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since))) - return o -} - -func orNoReason(s string) string { - if s == "" { - return "no reason given" - } - return s -} - -// applyFailing is one machine's S21 condition. -func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { - m := d.Node - since := d.Since.UTC().Format("2006-01-02 15:04 MST") - o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, - Severity: conditions.Warning, Said: doingWords(d), - Resolved: m + " applies what it is sent again"} - how := "refused its declaration whole" - if d.Outcome != inventory.OutcomeRefused { - how = "applied its declaration only in part" - } - o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d)) - o.Headline = m + " is not applying its update" - o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", - m, plainHow(d), humanDuration(now.Sub(*d.Since))) - if tool := unansweringTool(d); tool != "" { - o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ - "is applied, since %s (bound %s)", m, tool, since, ago(applyEvery)) - o.Headline = fmt.Sprintf("%s cannot install packages", m) - o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ - "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ - "look on the machine.", m, tool, humanDuration(now.Sub(*d.Since))) - } - return o -} - -// plainHow is how a machine failed, as the operator reads it. -func plainHow(d inventory.Doing) string { - if d.Outcome == inventory.OutcomeRefused { - return "refused every update" - } - return "applied its updates only in part" -} - -// unansweringTool is the package manager a machine said does not answer, or "". -func unansweringTool(d inventory.Doing) string { - for _, r := range d.Failed { - if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok { - return strings.TrimSuffix(rest, ")") - } - } - return "" -} - -// failedNames is what failed, by name: the refusal, or the failed resources' ids. -func failedNames(d inventory.Doing) string { - if d.Outcome == inventory.OutcomeRefused { - return "refused" - } - ids := make([]string, 0, len(d.Failed)) - for _, r := range d.Failed { - ids = append(ids, r.ID) - } - sort.Strings(ids) - return "failed " + strings.Join(ids, ", ") -} - -// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh. -func doingWords(d inventory.Doing) string { - if d.Outcome == inventory.OutcomeRefused { - return "refused: " + oneLine(d.Refused) - } - var parts []string - for _, r := range d.Failed { - parts = append(parts, r.ID+": "+oneLine(r.Error)) - } - sort.Strings(parts) - return strings.Join(parts, "; ") -} diff --git a/cmd/mesh-controller/apply_failing_test.go b/cmd/mesh-controller/apply_failing_test.go deleted file mode 100644 index bb76f910..00000000 --- a/cmd/mesh-controller/apply_failing_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its -// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer, -// the same failure on times reports since for before now. -func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing { - since := now.Add(-forHowLong) - return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second), - Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{ - {ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` + - "pacman does not answer here. Either this machine is not Arch, or its package database is broken: " + - "pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"}, - {ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " + - "declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " + - "file /etc/pacman.conf could not be read: No such file or directory"}}} -} - -// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the -// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary. -func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) { - now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) - f := calm(now) - f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)} - got := watchApplyFailing(f) - if len(got) != 1 { - t.Fatalf("raised %d conditions, want one: %+v", len(got), got) - } - o := got[0] - if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" { - t.Errorf("the condition is not the machine's: %s", o.Key()) - } - if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") { - t.Errorf("the summary does not name the tool and since when: %s", o.Summary) - } - if strings.Contains(o.Summary, "/etc/") { - t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary) - } - if !strings.Contains(o.Said, "could not be read") { - t.Errorf("the evidence does not carry the probe's words: %s", o.Said) - } - if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" { - t.Errorf("the plain words do not say it: %+v", o) - } -} - -// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the -// node-engine applied the rest. -func TestARefusalThatGoesOnIsRaised(t *testing.T) { - now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) - since := now.Add(-time.Hour) - f := calm(now) - f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12, - Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}} - got := watchApplyFailing(f) - if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") || - !strings.Contains(got[0].Said, "pacman does not answer") { - t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got) - } -} - -// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233): -// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one -// younger is not yet; and a silent or sleeping machine is S1's. -func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) { - now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) - f := calm(now) - f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)} - if got := watchApplyFailing(f); len(got) != 1 { - t.Errorf("one report of a failure six minutes old was not raised: %+v", got) - } - f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)} - if got := watchApplyFailing(f); len(got) != 0 { - t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got) - } - f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} - f.machines[0].lastHeard = now.Add(-time.Hour) - if got := watchApplyFailing(f); len(got) != 0 { - t.Errorf("a silent machine was raised for its last report: %+v", got) - } - f = calm(now) - f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} - f.doing[0].Node = "a-machine-the-mesh-does-not-know" - if got := watchApplyFailing(f); len(got) != 0 { - t.Errorf("a machine not heard of was raised: %+v", got) - } -} - -// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says -// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised -// is cleared through the keeper on the next tick. -func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) { - now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return now }}) - defer k.Close(context.Background()) - w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} - - failing := calm(now) - failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)} - w.see(t.Context(), failing) - open, err := k.Open(t.Context()) - if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing { - t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err) - } - - clean := calm(now.Add(5 * time.Minute)) - clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine - w.see(t.Context(), clean) - if open, _ := k.Open(t.Context()); len(open) != 0 { - t.Fatalf("the condition stayed open after the clean re-apply: %+v", open) - } -} diff --git a/cmd/mesh-controller/ask.go b/cmd/mesh-controller/ask.go deleted file mode 100644 index 9006b791..00000000 --- a/cmd/mesh-controller/ask.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "time" - - "github.com/novox/mesh-controller/internal/link" -) - -// ask calls one of a module's tools, through the control plane's own broker connection. -// -// A module serves tools under an account scoped to exactly that (novox/hq ADR 0047), and nothing -// else in the mesh held an account that could ask one — not an operator at a terminal, not an agent -// acting for one (novox/hq 04-ISSUES/049). The control plane does, so it is the way in: one -// process, one connection, one place a question can be seen to have been asked (ADR 0095). -func askCommand(ctx context.Context, args []string) error { - positionals, flags := split(args) - set := flag.NewFlagSet("ask", flag.ContinueOnError) - wait := set.Duration("wait", 60*time.Second, "how long to wait for the module's answer") - if err := set.Parse(flags); err != nil { - return err - } - if len(positionals) < 2 || len(positionals) > 3 { - return errors.New("ask [json arguments] [--wait 60s]") - } - module, tool := positionals[0], positionals[1] - var arguments json.RawMessage - if len(positionals) == 3 { - if !json.Valid([]byte(positionals[2])) { - return fmt.Errorf("the arguments are not JSON: %s", positionals[2]) - } - arguments = json.RawMessage(positionals[2]) - } - - server, err := connectLink(ctx, nil, nil, nil) - if err != nil { - return err - } - defer server.Close() - - answer, err := link.Ask(ctx, server.Bus(), module, tool, arguments, *wait) - if err != nil { - return err - } - // The answer as the module gave it, to standard output, for a person or a program. A tool - // that answered with an error has still answered: printed the same way, and the exit status - // says which. - body, err := json.Marshal(answer) - if err != nil { - return err - } - fmt.Fprintln(os.Stdout, string(body)) - if answer.Error != "" { - return fmt.Errorf("%s.%s answered with an error: %s", module, tool, answer.Error) - } - return nil -} diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go deleted file mode 100644 index 68f3a24a..00000000 --- a/cmd/mesh-controller/asker.go +++ /dev/null @@ -1,1009 +0,0 @@ -package main - -// The controller asks, and acts on the operator's warrant (novox/hq ADR 0259 §6). It holds no channel, no -// identity and no factor: it asks the router like any other module, and performs the answer chosen with its -// own grant. -// -// - **For every open, unsilenced condition that needs the operator and names its answers**, one ask is -// published on the `operator-channel` seat under the controller's own name: the condition's words, its -// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve), -// answered by the operator, expiring after a day (a week when every option only acknowledges). A -// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired -// unanswered is asked again while the condition lasts; one the router refused is asked again after a wait -// that grows with each refusal in a row, at most half an hour, so a refusal is never believed longer. Each ask is kept in the controller's bucket -// `asked`, so a restart neither asks twice nor forgets. -// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may -// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at -// that option's level, and only while the condition is still open. It performs the action as itself — -// a silence through its own conditions, any other through the verb the action names — with the warrant's -// words as its why, and records it in the hand-act log as the operator's decision, naming the channel, -// the ask and the proofs. An ask that ended without a choice is recorded and nothing is done. -// - **A warrant it missed** while away is read from the router's record of its asks, under its own name. - -import ( - "context" - "crypto/rand" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - "sync" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// The asker's name on the seat: the controller's module. -const askerName = broker.ControllerSeat - -// How long an ask lasts: a day when an answer approves, a week when every answer only acknowledges. -const ( - // askApproveFor is a day less a margin, so an ask is never refused at the router for lasting a day and - // a moment (the SDK's bound is a day). - askApproveFor = 24*time.Hour - 10*time.Minute - askAcknowledgeFor = 7 * 24 * time.Hour - // askEvery is how often what is open is asked about again, beside every change. - askEvery = time.Minute - // askCatchUpAfter is how old an open ask is before the router's record of it is read: a warrant heard - // on the event needs no reading. - askCatchUpAfter = 2 * time.Minute - // askAgainAfterAnswer is how long a condition the operator answered is not asked about again with the - // same answers: what was chosen takes a while to clear it, and asking again at once would ask twice. - askAgainAfterAnswer = time.Hour - // askMostOpen is how many asks the controller holds open at once (the router refuses a fourth): the - // most urgent conditions first, then the oldest. - askMostOpen = asks.MostOpen - // askRefusedRetryMost is the longest a refusal is believed without asking again (novox/hq issue 369): the - // router refused while its channels had not yet said they could send, the channels could send twenty - // minutes later, and the controller repeated that refusal for eleven hours. A refused ask is asked again - // after askEvery, then twice as long after each refusal in a row, never longer than this — and at once when - // the channels change. - askRefusedRetryMost = 30 * time.Minute - // askRefusedCounted is how far back refusals in a row are counted for that wait. - askRefusedCounted = 6 * time.Hour - // askVerdictWait is how long an ask made again after a refusal waits for the router's word before it is - // taken as taken: the router refuses an ask as it reads it, so a refusal comes within seconds. Meanwhile - // the condition saying it was not delivered stands as it was, neither cleared nor raised again. - askVerdictWait = 2 * time.Minute -) - -// refusedRetryAfter is how long a part refused n times in a row waits before it is asked again. -func refusedRetryAfter(n int) time.Duration { - wait := askEvery - for i := 1; i < n && wait < askRefusedRetryMost; i++ { - wait *= 2 - } - return min(wait, askRefusedRetryMost) -} - -// What became of an ask, as the controller keeps it. -const ( - askOpen = "open" - askCancelled = "cancelled" -) - -// asked is one ask the controller made, as it keeps it. -type asked struct { - ID string `json:"id"` - Condition string `json:"condition"` - // Channels is what the channels were when it was asked (asker.channels): an ask the router refused is asked - // again at once when the condition's answers or the channels change, and otherwise after a wait that grows - // with each refusal in a row (refusedRetryAfter, novox/hq issue 369). - Channels string `json:"channels,omitempty"` - Ask asks.Ask `json:"ask"` - Actions []conditions.Action `json:"actions"` - // Options are the actions by option id. - Options map[string]int `json:"options"` - State string `json:"state"` - Opened time.Time `json:"opened"` - Ended time.Time `json:"ended,omitempty"` - Warrant *asks.Warrant `json:"warrant,omitempty"` - // Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts, - // then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again. - Acted string `json:"acted,omitempty"` - // Part is which ask of its condition this is (askPart): empty for the one that carries the condition's - // answers, or the authorising ones where it has both; "acknowledge" for its acknowledging answers asked - // apart (the review of 2026-10-09, M1). - Part string `json:"part,omitempty"` - // Rehearsal is an ask started at the controller's terminal (rehearse.go): about no condition, its answers - // perform nothing, and the reconciling of conditions leaves it alone. - Rehearsal bool `json:"rehearsal,omitempty"` - // Proposal is a settings layer proposed through a verb (proposals.go, novox/hq ADR 0277): about no - // condition, set on Approve by the serving controller itself, and left alone by the reconciling of conditions. - Proposal *settingsProposal `json:"proposal,omitempty"` -} - -// ofACondition says an ask is one of a condition's: not a rehearsal, not a proposal. -func (r asked) ofACondition() bool { return !r.Rehearsal && r.Proposal == nil } - -// partKey is an ask's place among what is asked: its condition and its part. -func partKey(condition, part string) string { return condition + "#" + part } - -// partAcknowledge is the part of a condition asked apart for its acknowledging answers. -const partAcknowledge = "acknowledge" - -// askPart is one ask a condition is asked with: its part, what it is about, and its answers. -type askPart struct { - name string - about string - actions []conditions.Action -} - -// levelOf is an action's level as an option offers it: one that says none is never taken for less than -// approve. -func levelOf(act conditions.Action) asks.Level { - if act.Level == "" { - return asks.Approve - } - return asks.Level(act.Level) -} - -// partsOf is the asks a condition is asked with (the review of 2026-10-09, M1): one, when its answers are all -// of one kind; else its authorising answers (Release, Stop, Restart) in one ask, about the condition, and its -// acknowledging ones (Silence) in another. **An acknowledgement never shares an ask with an approval**: a -// channel that only acknowledges would otherwise answer the ask, and end the approval with it. -func partsOf(c conditions.Condition) []askPart { - var ack, auth []conditions.Action - for _, act := range c.Actions { - if levelOf(act) == asks.Acknowledge { - ack = append(ack, act) - } else { - auth = append(auth, act) - } - } - if len(ack) == 0 || len(auth) == 0 { - return []askPart{{about: c.Key, actions: c.Actions}} - } - return []askPart{{about: c.Key, actions: auth}, - {name: partAcknowledge, about: c.Key + "." + partAcknowledge, actions: ack}} -} - -// askedStore keeps the asks (broker.AskedBucket). **Every write after the first is a compare-and-set** (the -// review of 2026-10-09, L2): an ask is created once, and changed only over the revision it was read at, the -// change decided again on what is read — so two controllers, or two deliveries of one warrant, never write -// over each other, and of two that would act only the one whose write stands does. -type askedStore interface { - Get(ctx context.Context, id string) (*asked, error) - // Create keeps a new ask, and refuses one already kept under its id. - Create(ctx context.Context, a asked) error - // Change applies change to the ask kept under id, by compare-and-set, and says whether its write stood. - // change says whether to write at all; on a write that came between, it is asked again on what is read. - Change(ctx context.Context, id string, change func(*asked) bool) (bool, error) - All(ctx context.Context) ([]asked, error) -} - -// askChangeTries is how often a change is read and tried again when another write came between. -const askChangeTries = 5 - -// asker is the controller asking the operator and acting on the answer. -type asker struct { - open func(ctx context.Context) ([]conditions.Condition, error) - silence func(ctx context.Context, key string, d time.Duration, by, why string) error - store askedStore - // publish puts a message on a subject's stream, de-duplicated by id. - publish func(ctx context.Context, subject string, body []byte, id string) error - // call performs an action's verb with its arguments, as the controller. - call func(ctx context.Context, a conditions.Action, args map[string]string) error - // setLayer sets a proposed settings layer on the operator's warrant (novox/hq ADR 0277); nil cannot. - setLayer setOnWarrant - // record writes the hand-act log. - record func(ctx context.Context, act link.HandAct) error - // routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing. - routerRecord func(ctx context.Context, id string) (*asks.Warrant, error) - // routerHere says whether a router holds the seat and takes asks under the asker's name; nil is yes. - routerHere func(ctx context.Context) (bool, error) - // grantHeld says whether the bus holds the controller's grant to ask: the user list the bus's machine was - // last sent is the one the mesh composes now (novox/hq issue 353). The grant is composed from the router's - // assignment and reaches the bus only when that machine is next pushed, so between `assign` and `push` - // the record says a router is here and the bus refuses every ask. why says what to do; nil is yes. - grantHeld func(ctx context.Context) (held bool, why string, err error) - // channels is what the channels are now, as a fingerprint: who holds which kind, promising what. - channels func(ctx context.Context) string - // raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be - // asked, and why. Nil raises nothing (a test that does not look). - raise func(ctx context.Context, obs []conditions.Observation) error - // tell tells the operator one plain notice that needs no answer (operator-channel.tell, novox/hq ADR 0380): - // the refusal of an answer to a plan's ask (ADR 0449 §3). Nil tells nothing (a test that does not look). - tell func(ctx context.Context, headline string, lines []string) error - now func() time.Time - logf func(string, ...any) - - saidNoRouter bool - saidNoGrant bool - // saidGrantCaveat is whether a grant held with a caveat was said, so it is said once. - saidGrantCaveat bool - - mu sync.Mutex - nudged chan struct{} -} - -func (a *asker) nudge() { - if a == nil { - return - } - a.mu.Lock() - if a.nudged == nil { - a.nudged = make(chan struct{}, 1) - } - ch := a.nudged - a.mu.Unlock() - select { - case ch <- struct{}{}: - default: - } -} - -// keep asks until ctx ends: now, on every change of a condition, and every askEvery. -func (a *asker) keep(ctx context.Context) { - a.nudge() - tick := time.NewTicker(askEvery) - defer tick.Stop() - a.mu.Lock() - nudged := a.nudged - a.mu.Unlock() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - case <-nudged: - } - if err := a.reconcile(ctx); err != nil { - a.logf("what the operator is asked could not be brought up to date: %v", err) - } - } -} - -// wants says whether a condition is one to ask about now. -func wants(c conditions.Condition, now time.Time) bool { - return len(c.Actions) > 0 && c.Needs != "" && !c.SilencedAt(now) -} - -func sameAsked(a []conditions.Action, b []conditions.Action) bool { - x, _ := json.Marshal(a) - y, _ := json.Marshal(b) - return string(x) == string(y) -} - -// reconcile brings what is asked in line with what is open. -func (a *asker) reconcile(ctx context.Context) error { - now := a.now() - if a.routerHere != nil { - here, err := a.routerHere(ctx) - if err != nil { - return err - } - if !here { - if !a.saidNoRouter { - a.logf("no router takes asks under the controller's name (a module declaring %s with its ask "+ - "named by its caller, assigned): the operator is asked nothing until one is", broker.AsksSeat) - a.saidNoRouter = true - } - open, err := a.open(ctx) - if err != nil { - return err - } - var unasked []conditions.Condition - for _, c := range open { - if wants(c, now) { - unasked = append(unasked, c) - } - } - return a.sayUnasked(ctx, unasked, "no router takes the controller's asks: no module holding "+ - broker.AsksSeat+" that takes an ask under its asker's name is assigned") - } - a.saidNoRouter = false - } - if a.grantHeld != nil { - held, why, err := a.grantHeld(ctx) - if err != nil { - return err - } - if !held { - if !a.saidNoGrant { - a.logf("the bus does not hold the controller's grant to ask yet: %s; the operator is asked nothing until it does", why) - a.saidNoGrant = true - } - open, err := a.open(ctx) - if err != nil { - return err - } - var unasked []conditions.Condition - for _, c := range open { - if wants(c, now) { - unasked = append(unasked, c) - } - } - return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why) - } - if why != "" && !a.saidGrantCaveat { - // Held, with a caveat worth one line (an older bus module cannot confirm the reload). - a.logf("the controller asks: %s", why) - a.saidGrantCaveat = true - } else if why == "" { - a.saidGrantCaveat = false - } - a.saidNoGrant = false - } - channels := "" - if a.channels != nil { - channels = a.channels(ctx) - } - open, err := a.open(ctx) - if err != nil { - return err - } - all, err := a.store.All(ctx) - if err != nil { - return err - } - byCondition := map[string]asked{} // by partKey - // What is open and not a condition's — a rehearsal, a proposal — still counts toward what the router holds - // open for the controller (askMostOpen); expired unanswered, it is kept so, as the router says it too. - otherOpen := 0 - for _, r := range all { - if r.State == askOpen && !r.ofACondition() && !now.Before(r.Ask.Expires) { - if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { - if x.State != askOpen || x.Acted != "" { - return false - } - x.State, x.Ended, x.Acted = string(asks.OutcomeExpired), now, "nothing: the ask expired unanswered" - return true - }); err != nil { - return err - } - continue - } - if r.State == askOpen && !r.ofACondition() { - otherOpen++ - } - if r.State == askOpen && r.ofACondition() { - k := partKey(r.Condition, r.Part) - if prior, held := byCondition[k]; !held || r.Opened.After(prior.Opened) { - byCondition[k] = r - } - } - } - // A warrant missed while away, read from the router's record: for a condition's ask, and for a proposal's or a - // rehearsal's alike. - if a.routerRecord != nil { - var lookedUp []asked - for _, r := range byCondition { - lookedUp = append(lookedUp, r) - } - for _, r := range all { - if r.State == askOpen && !r.ofACondition() { - lookedUp = append(lookedUp, r) - } - } - for _, r := range lookedUp { - if now.Sub(r.Opened) < askCatchUpAfter { - continue - } - if w, err := a.routerRecord(ctx, r.ID); err == nil && w != nil { - body, _ := json.Marshal(w) - if err := a.Decided(ctx, body); err != nil { - return err - } - } - } - if all, err = a.store.All(ctx); err != nil { - return err - } - byCondition = map[string]asked{} - for _, r := range all { - if r.State == askOpen && r.ofACondition() { - byCondition[partKey(r.Condition, r.Part)] = r - } - } - } - // What the operator answered lately, by condition: not asked again at once; and what the router refused, - // newest first: not asked again until the answers or the channels change. - answered, refused := map[string]asked{}, map[string]asked{} - for _, r := range all { - k := partKey(r.Condition, r.Part) - if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer { - if prior, has := answered[k]; !has || r.Ended.After(prior.Ended) { - answered[k] = r - } - } - if r.State == string(asks.OutcomeRefused) { - if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) { - refused[k] = r - } - } - } - // Refusals in a row, by part: those since the last ask that was not refused, within askRefusedCounted. - // superseded: a part asked since its last refusal, by an ask the router did not refuse — open, answered, - // expired or cancelled. Its refusal is history then, never said again (the review of PR 198: an answered - // retry brought the refusal back). - inRow, superseded := map[string]int{}, map[string]asked{} - for k, last := range refused { - for _, r := range all { - if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent && - r.Opened.After(last.Opened) && r.Opened.After(superseded[k].Opened) { - superseded[k] = r - } - } - var since time.Time - for _, r := range all { - if partKey(r.Condition, r.Part) == k && r.State != string(asks.OutcomeRefused) && r.State != askUnsent && - !r.Opened.After(last.Opened) && r.Opened.After(since) { - since = r.Opened - } - } - for _, r := range all { - if partKey(r.Condition, r.Part) == k && r.State == string(asks.OutcomeRefused) && r.Opened.After(since) && - now.Sub(r.Opened) < askRefusedCounted { - inRow[k]++ - } - } - } - wanted := map[string]bool{} - var unasked []conditions.Condition // refused by the router, and nothing it was refused for changed - var refusedWords []string - // The most urgent first, then the oldest: those are asked when no more than askMostOpen may be. - sort.SliceStable(open, func(i, j int) bool { - ui, uj := open[i].Severity == conditions.Urgent, open[j].Severity == conditions.Urgent - if ui != uj { - return ui - } - if !open[i].Raised.Equal(open[j].Raised) { - return open[i].Raised.Before(open[j].Raised) - } - return open[i].Key < open[j].Key - }) - openNow := otherOpen - for _, c := range open { - if !wants(c, now) { - continue - } - for _, p := range partsOf(c) { - if r, held := byCondition[partKey(c.Key, p.name)]; held && sameAsked(r.Actions, p.actions) && now.Before(r.Ask.Expires) { - openNow++ - } - } - } - for _, c := range open { - if !wants(c, now) { - continue - } - saidUnasked := false - for _, p := range partsOf(c) { - key := partKey(c.Key, p.name) - wanted[key] = true - if r, was := refused[key]; was && sameAsked(r.Actions, p.actions) { - cur, held := byCondition[key] - ended := r.Ended - if ended.IsZero() { - ended = r.Opened - } - later, asked := superseded[key] - waiting := !held && !asked && r.Channels == channels && now.Sub(ended) < refusedRetryAfter(inRow[key]) - // Asked again now (the wait over), or lately and the router's word not in yet: said as it was until - // that word, so the condition neither clears nor is raised again at each try. - retrying := !held && !asked && !waiting - verdictDue := held && asked && later.ID == cur.ID && now.Sub(cur.Opened) < askVerdictWait - if waiting || retrying || verdictDue { - if !saidUnasked { - unasked, saidUnasked = append(unasked, c), true - } - if r.Warrant != nil && r.Warrant.Words != "" { - refusedWords = append(refusedWords, r.Warrant.Words) - } - } - if waiting { - continue // refused lately, and nothing it was refused for has changed: asked again after the wait - } - } - // **A condition raised after the answered ask ended is a new one** (novox/hq ADR 0449 §2): a retried - // plan that fails again is asked again, not held back for the hour its earlier answer holds. - if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) && !c.Raised.After(r.Ended) { - if _, held := byCondition[key]; !held { - continue - } - } - if r, held := byCondition[key]; held { - switch { - case !sameAsked(r.Actions, p.actions): - if err := a.cancel(ctx, r, "its answers changed"); err != nil { - return err - } - case !now.Before(r.Ask.Expires): - // Expired unanswered: the router says so too; asked again below while it lasts. - if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { - if x.State != askOpen { - return false - } - x.State, x.Ended = string(asks.OutcomeExpired), now - return true - }); err != nil { - return err - } - openNow-- - default: - continue - } - } - if openNow >= askMostOpen { - continue // asked when one of the open ones ends, most urgent first - } - if err := a.ask(ctx, c, p, channels); err != nil { - a.logf("the operator could not be asked about %s: %v", c.Key, err) - continue - } - openNow++ - } - } - stillOpen := map[string]conditions.Condition{} - for _, c := range open { - stillOpen[c.Key] = c - } - for key, r := range byCondition { - if wanted[key] { - continue - } - // **A silence never takes an approval back** (the confirmation review of 2026-10-09, M1). Silence is an - // acknowledgement — anyone at the desk may give it — so a condition silenced while its approval is asked - // keeps that ask open, unchanged, until it is answered on a channel that proves who answered, or expires. - // It is not asked again once it ends, while the silence lasts. - if c, open := stillOpen[r.Condition]; open && c.SilencedAt(now) && r.Ask.Highest() != asks.Acknowledge && - now.Before(r.Ask.Expires) && keepsItsAnswers(c, r) { - continue - } - if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil { - return err - } - } - why := "the router refused the ask" - if len(refusedWords) > 0 { - why += ": " + refusedWords[0] - } - return a.sayUnasked(ctx, unasked, why) -} - -// keepsItsAnswers says a condition still offers the answers an ask kept was asked with. -func keepsItsAnswers(c conditions.Condition, r asked) bool { - for _, p := range partsOf(c) { - if partKey(c.Key, p.name) == partKey(r.Condition, r.Part) { - return sameAsked(r.Actions, p.actions) - } - } - return false -} - -// sourceAsker raises the asker's own condition. -const sourceAsker = "asker" - -// sayUnasked keeps the asker's one condition: while a condition that needs the operator could not be asked -// on any channel, said loudly (failure must be loud), cleared when every one could be. -func (a *asker) sayUnasked(ctx context.Context, unasked []conditions.Condition, why string) error { - if a.raise == nil { - return nil - } - var obs []conditions.Observation - if len(unasked) > 0 { - keys := make([]string, 0, len(unasked)) - severity := conditions.Warning - for _, c := range unasked { - keys = append(keys, c.Key) - if c.Severity == conditions.Urgent { - severity = conditions.Urgent - } - } - sort.Strings(keys) - obs = append(obs, conditions.Observation{Scope: conditions.ScopeSeat, ID: broker.AsksSeat, Token: "unasked", - Kind: "asks-undelivered", Severity: severity, Source: sourceAsker, - Summary: fmt.Sprintf("%d condition(s) that need the operator could not be asked on any channel: %s; %s", - len(keys), strings.Join(keys, ", "), why), - Headline: "Questions for you not delivered", - Explanation: "The mesh could not send you its questions on any channel.", - Needs: "answer them from the mesh MCP server, and check why no channel carries them.", - Resolved: "The mesh can ask you again"}) - } - if err := a.raise(ctx, obs); err != nil { - a.logf("whether the operator could be asked could not be kept as a condition: %v", err) - } - return nil -} - -// optionID is an action's label as an option's id: "Silence for a week" is silence-for-a-week. -func optionID(label string) string { - var b strings.Builder - dash := false - for _, r := range strings.ToLower(label) { - switch { - case r >= 'a' && r <= 'z', r >= '0' && r <= '9': - b.WriteRune(r) - dash = false - case !dash && b.Len() > 0: - b.WriteByte('-') - dash = true - } - } - return strings.TrimSuffix(b.String(), "-") -} - -// doesWords is what an action does, in the words an option says it with. -func doesWords(act conditions.Action) string { - switch { - case act.Arguments["silence"] != "": - return "nothing more is said of it for a week" - case act.Verb == "mesh-delivery.release": - return "the delivery goes on" - case act.Verb == "mesh-delivery.stop": - return "the delivery ends" - case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["go"] != "": - return "the delivery starts" - case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "": - return "the delivery is stopped" - case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "": - return "the failed builds are asked again" - case act.Verb == broker.ControllerSeat+".push" && act.Arguments["node"] != "": - return "the held builds are sent to " + act.Arguments["node"] - case act.Verb == broker.ControllerSeat+".upgrade" && act.Arguments["release-backlog"] != "": - return "the builds held after the failed delivery are released" - case strings.HasSuffix(act.Verb, ".restart"): - return "its service is restarted on " + act.Machine - } - return strings.ToLower(act.Label) -} - -// askText is a condition's words as an ask says them: without where an answer is given when no channel can -// give it (FromMeshMCPServer), since the ask is answered on a channel and the router says where else. -func askText(s string) string { - for _, with := range []string{", " + FromMeshMCPServer, " " + FromMeshMCPServer} { - s = strings.ReplaceAll(s, with, ".") - } - return strings.ReplaceAll(s, "..", ".") -} - -// askOf is the ask one part of a condition is asked with. -func askOf(id string, c conditions.Condition, p askPart, now time.Time) (asks.Ask, map[string]int) { - q := asks.Ask{ID: id, Headline: c.Headline, Explanation: askText(c.Explanation), Who: asks.Operator, - OnExpiry: "nothing is done, and you are asked again while it lasts", About: p.about, - Urgent: c.Severity == conditions.Urgent} - options := map[string]int{} - approves := false - for i, act := range p.actions { - level := levelOf(act) // an action that says nothing of its level is never taken for less than approve - approves = approves || level != asks.Acknowledge - oid := optionID(act.Label) - options[oid] = i - // Every option binds the exact act it stands for (novox/hq ADR 0259 §6): the verb, the machine and - // every argument. The warrant then authorises that act and no other. - binds, _ := asks.ActDigest(boundAct(act)) - q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level, - Binds: binds}) - } - q.Expires = now.Add(askAcknowledgeFor) - if approves { - q.Expires = now.Add(askApproveFor) - } - return q, options -} - -// boundAct is what an option's Binds digests: the act exactly as the controller will perform it — its verb, -// machine, level, and each argument as "arg." — and never its label or words. -func boundAct(act conditions.Action) asks.Act { - out := asks.Act{"verb": act.Verb, "machine": act.Machine, "level": act.Level} - for k, v := range act.Arguments { - out["arg."+k] = v - } - return out -} - -func newAskID() string { - var b [8]byte - _, _ = rand.Read(b[:]) - return "c" + hex.EncodeToString(b[:]) -} - -// askUnsent is an ask kept and never published: asked again at the next look. -const askUnsent = "unsent" - -// ask publishes one ask about a part of a condition, kept before it is published (the review of 2026-10-09, -// L3): a warrant for it then always finds it, and one whose publishing failed is marked so and asked again. -func (a *asker) ask(ctx context.Context, c conditions.Condition, p askPart, channels string) error { - now := a.now() - id := newAskID() - q, options := askOf(id, c, p, now) - if err := q.Check(now); err != nil { - return err - } - body, err := json.Marshal(q) - if err != nil { - return err - } - if err := a.store.Create(ctx, asked{ID: id, Condition: c.Key, Part: p.name, Ask: q, Actions: p.actions, - Options: options, State: askOpen, Opened: now, Channels: channels}); err != nil { - return fmt.Errorf("the ask could not be kept, so it was not asked: %w", err) - } - if err := a.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil { - if _, cerr := a.store.Change(ctx, id, func(x *asked) bool { - if x.State != askOpen || x.Acted != "" { - return false - } - x.State, x.Ended, x.Acted = askUnsent, a.now(), "nothing: it could not be published: "+err.Error() - return true - }); cerr != nil { - a.logf("the ask %s could not be published, and could not be marked so: %v", id, cerr) - } - return err - } - a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options)) - return nil -} - -// cancel takes an ask back: kept cancelled first, so a warrant that comes after is refused, then said to the -// router; a cancel the router did not hear leaves the ask to expire there, and nothing is done on it here. -func (a *asker) cancel(ctx context.Context, r asked, why string) error { - stood, err := a.store.Change(ctx, r.ID, func(x *asked) bool { - if x.State != askOpen { - return false - } - x.State, x.Ended = askCancelled, a.now() - return true - }) - if err != nil || !stood { - return err - } - body, _ := json.Marshal(map[string]string{"id": r.ID}) - if err := a.publish(ctx, asks.CancelSubject(askerName), body, "cancel."+r.ID); err != nil { - a.logf("the ask %s about %s is taken back here, and the router could not be told (%v): it expires there, "+ - "and no answer to it is acted on", r.ID, r.Condition, err) - return nil - } - a.logf("took back the ask %s about %s: %s", r.ID, r.Condition, why) - return nil -} - -// Decided takes the router's word on one of the controller's asks (link.Decider). An error is returned only -// when what was decided could not be kept, so the word is held and heard again. -func (a *asker) Decided(ctx context.Context, body []byte) error { - var w asks.Warrant - if err := json.Unmarshal(body, &w); err != nil { - a.logf("the router's word on an ask could not be read; ignored: %v", err) - return nil - } - if w.Asker != askerName { - a.logf("REFUSED a warrant for %s's ask %s: the controller acts only on its own", w.Asker, w.Ask) - return nil - } - r, err := a.store.Get(ctx, w.Ask) - if err != nil { - return err - } - if r == nil { - a.logf("REFUSED a warrant for the ask %s, which the controller does not hold", w.Ask) - return nil - } - if r.Acted != "" { - return nil // heard again: acted on once - } - now := a.now() - if w.Outcome != asks.OutcomeChosen { - acted := "nothing: the ask " + string(w.Outcome) - if w.Words != "" { - acted += ": " + w.Words - } - if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { - if x.Acted != "" { - return false - } - x.State, x.Ended, x.Warrant, x.Acted = string(w.Outcome), now, &w, acted - return true - }); err != nil { - return err - } - a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome) - return nil - } - if r.State != askOpen { - // Cancelled, replaced or expired in the controller's own record: no answer to it is acted on. - a.logf("REFUSED a warrant for the ask %s, which is %s in the controller's own record", r.ID, r.State) - return nil - } - option, err := w.For(askerName, r.Ask) - if err != nil { - a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err) - return nil - } - index, offered := r.Options[option.ID] - if !offered || index >= len(r.Actions) { - a.logf("REFUSED a warrant for the ask %s: it chose %s, which no action stands for", r.ID, option.ID) - return nil - } - act := r.Actions[index] - // The act about to be performed is the one the option bound when the controller asked: a record changed - // since is refused, never performed. - if err := option.Performs(boundAct(act)); err != nil { - a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err) - return nil - } - open, err := a.open(ctx) - if err != nil { - return err - } - stillOpen := r.Rehearsal || r.Proposal != nil // a rehearsal and a proposal are about no condition - for _, c := range open { - stillOpen = stillOpen || c.Key == r.Condition - } - if !stillOpen { - // The asker checks the state is still what it asked about before it acts (to-be 46 §10, step 7). - if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { - if x.State != askOpen || x.Acted != "" { - return false - } - x.State, x.Warrant, x.Ended, x.Acted = string(asks.OutcomeChosen), &w, now, - "nothing: the condition ended before the answer" - return true - }); err != nil { - return err - } - a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition) - return nil - } - // Claimed before acting, by compare-and-set: only the delivery whose write stands acts (security review - // of 2026-10-08, finding 9). Not by the warrant's message id, which another publisher could take first: - // the controller's own record decides. - claimed, err := a.store.Change(ctx, r.ID, func(x *asked) bool { - if x.State != askOpen || x.Acted != "" { - return false - } - x.State, x.Warrant, x.Acted = string(asks.OutcomeChosen), &w, "acting" - return true - }) - if err != nil { - return err - } - if !claimed { - a.logf("the warrant for the ask %s was already taken by another delivery; nothing more is done", r.ID) - return nil - } - r.Acted = "acting" - - why := fmt.Sprintf("%s (ask %s)", w.Says(), r.ID) - args := map[string]string{} - for k, v := range act.Arguments { - args[k] = v - } - if v, takes := args["why"]; takes && v == "" { - args["why"] = why - } - var acted error - outcome := "done" - switch { - case r.Rehearsal && act.Verb == rehearsalVerb: - // A rehearsal's answer performs nothing: it is recorded below as the operator's decision. - case r.Proposal != nil && act.Verb == proposalVerb: - // A proposed settings layer, set by this controller itself on Approve (novox/hq ADR 0277): the act's - // digest of the values is held to the record's own values before anything is set. - outcome, acted = a.decideProposal(ctx, *r, act, w) - if acted == nil && outcome != "" && !strings.HasPrefix(outcome, "nothing") { - outcome = "done: " + outcome - } - case act.Arguments["silence"] != "": - acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why) - default: - acted = a.call(ctx, act, args) - } - ended := a.now() - if acted != nil { - outcome = "failed: " + acted.Error() - a.tellRefused(ctx, *r, act, acted) - } - r.Ended, r.Acted = ended, outcome - if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { - if x.Acted != "acting" { - return false - } - x.Ended, x.Acted = ended, outcome - return true - }); err != nil { - a.logf("%s was acted on (%s), and how it ended could NOT be kept: %v", r.ID, outcome, err) - } - verbArgs := []string{act.Verb} - if act.Machine != "" { - verbArgs = append(verbArgs, "on "+act.Machine) - } - keys := make([]string, 0, len(args)) - for k := range args { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - if k != "why" { - verbArgs = append(verbArgs, k+"="+args[k]) - } - } - if err := a.record(ctx, link.HandAct{Verb: handActWarrant, Args: verbArgs, Why: why, By: byWords(w), - Cause: conditions.CauseOperatorAnswer, Condition: r.Condition, Via: viaWords(w), Ask: r.ID, - Proofs: w.Proofs, RequestedBy: r.Condition, Outcome: r.Acted}); err != nil { - a.logf("%s was done, and could NOT be recorded in the hand-act log: %v", why, err) - } - a.logf("%s: %s", why, r.Acted) - return nil -} - -// tellRefused tells the operator what became of the answer they gave to a plan's ask, when it was not done (novox/hq -// ADR 0449 §3): the controller's `plans` verb refuses a Retry, Start or Stop when the plan moved on between the ask and -// the answer, and the refusal was only recorded, so the operator who pressed the answer heard nothing. **A refusal is -// told as one, and nothing else is**: a verb that said no changed nothing, but a call that got no answer (a timeout, a -// refused grant) may have run, and is told as not known. One plain notice with the ask's headline, beside the hand act -// the caller still records. An answer to any other ask is not told here. A notice that cannot be told is said in the -// log and changes nothing. -func (a *asker) tellRefused(ctx context.Context, r asked, act conditions.Action, failure error) { - if a.tell == nil || act.Verb != broker.ControllerSeat+".plans" { - return - } - var lines []string - var refused verbRefused - if errors.As(failure, &refused) { - lines = []string{ - fmt.Sprintf("Your answer, %s, could not be done: %s.", act.Label, strings.TrimRight(refusalWords(refused.words, act), ".")), - "Nothing was changed.", - } - } else { - lines = []string{ - fmt.Sprintf("Your answer, %s, was given, and it is not known whether it was done.", act.Label), - "Look at the delivery before you answer again.", - } - } - if err := a.tell(ctx, r.Ask.Headline, lines); err != nil { - a.logf("the operator could NOT be told that %s was refused (ask %s): %v", act.Label, r.ID, err) - } -} - -// refusalWords is a verb's refusal as the operator reads it: the plans named as "the plan" (plainRefusal), and, where -// the words still carry an identifier the operator's channel withholds, one sentence of the answer's own. -func refusalWords(words string, act conditions.Action) string { - return plainRefusalOr(words, refusalFallback(act)) -} - -// refusalFallback is what an answer's refusal says when its own words cannot be said: one sentence for each answer -// to a plan's ask, never another answer's. -func refusalFallback(act conditions.Action) string { - switch { - case act.Arguments["retry"] != "": - return "a retry would not run now; the details say why" - case act.Arguments["go"] != "": - return "the delivery could not be started now; the details say why" - case act.Arguments["stop"] != "": - return "the delivery could not be stopped now; the details say why" - } - return "the controller refused it; the details say why" -} - -// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision, -// never a repair (handActVerbs). -const handActWarrant = "warrant" - -// byWords is who chose, as the hand-act log says it: "the operator, as telegram identity 42". -func byWords(w asks.Warrant) string { - if w.By == nil { - return "the operator" - } - return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) -} - -// viaWords is the channel an answer came through: its module and kind, and how the sender was known. -func viaWords(w asks.Warrant) string { - if w.By == nil { - return w.Channel - } - via := w.Channel + " (" + w.By.Kind + ")" - if w.By.Verified != "" { - via += ", " + w.By.Verified - } - return via -} - -// errNotGranted is an action whose verb the controller's grant does not name. -var errNotGranted = errors.New("the controller's grant does not name this verb") diff --git a/cmd/mesh-controller/asker_bus_test.go b/cmd/mesh-controller/asker_bus_test.go deleted file mode 100644 index 32c53642..00000000 --- a/cmd/mesh-controller/asker_bus_test.go +++ /dev/null @@ -1,153 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "sync" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/asks" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// busAsker is an asker on a real bus's `asked` bucket, counting what it performs: two of them are two -// controllers sharing one record. -type busAskerRig struct { - mu sync.Mutex - called int - acts int - open []conditions.Condition - sent [][]byte -} - -func (rig *busAskerRig) asker(t *testing.T, conn *nats.Conn, now time.Time) *asker { - return &asker{ - open: func(context.Context) ([]conditions.Condition, error) { - rig.mu.Lock() - defer rig.mu.Unlock() - return rig.open, nil - }, - silence: func(context.Context, string, time.Duration, string, string) error { return nil }, - store: busAsked{conn: conn}, - publish: func(_ context.Context, subject string, body []byte, _ string) error { - rig.mu.Lock() - defer rig.mu.Unlock() - if subject == asks.AskSubject(askerName) { - rig.sent = append(rig.sent, body) - } - return nil - }, - call: func(context.Context, conditions.Action, map[string]string) error { - time.Sleep(20 * time.Millisecond) // long enough for the other delivery to arrive meanwhile - rig.mu.Lock() - defer rig.mu.Unlock() - rig.called++ - return nil - }, - record: func(context.Context, link.HandAct) error { - rig.mu.Lock() - defer rig.mu.Unlock() - rig.acts++ - return nil - }, - now: func() time.Time { return now }, - logf: t.Logf, - } -} - -func askedBus(t *testing.T) *nats.Conn { - t.Helper() - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - if _, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: broker.AskedBucket}); err != nil { - t.Fatal(err) - } - return conn -} - -// The review of 2026-10-09 (L7): two deliveries of one warrant, to two controllers at once, perform its act -// exactly once and record it once — the record's compare-and-set decides, never the warrant's message id. -func TestTwoAnswersAtOnceActOnce(t *testing.T) { - conn := askedBus(t) - now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC) - rig := &busAskerRig{open: []conditions.Condition{heldCondition()}} - first, second := rig.asker(t, conn, now), rig.asker(t, conn, now) - if err := first.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - if len(rig.sent) != 1 { - t.Fatalf("asked %d times", len(rig.sent)) - } - var q asks.Ask - _ = json.Unmarshal(rig.sent[0], &q) - release, _ := q.Option("release") - w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID, - Label: release.Label, Level: release.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now, - AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} - body, _ := json.Marshal(w) - var wg sync.WaitGroup - for _, a := range []*asker{first, second, first, second} { - wg.Add(1) - go func(a *asker) { - defer wg.Done() - if err := a.Decided(context.Background(), body); err != nil { - t.Error(err) - } - }(a) - } - wg.Wait() - if rig.called != 1 || rig.acts != 1 { - t.Fatalf("performed %d time(s), recorded %d time(s)", rig.called, rig.acts) - } - got, err := busAsked{conn: conn}.Get(context.Background(), q.ID) - if err != nil || got == nil || got.Acted != "done" { - t.Fatalf("kept as %+v (%v)", got, err) - } -} - -// The review of 2026-10-09 (L2): a write decided on a record read earlier never lands over one made since. A -// cancel read before the answer was acted on leaves the act's record as it is. -func TestAStaleCancelDoesNotWriteOverAnAct(t *testing.T) { - conn := askedBus(t) - now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC) - rig := &busAskerRig{open: []conditions.Condition{heldCondition()}} - a := rig.asker(t, conn, now) - if err := a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - var q asks.Ask - _ = json.Unmarshal(rig.sent[0], &q) - stale, _ := busAsked{conn: conn}.Get(context.Background(), q.ID) - release, _ := q.Option("release") - w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID, - Label: release.Label, Level: release.Level, Channel: "telegram", At: now, AskDigest: q.Digest(), - By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} - body, _ := json.Marshal(w) - if err := a.Decided(context.Background(), body); err != nil { - t.Fatal(err) - } - if err := a.cancel(context.Background(), *stale, "the condition ended"); err != nil { - t.Fatal(err) - } - got, _ := busAsked{conn: conn}.Get(context.Background(), q.ID) - if got.State != string(asks.OutcomeChosen) || got.Acted != "done" { - t.Errorf("a stale cancel wrote over the act: %+v", got) - } -} diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go deleted file mode 100644 index 4e732bee..00000000 --- a/cmd/mesh-controller/asker_test.go +++ /dev/null @@ -1,1006 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs -// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level. - -type memAskedStore map[string]asked - -// memAskedMu guards every memAskedStore: Change is a compare-and-set as the bus's is. -var memAskedMu sync.Mutex - -func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) { - memAskedMu.Lock() - defer memAskedMu.Unlock() - r, ok := m[id] - if !ok { - return nil, nil - } - return &r, nil -} -func (m memAskedStore) Create(_ context.Context, r asked) error { - memAskedMu.Lock() - defer memAskedMu.Unlock() - if _, kept := m[r.ID]; kept { - return errors.New("an ask is kept under that id") - } - m[r.ID] = r - return nil -} -func (m memAskedStore) Change(_ context.Context, id string, change func(*asked) bool) (bool, error) { - memAskedMu.Lock() - defer memAskedMu.Unlock() - r, ok := m[id] - if !ok || !change(&r) { - return false, nil - } - m[id] = r - return true, nil -} -func (m memAskedStore) All(context.Context) ([]asked, error) { - memAskedMu.Lock() - defer memAskedMu.Unlock() - var out []asked - for _, r := range m { - out = append(out, r) - } - return out, nil -} - -type published struct { - subject, id string - body []byte -} - -type askerRig struct { - a *asker - open []conditions.Condition - store memAskedStore - sent []published - called []string - silenced []string - acts []link.HandAct - now time.Time - // refuses, when set, is what every verb the asker calls answers; told is every notice the operator was told. - refuses error - told []told -} - -// told is one plain notice the asker told the operator. -type told struct { - headline string - lines []string -} - -func newAskerRig(t *testing.T) *askerRig { - r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)} - r.a = &asker{ - open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil }, - silence: func(_ context.Context, key string, d time.Duration, by, why string) error { - r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why) - // As the controller's conditions do (the confirmation review of 2026-10-09, M1): the condition is - // silenced from now on, so what is asked next sees it silenced. - for i := range r.open { - if r.open[i].Key == key { - r.open[i].Silenced = &conditions.Silence{Until: r.now.Add(d), By: by, Why: why, Since: r.now} - } - } - return nil - }, - store: r.store, - publish: func(_ context.Context, subject string, body []byte, id string) error { - r.sent = append(r.sent, published{subject, id, body}) - return nil - }, - call: func(_ context.Context, a conditions.Action, args map[string]string) error { - raw, _ := json.Marshal(args) - r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw)) - return r.refuses - }, - tell: func(_ context.Context, headline string, lines []string) error { - r.told = append(r.told, told{headline, lines}) - return nil - }, - record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil }, - now: func() time.Time { return r.now }, - logf: t.Logf, - } - return r -} - -func heldCondition() conditions.Condition { - release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove, - Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}} - stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove, - Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}} - o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096", Kind: "delivery-stalled", - Severity: conditions.Warning, Resolver: conditions.ResolverOperator, - Headline: "Delivery of hq held for 36 hours", - Explanation: "A delivery of hq has been held for 36 hours, past its limit.", - Needs: "release it, or stop it, " + FromMeshMCPServer, Actions: []conditions.Action{release, stop}} - return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} -} - -func unitsCondition() conditions.Condition { - key := "machine.shanks.units" - return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning, - Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.", - Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}} -} - -func (r *askerRig) asksSent(t *testing.T) []asks.Ask { - t.Helper() - var out []asks.Ask - for _, p := range r.sent { - if p.subject != asks.AskSubject("mesh-controller") { - continue - } - var q asks.Ask - if err := json.Unmarshal(p.body, &q); err != nil { - t.Fatal(err) - } - out = append(out, q) - } - return out -} - -func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) { - r := newAskerRig(t) - quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"} - r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - sent := r.asksSent(t) - if len(sent) != 2 { - t.Fatalf("asked %d times: %+v", len(sent), sent) - } - byAbout := map[string]asks.Ask{} - for _, q := range sent { - byAbout[q.About] = q - if err := q.Check(r.now); err != nil { - t.Errorf("%s: %v", q.About, err) - } - } - held := byAbout[heldCondition().Key] - if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve || - held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator || - held.OnExpiry == "" { - t.Errorf("the held delivery is asked %+v", held) - } - units := byAbout["machine.shanks.units"] - if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) { - t.Errorf("the failed units are asked %+v", units) - } - // No second ask while one is open. - r.now = r.now.Add(time.Minute) - _ = r.a.reconcile(context.Background()) - if n := len(r.asksSent(t)); n != 2 { - t.Errorf("asked again while open: %d", n) - } -} - -func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) { - r := newAskerRig(t) - r.open = []conditions.Condition{heldCondition(), unitsCondition()} - _ = r.a.reconcile(context.Background()) - // The units are silenced, the held delivery lasts past its ask's day. - units := unitsCondition() - units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)} - r.open = []conditions.Condition{heldCondition(), units} - r.now = r.now.Add(askApproveFor) - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - var cancels int - for _, p := range r.sent { - if p.subject == asks.CancelSubject("mesh-controller") { - cancels++ - } - } - if cancels != 1 { - t.Errorf("cancels %d, want the silenced one's", cancels) - } - if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key { - t.Errorf("the expired ask was not asked again: %+v", sent) - } -} - -// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label. -func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant { - t.Helper() - for _, a := range r.store { - if a.Condition != condition || a.State != askOpen { - continue - } - for _, o := range a.Ask.Options { - if o.Label == label { - return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen, - Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now, - AskDigest: a.Ask.Digest(), - By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} - } - } - } - t.Fatalf("no open ask about %s offers %s", condition, label) - return asks.Warrant{} -} - -func answerWith(t *testing.T, r *askerRig, w asks.Warrant) { - t.Helper() - body, _ := json.Marshal(w) - if err := r.a.Decided(context.Background(), body); err != nil { - t.Fatal(err) - } -} - -func TestAWarrantIsActedOnOnce(t *testing.T) { - r := newAskerRig(t) - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, heldCondition().Key, "Release") - answerWith(t, r, w) - answerWith(t, r, w) // heard again - if len(r.called) != 1 { - t.Fatalf("called %v", r.called) - } - want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}` - if r.called[0] != want { - t.Errorf("called\n %s\nwant\n %s", r.called[0], want) - } - if len(r.acts) != 1 { - t.Fatalf("hand-acts %+v", r.acts) - } - act := r.acts[0] - if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" || - act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" || - act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" { - t.Errorf("the hand-act %+v", act) - } - if !personsDecision(act) { - t.Error("an act on a warrant counts as a repair") - } - if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" { - t.Errorf("kept %+v", got) - } -} - -func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) { - for name, change := range map[string]func(*asks.Warrant){ - "another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" }, - "an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" }, - "an option not offered": func(w *asks.Warrant) { w.Option = "delete" }, - "another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge }, - "no person": func(w *asks.Warrant) { w.By = nil }, - "another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" }, - "no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" }, - } { - t.Run(name, func(t *testing.T) { - r := newAskerRig(t) - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, heldCondition().Key, "Stop") - change(&w) - answerWith(t, r, w) - if len(r.called)+len(r.acts)+len(r.silenced) != 0 { - t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced) - } - }) - } -} - -func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) { - r := newAskerRig(t) - r.open = []conditions.Condition{unitsCondition()} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, "machine.shanks.units", "Silence for a week") - w.Level, w.Proofs = asks.Acknowledge, nil - w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14", - Verified: "a desk click: whoever was at the operator's session on g14"} - w.Channel = "desk-channel" - answerWith(t, r, w) - if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") { - t.Fatalf("silenced %v, called %v", r.silenced, r.called) - } - if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 { - t.Errorf("%+v", r.acts) - } -} - -func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) { - r := newAskerRig(t) - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, heldCondition().Key, "Release") - w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time" - answerWith(t, r, w) - if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) || - !strings.HasPrefix(r.store[w.Ask].Acted, "nothing") { - t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask]) - } - // And a choice for a condition that ended meanwhile does nothing either. - r2 := newAskerRig(t) - r2.open = []conditions.Condition{heldCondition()} - _ = r2.a.reconcile(context.Background()) - w2 := r2.warrantFor(t, heldCondition().Key, "Release") - r2.open = nil - answerWith(t, r2, w2) - if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" { - t.Errorf("%v %+v", r2.called, r2.store[w2.Ask]) - } -} - -func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) { - r := newAskerRig(t) - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, heldCondition().Key, "Stop") - r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) { - if id != w.Ask { - return nil, errors.New("another ask") - } - return &w, nil - } - r.now = r.now.Add(askCatchUpAfter) - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") { - t.Errorf("called %v", r.called) - } - if n := len(r.asksSent(t)); n != 1 { - t.Errorf("asked again after the answer: %d", n) - } -} - -// After review (2026-10-08): a refused ask is not asked again at once; issue 369: nor is the refusal believed -// for ever — it is asked again after a wait that doubles with each refusal in a row, and at once when the -// channels change. -func TestAnAskTheRouterRefusedIsAskedAgainAfterAGrowingWait(t *testing.T) { - r := newAskerRig(t) - channels := "channel/telegram=telegram@anchor[choice]own:true" - r.a.channels = func(context.Context) string { return channels } - r.open = []conditions.Condition{heldCondition()} - refuse := func() { - t.Helper() - sent := r.asksSent(t) - refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller", - Outcome: asks.OutcomeRefused, Words: "no channel can carry any of its answers now", At: r.now}) - if err := r.a.Decided(context.Background(), refusal); err != nil { - t.Fatal(err) - } - } - _ = r.a.reconcile(context.Background()) - first := r.asksSent(t)[0] - refuse() - if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") { - t.Fatalf("the refusal was kept as %+v", got) - } - // Each wait: nothing asked before it ends, asked once when it has. - for i, wait := range []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute, - 16 * time.Minute, 30 * time.Minute, 30 * time.Minute} { - before := len(r.asksSent(t)) - r.now = r.now.Add(wait - 10*time.Second) - _ = r.a.reconcile(context.Background()) - if n := len(r.asksSent(t)); n != before { - t.Fatalf("refusal %d: asked again before %s", i+1, wait) - } - r.now = r.now.Add(10 * time.Second) - _ = r.a.reconcile(context.Background()) - if n := len(r.asksSent(t)); n != before+1 { - t.Fatalf("refusal %d: not asked again after %s (%d asks)", i+1, wait, n) - } - refuse() - } - before := len(r.asksSent(t)) - channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true" - _ = r.a.reconcile(context.Background()) - if n := len(r.asksSent(t)); n != before+1 { - t.Errorf("not asked again once the channels changed: %d", n-before) - } -} - -// After review: at most three asks open at once, the most urgent first, then the oldest. -func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) { - r := newAskerRig(t) - var open []conditions.Condition - for i := 0; i < 4; i++ { - c := unitsCondition() - c.Key = "machine.m" + string(rune('a'+i)) + ".units" - c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)} - c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour) - open = append(open, c) - } - urgent := heldCondition() - urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute) - r.open = append(open, urgent) - _ = r.a.reconcile(context.Background()) - sent := r.asksSent(t) - if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" { - var about []string - for _, q := range sent { - about = append(about, q.About) - } - t.Fatalf("asked %v", about) - } -} - -// After review: nothing is asked while no router takes asks under the controller's name, and that is said once. -func TestNothingIsAskedWithoutARouter(t *testing.T) { - r := newAskerRig(t) - var said []string - r.a.logf = func(f string, a ...any) { said = append(said, f) } - r.a.routerHere = func(context.Context) (bool, error) { return false, nil } - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - _ = r.a.reconcile(context.Background()) - if len(r.asksSent(t)) != 0 { - t.Error("asked with no router") - } - n := 0 - for _, s := range said { - if strings.Contains(s, "no router takes asks") { - n++ - } - } - if n != 1 { - t.Errorf("said %d times", n) - } -} - -// After review: the condition's words keep where an answer is given without a channel; the ask's text does not. -func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) { - c := heldCondition() - if !strings.Contains(c.Explanation, FromMeshMCPServer) { - t.Fatalf("the condition lost where it is answered: %q", c.Explanation) - } - q, _ := askOf("x", c, partsOf(c)[0], time.Now()) - if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") { - t.Errorf("the ask says %q", q.Explanation) - } - if askApproveFor >= 24*time.Hour { - t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor) - } -} - -// After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record, -// once the claim stands, and never when given after the ask expired. -func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) { - r := newAskerRig(t) - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, heldCondition().Key, "Release") - late := w - late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute) - body, _ := json.Marshal(late) - _ = r.a.Decided(context.Background(), body) - if len(r.called) != 0 { - t.Fatalf("acted on a warrant given after the ask expired: %v", r.called) - } - // Claimed already by another delivery: nothing done here. - kept := r.store[w.Ask] - kept.Acted = "acting" - r.store[w.Ask] = kept - body, _ = json.Marshal(w) - _ = r.a.Decided(context.Background(), body) - if len(r.called) != 0 { - t.Fatalf("acted though the claim was another's: %v", r.called) - } - // Cancelled in its own record: refused. - kept.Acted, kept.State = "", askCancelled - r.store[w.Ask] = kept - _ = r.a.Decided(context.Background(), body) - if len(r.called) != 0 { - t.Errorf("acted on a cancelled ask: %v", r.called) - } -} - -// novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no -// other. A record of the act changed after the ask — another delivery, another machine, another argument — -// is refused and nothing is performed. -func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) { - for name, change := range map[string]func(*conditions.Action){ - "another argument": func(a *conditions.Action) { - a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"} - }, - "another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" }, - "another machine": func(a *conditions.Action) { a.Machine = "anchor" }, - } { - t.Run(name, func(t *testing.T) { - r := newAskerRig(t) - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, heldCondition().Key, "Release") - kept := r.store[w.Ask] - acts := append([]conditions.Action(nil), kept.Actions...) - i := kept.Options[w.Option] - change(&acts[i]) - kept.Actions = acts - r.store[w.Ask] = kept - answerWith(t, r, w) - if len(r.called)+len(r.acts) != 0 { - t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts) - } - }) - } - // Every option of an ask binds its act. - q, _ := askOf("x", heldCondition(), partsOf(heldCondition())[0], time.Now()) - for _, o := range q.Options { - if o.Binds == "" { - t.Errorf("the option %s binds nothing", o.ID) - } - } -} - -// Failure is loud (novox/hq ADR 0259, the self-review of 2026-10-09): a condition that needs the operator and -// could not be asked on any channel — no router, or the router refused the ask — is a condition of its own, -// cleared once it can be asked again. -func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) { - r := newAskerRig(t) - var raised [][]conditions.Observation - r.a.raise = func(_ context.Context, obs []conditions.Observation) error { - raised = append(raised, obs) - return nil - } - last := func() []conditions.Observation { return raised[len(raised)-1] } - routerHere := false - r.a.routerHere = func(context.Context) (bool, error) { return routerHere, nil } - channels := "channel/telegram=telegram@anchor[choice]own:true" - r.a.channels = func(context.Context) string { return channels } - r.open = []conditions.Condition{heldCondition()} - - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 1 || got[0].Kind != "asks-undelivered" || - !strings.Contains(got[0].Summary, heldCondition().Key) || !strings.Contains(got[0].Summary, "no router") { - t.Fatalf("no router, said as %+v", got) - } - - routerHere = true - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 0 { - t.Fatalf("asked, and still said undelivered: %+v", got) - } - first := r.asksSent(t)[0] - refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused, - Words: "no channel can carry any of its answers now", At: r.now}) - if err := r.a.Decided(context.Background(), refusal); err != nil { - t.Fatal(err) - } - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no channel can carry") { - t.Fatalf("the router's refusal, said as %+v", got) - } - if why, ok := conditions.PlainWords(conditions.Words{Headline: last()[0].Headline, Explanation: last()[0].Explanation, - Needs: last()[0].Needs, Resolved: last()[0].Resolved}, ""); !ok { - t.Errorf("not plain: %s", why) - } - r.open = nil - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 0 { - t.Errorf("nothing needs asking, and still said: %+v", got) - } -} - -// The review of 2026-10-09 (M1): an acknowledging answer never shares an ask with an authorising one. A -// condition offering Restart and Silence is asked twice — Restart alone, about the condition, and Silence -// alone, apart — so Silence chosen on a channel that only acknowledges leaves the Restart ask open. -func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) { - r := newAskerRig(t) - key := "module.shanks.plex.down" - c := conditions.Condition{Key: key, Kind: "module-down", Severity: conditions.Urgent, Headline: "Plex down on shanks", - Explanation: "Needs you: restart it, or silence this.", Needs: "restart it, or silence this.", - Actions: []conditions.Action{ - {Label: "Restart", Verb: "node-service-manager.restart", Machine: "shanks", Level: conditions.LevelApprove, - Arguments: map[string]string{"unit": "plex"}}, - conditions.SilenceAction(key)}} - r.open = []conditions.Condition{c} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - sent := r.asksSent(t) - if len(sent) != 2 { - t.Fatalf("asked %d time(s): %+v", len(sent), sent) - } - for _, q := range sent { - if err := q.Check(r.now); err != nil { - t.Errorf("%s: %v", q.About, err) - } - levels := map[asks.Level]bool{} - for _, o := range q.Options { - levels[o.Level] = true - } - if len(levels) != 1 { - t.Errorf("the ask about %s mixes levels: %+v", q.About, q.Options) - } - } - byAbout := map[string]asks.Ask{} - for _, q := range sent { - byAbout[q.About] = q - } - if q := byAbout[key]; len(q.Options) != 1 || q.Options[0].Label != "Restart" { - t.Errorf("the condition's own ask: %+v", q) - } - if q := byAbout[key+".acknowledge"]; len(q.Options) != 1 || q.Options[0].Level != asks.Acknowledge { - t.Errorf("the acknowledging ask: %+v", q) - } - // Silence chosen: performed, and the Restart ask stays open, never asked twice. - answerWith(t, r, r.warrantFor(t, key, "Silence for a week")) - if len(r.silenced) != 1 || len(r.called) != 0 { - t.Fatalf("silenced %v called %v", r.silenced, r.called) - } - _ = r.a.reconcile(context.Background()) - open := 0 - for _, a := range r.store { - if a.State == askOpen && a.Condition == key { - open++ - if a.Part != "" || a.Ask.Options[0].Label != "Restart" { - t.Errorf("the open ask is %+v", a) - } - } - } - if open != 1 || len(r.asksSent(t)) != 2 { - t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t))) - } - // And the approval still answers: Restart chosen on a channel that proves who answered is performed. - answerWith(t, r, r.warrantFor(t, key, "Restart")) - if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "node-service-manager.restart@shanks") { - t.Errorf("the approval kept through a silence was not performed: %v", r.called) - } -} - -// novox/hq issue 353: the controller's grant to ask is composed from the router's assignment and reaches the -// bus only when its machine is pushed. Between the two, the bus refuses every ask (measured 2026-10-09, 17:54 to -// 17:56 local: seven refusals of mesh.seat.operator-channel.accept.ask.mesh-controller). So nothing is asked -// while the bus's user list is behind, it is said once, the conditions that need the operator are raised as -// undelivered with what to do, and the asks go out once the bus holds the grant. -func TestNothingIsAskedWhileTheBusLacksTheControllersGrant(t *testing.T) { - r := newAskerRig(t) - var said []string - r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) } - var raised [][]conditions.Observation - r.a.raise = func(_ context.Context, obs []conditions.Observation) error { - raised = append(raised, obs) - return nil - } - held := false - r.a.grantHeld = func(context.Context) (bool, string, error) { - return held, "the bus's user list on anchor is behind what the mesh composes; `push anchor` carries it", nil - } - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - _ = r.a.reconcile(context.Background()) - if len(r.asksSent(t)) != 0 { - t.Error("asked while the bus lacks the grant") - } - n := 0 - for _, s := range said { - if strings.Contains(s, "does not hold the controller's grant") { - n++ - } - } - if n != 1 { - t.Errorf("said %d times: %q", n, said) - } - if len(raised) == 0 || len(raised[len(raised)-1]) != 1 || - !strings.Contains(raised[len(raised)-1][0].Summary, "`push anchor` carries it") || - raised[len(raised)-1][0].Kind != "asks-undelivered" { - t.Fatalf("not said as a condition with what to do: %+v", raised) - } - held = true - _ = r.a.reconcile(context.Background()) - if sent := r.asksSent(t); len(sent) != 1 || sent[0].About != heldCondition().Key { - t.Errorf("not asked once the bus holds the grant: %+v", sent) - } - if last := raised[len(raised)-1]; len(last) != 0 { - t.Errorf("the undelivered condition was not cleared: %+v", last) - } -} - -// novox/hq issue 369: the router refused while its channels had not yet said they could send; they could twenty -// minutes later, and "Questions for you not delivered" repeated that refusal for eleven hours, escalating on it. -// The ask is made again; while the router's word on it is awaited the condition stands unchanged (neither -// cleared nor raised again); once the router took it, the condition clears; a new refusal is said in its own -// words. -func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testing.T) { - r := newAskerRig(t) - var raised [][]conditions.Observation - r.a.raise = func(_ context.Context, obs []conditions.Observation) error { - raised = append(raised, obs) - return nil - } - last := func() []conditions.Observation { return raised[len(raised)-1] } - r.a.channels = func(context.Context) string { return "channel/telegram=telegram@anchor[choice]own:true" } - r.open = []conditions.Condition{unitsCondition()} - refuse := func(words string) { - t.Helper() - sent := r.asksSent(t) - refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller", - Outcome: asks.OutcomeRefused, Words: words, At: r.now}) - if err := r.a.Decided(context.Background(), refusal); err != nil { - t.Fatal(err) - } - } - _ = r.a.reconcile(context.Background()) - refuse("no channel can carry any of its answers now: telegram: telegram has not said whether it can send") - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "telegram has not said") { - t.Fatalf("the refusal, said as %+v", got) - } - // The wait ends: asked again; until the router's word on it is in, the condition stands as it was. - r.now = r.now.Add(askEvery) - _ = r.a.reconcile(context.Background()) - if n := len(r.asksSent(t)); n != 2 { - t.Fatalf("not asked again: %d asks", n) - } - if got := last(); len(got) != 1 { - t.Fatalf("cleared while the router's word on the new ask was awaited: the condition would flap") - } - r.now = r.now.Add(askVerdictWait / 2) - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 1 { - t.Fatalf("cleared inside the verdict wait: the condition would flap") - } - // Refused again, now for a reason of today: said in those words. - refuse("no channel can carry any of its answers now: telegram: no account is linked") - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no account is linked") || - strings.Contains(got[0].Summary, "has not said") { - t.Fatalf("an old refusal's words repeated: %+v", got) - } - // Asked again after the doubled wait, and taken: no refusal comes, and the condition clears. - r.now = r.now.Add(2 * askEvery) - _ = r.a.reconcile(context.Background()) - if n := len(r.asksSent(t)); n != 3 { - t.Fatalf("not asked again after the doubled wait: %d asks", n) - } - r.now = r.now.Add(askVerdictWait) - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 0 { - t.Fatalf("taken, and still said undelivered: %+v", got) - } - if n := len(r.asksSent(t)); n != 3 { - t.Fatalf("an ask taken was asked again: %d asks", n) - } - // The operator answers it; the condition stays open (its fix takes a while): the old refusal is not said again. - taken := r.asksSent(t)[2] - r.store.Change(context.Background(), taken.ID, func(x *asked) bool { - x.State, x.Ended = string(asks.OutcomeChosen), r.now - return true - }) - for i := 0; i < 5; i++ { - r.now = r.now.Add(askEvery) - _ = r.a.reconcile(context.Background()) - if got := last(); len(got) != 0 { - t.Fatalf("an answered ask brought its old refusal back: %+v", got) - } - } - // Its explanation opens with one verdict, not two. - r.open = append(r.open, heldCondition()) - _ = r.a.reconcile(context.Background()) - refuse("no channel can carry any of its answers now") - _ = r.a.reconcile(context.Background()) - if e := conditions.Verdict(last()[0].Needs, last()[0].Explanation); strings.Count(e, "Needs you") != 1 { - t.Errorf("the explanation says its verdict twice: %q", e) - } -} - -// hq issue 353, the review: the bus holds the controller's grant to ask only when both facts hold — the mesh -// composes the controller's own grant, and the bus's module says it is in force on the running server — and -// each fact not known is not held, said with why. -func TestTheGrantIsHeldOnlyWhenComposedAndInForceOnTheBus(t *testing.T) { - yes := func(context.Context) (bool, string, error) { return true, "", nil } - no := func(why string) grantJudge { - return func(context.Context) (bool, string, error) { return false, why, nil } - } - broken := func(context.Context) (bool, string, error) { return false, "", errors.New("the store is away") } - for _, c := range []struct { - name string - composed, inForce grantJudge - held bool - says string - }{ - {"composed and in force", yes, yes, true, ""}, - {"not composed", no("no router is assigned"), yes, false, "no router is assigned"}, - {"composed, written, not reloaded", yes, no("the server has not reloaded"), false, "has not reloaded"}, - {"the composition unknown", broken, yes, false, "could not be worked out: the store is away"}, - {"the bus unknown", yes, broken, false, "could not be worked out: the store is away"}, - {"nothing judges the bus", yes, nil, false, "not judged"}, - } { - held, why, err := grantHeldBy(c.composed, c.inForce, time.Now)(context.Background()) - if err != nil || held != c.held || !strings.Contains(why, c.says) { - t.Errorf("%s: held %v (%q, %v)", c.name, held, why, err) - } - } - // Remembered for grantLookEvery, then judged again. - now := time.Date(2026, 10, 9, 18, 0, 0, 0, time.UTC) - inForce := false - judge := grantHeldBy(yes, func(context.Context) (bool, string, error) { return inForce, "not yet", nil }, - func() time.Time { return now }) - if held, _, _ := judge(context.Background()); held { - t.Fatal("held before the bus enforces it") - } - inForce = true - if held, _, _ := judge(context.Background()); held { - t.Error("judged again inside grantLookEvery") - } - now = now.Add(grantLookEvery) - if held, _, _ := judge(context.Background()); !held { - t.Error("not judged again after grantLookEvery") - } -} - -// The controller's own grant, as composed: present while a router that takes asks is assigned, absent -// otherwise — whatever else in the user list changes. -func TestTheControllersOwnGrantToAskIsReadFromTheComposition(t *testing.T) { - router := broker.Declared{Module: "messenger", Holds: []broker.Seat{{Name: broker.AsksSeat, Scope: "mesh", - Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "decided"}}}} - other := broker.Declared{Module: "dunst"} - with := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {router, other}}} - without := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {other}}} - if may, err := controllerMayAsk(with); err != nil || !may { - t.Errorf("with a router assigned: %v %v", may, err) - } - if may, err := controllerMayAsk(without); err != nil || may { - t.Errorf("with no router assigned: %v %v", may, err) - } -} - -// The bus module's answer, with the record's word on the list sent as the check only where the answer does -// not settle it (reviews 22 and 31 of PR 188); each way it is not held names its own remedy. -func TestTheBusModulesAnswerNamesTheRemedyForEachCase(t *testing.T) { - const ( - reloaded = `{"allowed":true,"in_force":true,"file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:56:05Z"}` - notLoaded = `{"allowed":true,"in_force":false,"why":"the user list on the machine allows it, and the server has not reloaded since it was written","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}` - noVarz = `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}` - noStat = `{"allowed":true,"in_force":false,"why":"when the bus's user list was written could not be read inside mesh-broker-nats: exit 1"}` - notAllowed = `{"allowed":false,"in_force":false,"why":"the user list on the machine does not allow it"}` - ) - current := busListState{Machine: "anchor"} - behind := busListState{Machine: "anchor", Behind: true} - stepped := busListState{Machine: "anchor", Behind: true, BusStep: "sending anchor would replace the bus; `bus upgrade --why …`"} - uncomposed := busListState{Machine: "anchor", Behind: true, Uncomposed: errors.New("module foo requires a provision nobody provides")} - for _, c := range []struct { - name, answer string - askErr error - list busListState - listErr error - held bool - says, not string - }{ - {"in force", reloaded, nil, behind, nil, true, "", "push"}, - {"written, not reloaded", notLoaded, nil, current, nil, false, "within about 5 s", "push"}, - {"in force not known: varz", noVarz, nil, current, nil, false, "is not known: the server's last configuration load", "wait"}, - {"in force not known: stat", noStat, nil, current, nil, false, "is not known: when the bus's user list was written", "wait"}, - {"not allowed, list behind", notAllowed, nil, behind, nil, false, "`push anchor` carries it", "5 s"}, - {"not allowed, bus step pending", notAllowed, nil, stepped, nil, false, "`bus upgrade", "`push anchor`"}, - {"not allowed, list uncomposed", notAllowed, nil, uncomposed, nil, false, "`plan anchor`", "`push anchor`"}, - {"older module, allowed, list current", `{"allowed":true}`, nil, current, nil, true, "not confirmed", "push"}, - {"older module, allowed, list behind", `{"allowed":true}`, nil, behind, nil, true, "not confirmed", "push"}, - {"older module, allowed, list unknown", `{"allowed":true}`, nil, current, errors.New("the store is away"), true, "`bus upgrade`", "push"}, - {"older module, not allowed, list behind", `{"allowed":false}`, nil, behind, nil, false, "`push anchor` carries it", "5 s"}, - {"module cannot answer, list current", "", errors.New("no responders"), current, nil, true, "could not answer", "push"}, - {"module cannot answer, list behind", "", errors.New("no responders"), behind, nil, false, "`push anchor` carries it", ""}, - {"module cannot answer, list unknown", "", errors.New("no responders"), current, errors.New("the store is away"), false, "could not be worked out", ""}, - {"answer unreadable, list behind", `not json`, nil, behind, nil, false, "could not be read", ""}, - } { - judge := busGrantBy( - func(context.Context) (string, json.RawMessage, error) { - return "anchor", json.RawMessage(c.answer), c.askErr - }, - func(context.Context) (busListState, error) { return c.list, c.listErr }) - held, why, _ := judgeGrantOnce(judge) - if held != c.held || !strings.Contains(why, c.says) || (c.not != "" && strings.Contains(why, c.not)) { - t.Errorf("%s: held %v (%q)", c.name, held, why) - } - } - none := busGrantBy(func(context.Context) (string, json.RawMessage, error) { return "", nil, nil }, nil) - if held, why, _ := judgeGrantOnce(none); held || !strings.Contains(why, "no module holds the bus") { - t.Errorf("no bus: held %v (%q)", held, why) - } -} - -// judgeGrantOnce is the whole judgement with the composed grant held, as the asker makes it. -func judgeGrantOnce(bus grantJudge) (bool, string, error) { - yes := func(context.Context) (bool, string, error) { return true, "", nil } - return grantHeldBy(yes, bus, time.Now)(context.Background()) -} - -// The orders reviews 22 and 31 of PR 188 found: this controller live before the bus module that answers -// in_force, with the bus's machine behind on its list — and, after mesh-catalog #166 merged, unpushable until a -// person's bus upgrade. An older module that allows the ask asks (once, with the caveat said); a new module that -// has not reloaded names the wait, not a push; one that cannot say whether it reloaded names what is not known. -func TestAnOlderBusModuleNeitherSilencesTheAsksNorNamesTheWrongRemedy(t *testing.T) { - for _, c := range []struct { - name, answer string - list busListState - asks bool - says, not string - }{ - {"older module, list current", `{"allowed":true}`, busListState{Machine: "anchor"}, true, "not confirmed", ""}, - {"older module, list behind, bus step pending", `{"allowed":true}`, - busListState{Machine: "anchor", Behind: true, BusStep: "a planned step"}, true, "not confirmed", ""}, - {"new module, not reloaded", - `{"allowed":true,"in_force":false,"why":"not reloaded","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`, - busListState{Machine: "anchor"}, false, "within about 5 s", "push"}, - {"new module, reload not known", `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`, - busListState{Machine: "anchor"}, false, "is not known", "wait"}, - } { - t.Run(c.name, func(t *testing.T) { - r := newAskerRig(t) - var said []string - r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) } - var raised [][]conditions.Observation - r.a.raise = func(_ context.Context, obs []conditions.Observation) error { - raised = append(raised, obs) - return nil - } - yes := func(context.Context) (bool, string, error) { return true, "", nil } - r.a.grantHeld = grantHeldBy(yes, busGrantBy( - func(context.Context) (string, json.RawMessage, error) { - return "anchor", json.RawMessage(c.answer), nil - }, - func(context.Context) (busListState, error) { return c.list, nil }), - func() time.Time { return r.now }) - r.open = []conditions.Condition{heldCondition()} - _ = r.a.reconcile(context.Background()) - _ = r.a.reconcile(context.Background()) - sent := r.asksSent(t) - if c.asks { - if len(sent) != 1 || sent[0].About != heldCondition().Key { - t.Fatalf("not asked: %+v", sent) - } - n := 0 - for _, s := range said { - if strings.Contains(s, c.says) { - n++ - } - } - if n != 1 { - t.Errorf("the caveat said %d times: %q", n, said) - } - return - } - if len(sent) != 0 { - t.Fatalf("asked while the bus does not hold the grant: %+v", sent) - } - if len(raised) == 0 || len(raised[len(raised)-1]) != 1 || - raised[len(raised)-1][0].Kind != "asks-undelivered" { - t.Fatalf("not said as undelivered: %+v", raised) - } - summary := raised[len(raised)-1][0].Summary - if !strings.Contains(summary, c.says) || (c.not != "" && strings.Contains(summary, c.not)) { - t.Errorf("the condition names the wrong remedy: %q", summary) - } - }) - } -} - -// The serving asker is wired to the real judgement: with records and a bus it cannot read, the grant is not -// held, and so nothing is asked (a stub saying yes, or no judgement at all, would ask). -func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) { - a := servingAsker(&stores{}, nil, nil, nil) - if a.grantHeld == nil { - t.Fatal("the serving asker does not judge the bus's grant") - } - held, why, err := a.grantHeld(context.Background()) - if err != nil || held || !strings.Contains(why, "could not be worked out") { - t.Errorf("held %v (%q, %v) with nothing to read", held, why, err) - } -} diff --git a/cmd/mesh-controller/asker_wire.go b/cmd/mesh-controller/asker_wire.go deleted file mode 100644 index 7f756226..00000000 --- a/cmd/mesh-controller/asker_wire.go +++ /dev/null @@ -1,613 +0,0 @@ -package main - -// The asker on the bus: its asks in the controller's bucket `asked`, its asks and cancels published on the -// seat under the controller's name, the verbs a warrant chooses called with the controller's grant, and the -// router's record of its asks read under its name (novox/hq ADR 0259). - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/asks" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// askerFrom is the serving controller's asker; nil in any other process. -var askerFrom atomic.Pointer[asker] - -// askWithin is how long a verb a warrant chose is given to answer. -const askWithin = time.Minute - -type busAsked struct{ conn *nats.Conn } - -func (b busAsked) kv(ctx context.Context) (jetstream.KeyValue, error) { - js, err := jetstream.New(b.conn) - if err != nil { - return nil, err - } - return js.KeyValue(ctx, broker.AskedBucket) -} - -func (b busAsked) Get(ctx context.Context, id string) (*asked, error) { - kv, err := b.kv(ctx) - if err != nil { - return nil, err - } - e, err := kv.Get(ctx, id) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return nil, nil - } - if err != nil { - return nil, err - } - var r asked - return &r, json.Unmarshal(e.Value(), &r) -} - -// Create keeps a new ask under its id, and only where none is kept: never over another. -func (b busAsked) Create(ctx context.Context, r asked) error { - kv, err := b.kv(ctx) - if err != nil { - return err - } - body, err := json.Marshal(r) - if err != nil { - return err - } - _, err = kv.Create(ctx, r.ID, body) - return err -} - -// Change applies change to the ask kept under id by compare-and-set on its key's revision (the review of -// 2026-10-09, L2): read, changed, and written only over the revision read; when another write came between, -// read again and asked again, at most askChangeTries times. change says whether to write at all. -func (b busAsked) Change(ctx context.Context, id string, change func(*asked) bool) (bool, error) { - kv, err := b.kv(ctx) - if err != nil { - return false, err - } - for try := 0; try < askChangeTries; try++ { - e, err := kv.Get(ctx, id) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return false, nil - } - if err != nil { - return false, err - } - var r asked - if err := json.Unmarshal(e.Value(), &r); err != nil { - return false, err - } - if !change(&r) { - return false, nil - } - body, err := json.Marshal(r) - if err != nil { - return false, err - } - if _, err := kv.Update(ctx, id, body, e.Revision()); err != nil { - var api *jetstream.APIError - if errors.Is(err, jetstream.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence) { - continue - } - return false, err - } - return true, nil - } - return false, fmt.Errorf("the ask %s changed under every one of %d tries", id, askChangeTries) -} - -func (b busAsked) All(ctx context.Context) ([]asked, error) { - kv, err := b.kv(ctx) - if err != nil { - return nil, err - } - lister, err := kv.ListKeys(ctx) - if err != nil { - return nil, err - } - defer func() { _ = lister.Stop() }() - var out []asked - for k := range lister.Keys() { - e, err := kv.Get(ctx, k) - if err != nil { - continue - } - var r asked - if json.Unmarshal(e.Value(), &r) == nil { - out = append(out, r) - } - } - return out, nil -} - -// verbRefused is a verb's own refusal of an act: the seat answered, and said no. Any other error of callAction (a bus -// that did not answer, a grant that is refused) is not one: the verb may have run. -type verbRefused struct{ verb, words string } - -func (e verbRefused) Error() string { return e.verb + " refused: " + e.words } - -// callAction performs an action's verb as the controller, through the grant that names it. -func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error { - return func(ctx context.Context, a conditions.Action, args map[string]string) error { - seat, verb, ok := strings.Cut(a.Verb, ".") - if !ok { - return fmt.Errorf("%q names no seat and verb", a.Verb) - } - body := map[string]any{} - for k, v := range args { - body[k] = v - } - if seat == catalogue.DeliverySeat { - _, err := askDeliveryOwner(ctx, conn, verb, body) - return err - } - granted := false - for _, v := range broker.VerbsTheControllerActsOnAWarrant { - granted = granted || (v.Seat == seat && v.Verb == verb) - } - if !granted { - return fmt.Errorf("%s: %w", a.Verb, errNotGranted) - } - var answer link.Answer - var err error - if a.Machine != "" { - answer, err = link.AskSeatTool(ctx, conn, seat, verb, a.Machine, body, askWithin) - } else { - answer, err = link.AskMeshSeatTool(ctx, conn, seat, verb, body, askWithin) - } - if err != nil { - return err - } - if answer.Error != "" { - return verbRefused{verb: a.Verb, words: answer.Error} - } - return nil - } -} - -// tellThrough tells the operator one plain notice through the operator channel's `tell`, over the controller's own -// connection and grant (novox/hq ADR 0380); a holder's refusal is an error naming it. -func tellThrough(conn *nats.Conn) func(ctx context.Context, headline string, lines []string) error { - return func(ctx context.Context, headline string, lines []string) error { - answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell", - map[string]any{"headline": headline, "lines": lines}, tellWithin) - if err != nil { - return err - } - if answer.Error != "" { - return fmt.Errorf("operator-channel.tell refused: %s", answer.Error) - } - return nil - } -} - -// routerRecordOf reads the router's record of one of the controller's asks, under its name, and answers -// how it ended when it did: the bucket is the one the asks seat's declarer names as its records. -func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.Context, id string) (*asks.Warrant, error) { - return func(ctx context.Context, id string) (*asks.Warrant, error) { - bucket, err := asksRecords(ctx, inv) - if err != nil || bucket == "" { - return nil, err - } - reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+askerName+"."+id, nil) - if err != nil { - return nil, err - } - if reply.Header.Get("Status") != "" { - return nil, nil // none, or not readable: the event says it - } - var rec struct { - State string `json:"state"` - Warrant *asks.Warrant `json:"warrant"` - } - if json.Unmarshal(reply.Data, &rec) != nil || rec.State == "open" || rec.Warrant == nil { - return nil, nil - } - return rec.Warrant, nil - } -} - -// asksRecords is the bucket the asks seat's declarer keeps its record of asks in. -func asksRecords(ctx context.Context, inv catalogueReader) (string, error) { - declared, err := inv.Catalogue(ctx) - if err != nil { - return "", err - } - for _, m := range declared { - for _, s := range m.DefinesSeats { - if s.Name == broker.AsksSeat && len(s.Records) > 0 { - return broker.BucketName(m.Module, s.Records[0]), nil - } - } - } - return "", nil -} - -// routerHereIn says whether a module declaring the asks seat, with its ask named by its caller, is assigned: -// without it nothing takes an ask, and asking would only fill a queue nobody reads. -func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, error) { - return func(ctx context.Context) (bool, error) { - entries, err := inv.Catalogued(ctx) - if err != nil { - return false, err - } - for _, e := range entries { - for _, s := range e.Manifest.DefinesSeats { - if s.Name == broker.AsksSeat && s.NamedByCaller("ask") && len(e.On) > 0 { - return true, nil - } - } - } - return false, nil - } -} - -// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353), judged on the two -// facts that make it so, and failing closed on each: -// -// 1. **composed**: the user list the mesh composes now grants the controller itself to publish its ask -// (controllerMayAsk) — the controller's own grant, not whether any user of the list changed; -// 2. **in force**: the bus's own module, on the machine holding the bus, says the user list there allows it -// and the server reloaded after that list was written (the nats module's `nats_user_can`, `in_force`) — a -// list sent is not a list the bus enforces until it reloads. -// -// Anything that cannot be worked out is not held, with why, so the conditions that need the operator are said -// as undelivered rather than asked into a refusal. -func grantHeldIn(open *stores, conn *nats.Conn) func(ctx context.Context) (bool, string, error) { - return grantHeldBy(composedGrantIn(open), busGrantIn(open, conn), time.Now) -} - -// grantJudge is one of the two facts: whether it holds, and why not. -type grantJudge func(ctx context.Context) (bool, string, error) - -// grantHeldBy is the judgement over the two facts, remembered for grantLookEvery: composing the list resolves -// the bus's machine whole, and asking the bus's module is a call. -func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx context.Context) (bool, string, error) { - var mu sync.Mutex - var at time.Time - var held bool - var why string - return func(ctx context.Context) (bool, string, error) { - mu.Lock() - defer mu.Unlock() - if !at.IsZero() && now().Sub(at) < grantLookEvery { - return held, why, nil - } - held, why = judgeGrant(ctx, composed, inForce) - at = now() - return held, why, nil - } -} - -// judgeGrant is held only when both facts hold; an error is a fact not known, never a yes. A fact that holds -// with a caveat (a reload not confirmed) gives that caveat as the why of a yes. -func judgeGrant(ctx context.Context, composed, inForce grantJudge) (bool, string) { - var caveat []string - for _, fact := range []struct { - judge grantJudge - what string - }{{composed, "whether the mesh composes the controller a grant to ask"}, {inForce, "whether the bus enforces it"}} { - if fact.judge == nil { - return false, fact.what + " is not judged here" - } - ok, why, err := fact.judge(ctx) - if err != nil { - return false, fact.what + " could not be worked out: " + err.Error() - } - if !ok { - return false, why - } - if why != "" { - caveat = append(caveat, why) - } - } - return true, strings.Join(caveat, "; ") -} - -// controllerMayAsk says whether the controller's own grant, as composed from these records, lets it publish its -// ask (ADR 0259 §3: composed while a router that takes asks under its asker's name is assigned). -func controllerMayAsk(records broker.Records) (bool, error) { - users, err := broker.Users(records) - if err != nil { - return false, err - } - for _, u := range users { - if u.Kind != broker.KindController { - continue - } - perms, err := broker.PermissionsFor(u) - if err != nil { - return false, err - } - return broker.MayPublish(perms, asks.AskSubject(askerName)), nil - } - return false, errors.New("the composition holds no controller") -} - -// composedGrantIn is the first fact, from the mesh's records. -func composedGrantIn(open *stores) grantJudge { - return func(ctx context.Context) (bool, string, error) { - if open == nil || open.inventory == nil { - return false, "", errors.New("the mesh's records are not open") - } - records, err := open.inventory.BusRecords(ctx) - if err != nil { - return false, "", err - } - may, err := controllerMayAsk(records) - if err != nil || may { - return may, "", err - } - return false, "the mesh composes the controller no grant to ask: no router that takes asks under its " + - "asker's name is assigned", nil - } -} - -// busGrantIn is the second fact, asked of the bus's own module on the machine holding the bus, with the check -// main made before it (novox/hq issue 353, PR 186: is the user list sent to the bus's machine the one composed -// now) kept for the cases the module's word does not settle. -func busGrantIn(open *stores, conn *nats.Conn) grantJudge { - ask := func(ctx context.Context) (string, json.RawMessage, error) { - if open == nil || open.inventory == nil || conn == nil { - return "", nil, errors.New("the mesh's records or the bus are not open") - } - holders, err := seatHolders(ctx, open.inventory) - if err != nil { - return "", nil, err - } - h, held := holders[theBrokerSeat] - if !held || h.Node == "" || h.Module == "" { - return "", nil, nil - } - a, err := link.AskModuleToolOn(ctx, conn, h.Module, busUserCanTool, h.Node, map[string]any{ - "user": broker.ControllerName, "action": "publish", "subject": asks.AskSubject(askerName)}, 15*time.Second) - if err != nil { - return h.Node, nil, err - } - if a.Error != "" { - return h.Node, nil, errors.New(a.Error) - } - return h.Node, a.Result, nil - } - list := func(ctx context.Context) (busListState, error) { - if open == nil || open.inventory == nil { - return busListState{}, errors.New("the mesh's records are not open") - } - s, err := brokerListState(ctx, open, nil) - if err != nil || s.Machine == "" || !s.Behind { - return s, err - } - step, err := busHeld(ctx, open.inventory, []string{s.Machine}) - if err != nil { - return busListState{}, err - } - s.BusStep = step[s.Machine] - return s, nil - } - return busGrantBy(ask, list) -} - -// busListState is the user list on the bus's machine as the controller's record knows it: whether the list -// sent there is behind the one composed now, why it could not be composed, and the bus step a send there waits -// on (that machine cannot be pushed until a person's `bus upgrade`). -type busListState struct { - Machine string - Behind bool - Uncomposed error - BusStep string -} - -// remedy is the one thing that sends the bus's machine the list composed now: the plan's own reason when it -// cannot be composed, the bus upgrade while sends there wait on it, and otherwise a push of that machine. -func (s busListState) remedy(node string) string { - if s.Machine != "" { - node = s.Machine - } - switch { - case s.Uncomposed != nil: - return fmt.Sprintf("the user list for %s cannot be composed (%v); `plan %s` says what to fix first", - node, s.Uncomposed, node) - case s.BusStep != "": - return fmt.Sprintf("sends to %s wait on the bus's planned step, so a push will not carry it: %s", node, s.BusStep) - } - return fmt.Sprintf("`push %s` carries it", node) -} - -// busGrantBy judges the bus's word, and names for each way it is not held the one thing that makes it held: -// -// - **in force**: held; -// - **allowed, from a module older than `in_force`**: held, with the caveat that the reload is not confirmed. -// The file allows the publish, and the bus reloads within about 5 s of a change to it, so the worst case is -// one ask refused in those seconds, which is asked again (issue 369). The list check is not consulted: -// while a bus step is pending the bus's machine cannot be pushed, so any change to the composed list would -// read as behind for good and silence every ask until a person's bus upgrade (the review of PR 188); -// - **allowed, written, not reloaded** (the module gives both times): wait about 5 s; -// - **allowed, in force not known** (the module could not read a time): what is not known, said; -// - **not allowed, or the module cannot answer**: the list check decides — the remedy that sends the list -// when it is behind (busListState.remedy), and, for a module that cannot answer, held while the list sent -// is current, as main judged it. -// -// ask answers the machine holding the bus and its module's answer ("" when nothing holds the bus); list is the -// record's word on the list sent to that machine. -func busGrantBy(ask func(ctx context.Context) (string, json.RawMessage, error), - list func(ctx context.Context) (busListState, error)) grantJudge { - return func(ctx context.Context) (bool, string, error) { - node, raw, askErr := ask(ctx) - if askErr == nil && node == "" { - return false, "no module holds the bus, so nothing says what it enforces", nil - } - var answer busAnswer - if askErr == nil { - if answer, askErr = readBusAnswer(raw); askErr != nil { - askErr = fmt.Errorf("its answer could not be read: %w", askErr) - } - } - if askErr != nil { - if node == "" { - node = "the bus's machine" - } - s, err := list(ctx) - if err != nil { - return false, "", fmt.Errorf("the bus module on %s could not answer (%v), and whether the list sent "+ - "there is the one composed now could not be worked out: %w", node, askErr, err) - } - if s.Behind { - return false, fmt.Sprintf("the bus module on %s could not answer (%v), and the user list sent there is "+ - "behind what the mesh composes; %s", node, askErr, s.remedy(node)), nil - } - return true, fmt.Sprintf("the bus module on %s could not answer (%v); judged by the user list sent there, "+ - "which is the one composed now", node, askErr), nil - } - notAllowed := func(older string) (bool, string, error) { - s, err := list(ctx) - if err != nil { - return false, "", fmt.Errorf("the user list on %s does not allow the controller to ask%s, and whether "+ - "the list sent there is the one composed now could not be worked out: %w", node, older, err) - } - if s.Behind { - return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s: it is behind "+ - "what the mesh composes; %s", node, older, s.remedy(node)), nil - } - return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s, though the list "+ - "sent there is the one composed now; %s again", node, older, s.remedy(node)), nil - } - switch { - case answer.InForce == nil && answer.Allowed: - return true, fmt.Sprintf("the user list on %s allows the controller to ask, but its reload is not "+ - "confirmed: the bus module there is older than the answer whether a grant is in force, and the bus "+ - "upgrade (`bus upgrade`) carries one that confirms it", node), nil - case answer.InForce == nil: - return notAllowed(" (the bus module there is older than the answer whether a grant is in force)") - case !answer.Allowed: - return notAllowed("") - case *answer.InForce: - return true, "", nil - case answer.FileWritten != "" && answer.ServerLoaded != "": - return false, fmt.Sprintf("the bus on %s has the controller's grant to ask written (%s) but has not "+ - "reloaded since (last load %s); it reloads in place within about 5 s, so nothing is to be done but wait", - node, answer.FileWritten, answer.ServerLoaded), nil - } - why := answer.Why - if why == "" { - why = "it gives no reason" - } - return false, fmt.Sprintf("the user list on %s allows the controller to ask, but whether the bus enforces it "+ - "is not known: %s", node, why), nil - } -} - -// busUserCanTool is the bus module's answer to whether a user may do something, and whether it is in force. -const busUserCanTool = "nats_user_can" - -// busAnswer is the bus module's answer to busUserCanTool; InForce is nil from a module older than that field, -// and FileWritten and ServerLoaded are given only when it could read both times (mesh-catalog's nats InForce). -type busAnswer struct { - Allowed bool `json:"allowed"` - InForce *bool `json:"in_force"` - Why string `json:"why"` - FileWritten string `json:"file_written"` - ServerLoaded string `json:"server_loaded"` -} - -func readBusAnswer(raw json.RawMessage) (busAnswer, error) { - var answer busAnswer - err := json.Unmarshal(raw, &answer) - return answer, err -} - -// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent. -const grantLookEvery = 30 * time.Second - -// channelsIn is what the channels are now, as a fingerprint: each module claiming a kind of the channel -// bench, where, promising what, and whether of its own account. An ask the router refused is asked again -// once this changes. -func channelsIn(inv *inventory.Inventory) func(ctx context.Context) string { - return func(ctx context.Context) string { - entries, err := inv.Catalogued(ctx) - if err != nil { - return "" - } - var parts []string - for _, e := range entries { - for _, c := range e.Manifest.Claims { - if c.Kind == "" || !catalogue.KindedBenches[c.Name] { - continue - } - on := append([]string(nil), e.On...) - sort.Strings(on) - caps := append([]string(nil), c.Capabilities...) - sort.Strings(caps) - parts = append(parts, fmt.Sprintf("%s/%s=%s@%s[%s]own:%t", c.Name, c.Kind, e.Manifest.Module, - strings.Join(on, ","), strings.Join(caps, ","), e.Manifest.RunsAs != "")) - } - } - sort.Strings(parts) - return strings.Join(parts, ";") - } -} - -// startAsking makes the serving controller's asker and hands it the router's words. -func startAsking(ctx context.Context, open *stores, server *link.Server, conn *nats.Conn, keeper *conditions.Keeper) { - js, err := jetstream.New(conn) - if err != nil { - fmt.Printf("the operator cannot be asked: %v\n", err) - return - } - a := servingAsker(open, conn, js, keeper) - if err := server.Decides(a); err != nil { - fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err) - return - } - askerFrom.Store(a) - goServing(ctx, func() { a.keep(ctx) }) -} - -// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and -// records, and the judgements it asks by — the bus's grant to ask among them (novox/hq issue 353). -func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper *conditions.Keeper) *asker { - return &asker{ - open: keeper.Open, - silence: func(ctx context.Context, key string, d time.Duration, by, why string) error { - _, err := keeper.Silence(ctx, key, d, by, why) - return err - }, - store: busAsked{conn: conn}, - publish: func(ctx context.Context, subject string, body []byte, id string) error { - _, err := js.Publish(ctx, subject, body, jetstream.WithMsgID(id)) - return err - }, - call: callAction(conn), - // A proposed settings layer is set by this controller itself on the warrant (novox/hq ADR 0277). - setLayer: setLayerIn(open), - record: func(ctx context.Context, act link.HandAct) error { - _, err := link.RecordHandAct(ctx, conn, act) - return err - }, - tell: tellThrough(conn), - routerRecord: routerRecordOf(conn, open.inventory), - routerHere: routerHereIn(open.inventory), - grantHeld: grantHeldIn(open, conn), - channels: channelsIn(open.inventory), - raise: func(ctx context.Context, obs []conditions.Observation) error { - return keeper.Reconcile(ctx, sourceAsker, obs) - }, - now: time.Now, - logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }, - } -} diff --git a/cmd/mesh-controller/assignment_accounts.go b/cmd/mesh-controller/assignment_accounts.go deleted file mode 100644 index 242853de..00000000 --- a/cmd/mesh-controller/assignment_accounts.go +++ /dev/null @@ -1,489 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" - "strings" - "sync" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" -) - -// An assignment that stands issues its node's bus accounts and the module's grant (novox/hq ADR 0543, issue 1945; a -// step of research 072's grants: the memberships and credentials heard from placement, not from a send or a build). -// -// ADR 0539 clause 7 issues a module's credential and grant when its build is done. A module already built and assigned -// to one more node never has a build done again, so nothing issued that node's account: on 2026-10-25 at 17:19 -// mesh-build-node and mesh-build-go were assigned to ace, composed for it, and the bus's user list never named -// ace.mesh-build-node nor ace.mesh-build-go; one runtime found no credential, the other was refused by the bus, both -// restarting every 5 s. The assign act composes the assigned node alone (composedAfterAct): never the node holding the -// bus, never the grant. -// -// So the write of an assignments key is the event. When a key `.` of the assignments bucket comes to stand -// (a new key, or one that was ended or pending), the lease holder issues the module's accounts on that node that are not -// minted (idempotent: a minted account is never issued again, a rotation is on purpose), composes the node holding the -// bus so its user list carries them, and writes the grants. A rewrite of an assignment that already stood (the -// provisioner's plan kept in its value) does nothing. A starting controller looks at every standing assignment once, -// from one reading of the catalogue: it issues what is missing, then composes the bus's node and writes the grants -// once. The assign act issues too; both mint only an account no row names, in one statement, so exactly one of them -// mints and seals it (mintBusUser, the review of issue 1945). - -// assignmentAccountSteps are what an assignment that comes to stand does, each a step a test may stand in for. -type assignmentAccountSteps struct { - // holds says this controller holds the lease: only its holder issues. - holds func(ctx context.Context) bool - // forgetStale forgets every module's accounts on a node where the module no longer resolves, and answers a line for - // each (issue 1945: rows of an unassignment before this release, marked sealed by migration 0108). - forgetStale func(ctx context.Context) ([]string, error) - // catalogue reads the catalogue once, for every assignment a look issues for. - catalogue func(ctx context.Context) (map[string]catalogue.Manifest, error) - // issue issues the module's accounts on the node that are not minted, from the catalogue read; issued says it - // minted one, line what it did. - issue func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (issued bool, line string, err error) - // busHolder is the node holding the bus, "" for none. - busHolder func(ctx context.Context) (string, error) - // compose writes the nodes' compositions; grants writes the grants issued. - compose func(ctx context.Context, nodes []string) error - grants func(ctx context.Context) error -} - -// settle issues what the standing assignment of module on node lacks. fresh says the assignment just came to stand: the -// bus's node is composed and the grants written even when every account was minted before (a module assigned again -// keeps its account, and the bus's user list left it out while it was unassigned). It answers what it did, "" for -// nothing. -func (s assignmentAccountSteps) settle(ctx context.Context, node, module string) (string, error) { - if module == controllerModule { - // The controller's own module is issued at the terminal alone: its `broker` secret is the credential it is - // using (rollout.go), and it is assigned and unassigned there only (refusedMovingTheController). - return "", nil - } - if !s.holds(ctx) { - return "", errNotHolding - } - shelf, err := s.catalogue(ctx) - if err != nil { - return "", fmt.Errorf("the catalogue cannot be read: %w", err) - } - _, line, err := s.issue(ctx, shelf, node, module) - if err != nil { - return "", fmt.Errorf("its bus accounts on %s are not issued: %w", node, err) - } - said, err := s.cover(ctx) - prefix := module + " on " + node + ": " - if line != "" { - prefix += line + "; " - } - return prefix + said, err -} - -// look is the start's look: the accounts every standing assignment lacks, issued one after another from one reading of -// the catalogue, then the bus's node composed and the grants written once, when it looked at any assignment, whatever -// it issued (an assignment that came to stand while no controller heard it may have its accounts and still lack its -// place in the bus's user list or its grant). It answers what it did. -func (s assignmentAccountSteps) look(ctx context.Context, standing []chain.Assignment) (string, error) { - var forgot []string - if len(standing) > 0 && s.forgetStale != nil { - // Only from a reading of the assignments that holds some: an empty or failed one would forget every account. - if !s.holds(ctx) { - return "", errNotHolding - } - var err error - if forgot, err = s.forgetStale(ctx); err != nil { - return "", fmt.Errorf("the accounts of modules no longer on their nodes are not forgotten: %w", err) - } - } - var looked []chain.Assignment - for _, a := range standing { - if a.Module != controllerModule { - looked = append(looked, a) - } - } - if len(looked) == 0 { - return strings.Join(forgot, "\n"), nil - } - if !s.holds(ctx) { - return "", errNotHolding - } - shelf, err := s.catalogue(ctx) - if err != nil { - return "", fmt.Errorf("the catalogue cannot be read: %w", err) - } - var lines []string - for _, a := range looked { - if ctx.Err() != nil { - return "", ctx.Err() - } - if _, line, err := s.issue(ctx, shelf, a.Node, a.Module); err != nil { - return "", fmt.Errorf("%s's bus accounts on %s are not issued: %w", a.Module, a.Node, err) - } else if line != "" { - lines = append(lines, a.Module+" on "+a.Node+": "+line) - } - } - said, err := s.cover(ctx) - lines = append(append(forgot, lines...), fmt.Sprintf("%d standing assignment(s) looked at; %s", len(looked), said)) - return strings.Join(lines, "\n"), err -} - -// errNotHolding is a controller that does not hold the lease: it issues nothing, and tries again until it holds it. -var errNotHolding = errors.New("this controller does not hold the lease") - -// cover composes the node holding the bus and writes the grants, and says so. -func (s assignmentAccountSteps) cover(ctx context.Context) (string, error) { - holder, err := s.busHolder(ctx) - if err != nil { - return "", fmt.Errorf("the node holding the bus cannot be read: %w", err) - } - if holder != "" { - if err := s.compose(ctx, []string{holder}); err != nil { - return "", fmt.Errorf("the bus's user list on %s is not composed: %w", holder, err) - } - } - if err := s.grants(ctx); err != nil { - return "", err - } - if holder != "" { - return "the bus's user list composed on " + holder + ", the grants written", nil - } - return "the grants written", nil -} - -// assignmentStands says a value of the assignments bucket holds an assignment that stands: made (a module only pulled in -// by another names no time assigned), not ended and in no state (chain.Assignment.Stands). -func assignmentStands(a chain.Assignment) bool { - return !a.Assigned.IsZero() && a.Stands() && a.Node != "" && a.Module != "" -} - -// assignmentTurns keeps which keys of the assignments bucket stood when last heard, to tell an assignment that comes to -// stand from a rewrite of one that stood. -type assignmentTurns struct { - mu sync.Mutex - stood map[string]bool -} - -// heard records a value of key (nil for a delete or a purge) and answers whether it holds an assignment that just came -// to stand. -func (t *assignmentTurns) heard(key string, value []byte) (chain.Assignment, bool) { - var a chain.Assignment - stands := false - if value != nil && json.Unmarshal(value, &a) == nil { - stands = assignmentStands(a) - } - t.mu.Lock() - defer t.mu.Unlock() - if t.stood == nil { - t.stood = map[string]bool{} - } - was := t.stood[key] - t.stood[key] = stands - return a, stands && !was -} - -// ownAccountOf is the username of the account a module speaks on the bus with on node, its own and no process's: the -// module's when it reads a `broker` own secret, else its provisioner client's when it runs one; false when it has none -// (a module whose accounts are its processes' alone, or one that does not speak on the bus). -func ownAccountOf(m catalogue.Manifest, node string) (string, bool) { - if _, reads := m.OwnSecrets["broker"]; reads { - return broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username(), true - } - if runsTheProvisionerClient(m) { - return broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username(), true - } - return "", false -} - -// issueAssignmentAccounts issues module's accounts on node that are not minted: its own (issueOnTheNewBus) and its -// processes' (issueMissingProcessAccounts). A module the catalogue does not hold, or one with nowhere to deliver an -// account (mayIssue), has nothing to issue; an account minted before is kept. -func issueAssignmentAccounts(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest, - node, module string) (bool, string, error) { - // Mint only what no row names: the assign act hears its own write and may issue at once (the review of issue 1945). - ctx = onlyIfAbsent(ctx) - m, known := shelf[module] - if !known || mayIssue(m) != nil { - return false, "", nil - } - if _, own := ownAccountOf(m, node); own { - minted, err := inv.BusMinted(ctx) // sealed accounts alone: a row with no sealed credential is not issued - if err != nil { - return false, "", fmt.Errorf("the issued accounts cannot be read: %w", err) - } - if missingAccounts(m, node, minted) { - busAddress, err := broker.BusAddress() - if err != nil { - return false, "", err - } - // Its processes' and provisioner client's accounts too: issueOnTheNewBus issues each that is not issued. - ctx, got := withIssued(ctx) - if err := issueOnTheNewBus(ctx, inv, m, node, busAddress); err != nil { - return false, "", err - } - if issued := got.list(); len(issued) > 0 { - return true, "issued and sealed to " + node + ": " + strings.Join(issued, ", "), nil - } - return false, "", nil - } - return false, "its bus accounts on " + node + " stood issued and sealed", nil - } - line, err := issueMissingProcessAccounts(ctx, inv, m, node) - if err != nil { - return false, "", err - } - return line != "", line, nil -} - -// missingAccounts says some account of m on node is not issued: its own, a process's, or its provisioner client's. -func missingAccounts(m catalogue.Manifest, node string, minted []string) bool { - if user, own := ownAccountOf(m, node); own && !slices.Contains(minted, user) { - return true - } - if runsTheProvisionerClient(m) && - !slices.Contains(minted, broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()) { - return true - } - return len(processAccountsMissing(m, node, minted)) > 0 -} - -// liveAssignmentAccountSteps are the steps on the controller's own stores and the bus: the composer's passes, as the -// done keys' take-ins use them, so a node asked for by both is composed once a pass. -func liveAssignmentAccountSteps(open *stores) assignmentAccountSteps { - inv := open.inventory - return assignmentAccountSteps{ - holds: func(ctx context.Context) bool { - // The lease's gate, as every act's: a serving controller the bus does not grant the lease yet acts unleased. - _, err := theLease.epoch(ctx) - return err == nil - }, - forgetStale: func(ctx context.Context) ([]string, error) { return forgetStaleAccounts(ctx, open) }, - catalogue: func(ctx context.Context) (map[string]catalogue.Manifest, error) { return inv.Catalogue(ctx) }, - issue: func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (bool, string, error) { - return issueAssignmentAccounts(ctx, inv, shelf, node, module) - }, - busHolder: func(ctx context.Context) (string, error) { - holders, err := seatHolders(ctx, inv) - if err != nil { - return "", err - } - return holders[catalogue.BrokerSeat].Node, nil - }, - compose: func(ctx context.Context, nodes []string) error { - return composerFor(open).compose(ctx, nodes, false) - }, - grants: func(ctx context.Context) error { - return composerFor(open).compose(ctx, nil, true) - }, - } -} - -// assignmentAccountsRetry is how long a settle that failed waits before it tries again. -var assignmentAccountsRetry = 30 * time.Second - -// untilSettled runs settle until it works or ctx ends: a failure, or a controller not holding the lease, is said and -// tried again every assignmentAccountsRetry, never dropped, since no later write of the key may come. -func untilSettled(ctx context.Context, what string, settle func(context.Context) (string, error)) { - for { - said, err := settle(ctx) - if err == nil { - if said != "" { - for _, line := range strings.Split(said, "\n") { - fmt.Printf("assignments: %s\n", line) - } - } - return - } - if ctx.Err() != nil { - return - } - fmt.Printf("assignments: %s: %v; tried again in %s\n", what, err, assignmentAccountsRetry) - select { - case <-ctx.Done(): - return - case <-time.After(assignmentAccountsRetry): - } - } -} - -// watchAssignmentAccounts settles every assignment of the assignments bucket that comes to stand, for as long as ctx: the -// bucket's current values first (the start's look, settled as not fresh), then each write. A watch that ends, or a -// bucket not there yet, is watched again a minute later. -func watchAssignmentAccounts(ctx context.Context, open *stores) { - steps := liveAssignmentAccountSteps(open) - goServing(ctx, func() { - runAssignmentWatches(ctx, open.inventory.WatchAssignments, func(a chain.Assignment) { - inBackground(ctx, func() { - untilSettled(ctx, a.Module+" on "+a.Node, func(ctx context.Context) (string, error) { - return steps.settle(ctx, a.Node, a.Module) - }) - }) - }, func(look []chain.Assignment) { - // The start's look, once in the process: one reading of the catalogue, one assignment after another, - // then one composition of the bus's node and the grants (issues 1945 and 1277). - inBackground(ctx, func() { - untilSettled(ctx, "the start's look", func(ctx context.Context) (string, error) { - return steps.look(ctx, look) - }) - }) - }, func() bool { - select { - case <-ctx.Done(): - return false - case <-time.After(time.Minute): - return true - } - }) - }) -} - -// runAssignmentWatches watches the assignments again each time a watch ends, until ctx ends or pause says stop: each -// assignment that comes to stand is handed to fresh, and the standing ones of the first watch that reads them all (its -// Caught mark) to look, once. A watch that ends, or fails to open, before the start's look ran is no start's look: the -// next one is, with what it hears from nothing (a fresh assignmentTurns), so no key it handed is dropped and no burst of -// fresh settles replaces the look (the review of issue 1945). -func runAssignmentWatches(ctx context.Context, watch func(context.Context) (<-chan chain.Entry, error), - fresh func(chain.Assignment), look func([]chain.Assignment), pause func() bool) { - turns := &assignmentTurns{} - first := true - for { - var atStart []chain.Assignment - if err := watchAssignmentsOnce(ctx, watch, turns, first, func(a chain.Assignment, isFresh bool) { - if !isFresh { - atStart = append(atStart, a) - return - } - fresh(a) - }, func() { - first = false - look(atStart) - atStart = nil - }); err != nil && ctx.Err() == nil { - fmt.Printf("assignments: the assignments are not heard, so a new assignment's bus accounts wait: %v\n", err) - } - if first { - turns = &assignmentTurns{} // the start's look has not run: the next watch hears every standing key anew - } - if ctx.Err() != nil || !pause() { - return - } - } -} - -// watchAssignmentsOnce watches the assignments until the watch or ctx ends, handing each assignment that comes to stand -// to settle. On the process's first watch, the current values are the start's look (not fresh) and read is called once -// they are all handed (the chain's Caught mark); every other value, and every value of a watch opened again after one -// ended, is fresh: an assignment that came to stand while no watch heard it is settled whole, never as the start's look -// (the review of issue 1945). -func watchAssignmentsOnce(ctx context.Context, watch func(context.Context) (<-chan chain.Entry, error), - turns *assignmentTurns, first bool, settle func(a chain.Assignment, fresh bool), read func()) error { - ctx, stop := context.WithCancel(ctx) - defer stop() - entries, err := watch(ctx) - if err != nil { - return err - } - current := true - for { - select { - case <-ctx.Done(): - return nil - case e, ok := <-entries: - if !ok { - if ctx.Err() != nil { - return nil - } - return fmt.Errorf("the watch ended") - } - if e.Caught { - if current { - current = false // the current values are read: what follows is written from now on - if first { - read() - } - } - continue - } - var value []byte - if !e.Deleted { - value = e.Value - } - if a, turned := turns.heard(e.Key, value); turned { - settle(a, !current || !first) - } - } - } -} - -// staleAccounts is, by node, each module some account of which stands on a node where the module no longer resolves: -// the accounts of a module's own, its processes' and its provisioner client's. A node whose resolution is unknown is -// left alone. -func staleAccounts(users map[string]inventory.BusUser, resolved map[string]map[string]bool) map[string][]string { - seen := map[string]map[string]bool{} - for _, u := range users { - switch u.Kind { - case inventory.BusModule, inventory.BusNodeTools, inventory.BusProcess, inventory.BusProvisioner: - default: - continue - } - on, known := resolved[u.Node] - if u.Node == "" || u.Module == "" || !known || on[u.Module] { - continue - } - if seen[u.Node] == nil { - seen[u.Node] = map[string]bool{} - } - seen[u.Node][u.Module] = true - } - out := map[string][]string{} - for node, modules := range seen { - out[node] = slices.Sorted(maps.Keys(modules)) - } - return out -} - -// forgetStaleAccounts forgets the accounts of every module on a node where it no longer resolves (assigned or -// required), from each node's own resolution; a node that does not resolve is left alone. -func forgetStaleAccounts(ctx context.Context, open *stores) ([]string, error) { - inv := open.inventory - users, err := inv.BusUsers(ctx) - if err != nil { - return nil, err - } - ctx = withInputCache(ctx) // one reading of the mesh for every node resolved - resolved := map[string]map[string]bool{} - for _, u := range users { - if _, done := resolved[u.Node]; done || u.Node == "" { - continue - } - plan, _, err := resolveNode(ctx, open, u.Node) - if err != nil { - continue // unknown: nothing of this node is forgotten - } - on := map[string]bool{} - for _, m := range plan.Modules { - on[m.Module] = true - } - resolved[u.Node] = on - } - var lines []string - stale := staleAccounts(users, resolved) - for _, node := range slices.Sorted(maps.Keys(stale)) { - for _, module := range stale[node] { - forgot, err := inv.ForgetBusAccountsOf(ctx, node, module, true) - if err != nil { - return lines, err - } - if len(forgot) > 0 { - lines = append(lines, fmt.Sprintf("%s no longer runs on %s: its bus accounts forgotten: %s", module, node, - strings.Join(forgot, ", "))) - } - } - } - return lines, nil -} diff --git a/cmd/mesh-controller/assignment_accounts_test.go b/cmd/mesh-controller/assignment_accounts_test.go deleted file mode 100644 index 646da68f..00000000 --- a/cmd/mesh-controller/assignment_accounts_test.go +++ /dev/null @@ -1,498 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" -) - -// An assignment that comes to stand issues its node's bus accounts and the module's grant (ADR 0543, issue 1945): on -// 2026-10-25 at 17:19 mesh-build-node and mesh-build-go, built long before, were assigned to ace, and nothing issued -// ace's accounts nor put them in the bus's user list. - -func anAssignmentValue(t *testing.T, a chain.Assignment) []byte { - t.Helper() - raw, err := json.Marshal(a) - if err != nil { - t.Fatal(err) - } - return raw -} - -// Only a write that makes the key stand turns it: a rewrite of a standing assignment (the provisioner's plan kept in its -// value), an ended one, a pending one or a module only pulled in by another does not. The process's first watch hands -// the assignments standing when it starts to the start's look (not fresh), says once that they are all read, and hands -// each assignment written afterwards as fresh; a watch opened again after one ended hands every standing assignment it -// has not seen stand as fresh and never calls the start's look (the review of issue 1945). -func TestAnAssignmentThatComesToStand(t *testing.T) { - at := time.Date(2026, 10, 25, 16, 19, 42, 0, time.UTC) - stands := chain.Assignment{Node: "ace", Module: "mesh-build-node", Assigned: at} - ended := stands - ended.Ended = at.Add(time.Hour) - pending := stands - pending.State = chain.StatePending - pulled := chain.Assignment{Node: "ace", Module: "mesh-build-node", RequiredBy: "mesh-build-go"} - replanned := stands - replanned.Provisioning = json.RawMessage(`{"plan":1}`) - - type write struct { - value *chain.Assignment // nil for a delete - turning bool - } - for _, c := range []struct { - name string - writes []write - }{ - {"a new assignment turns", []write{{&stands, true}}}, - {"a rewrite of one that stands does not", []write{{&stands, true}, {&replanned, false}, {&stands, false}}}, - {"an ended one does not", []write{{&ended, false}}}, - {"assigned again after it ended turns again", []write{{&stands, true}, {&ended, false}, {&stands, true}}}, - {"a pending one does not, and turns once made", []write{{&pending, false}, {&stands, true}}}, - {"a module only pulled in does not", []write{{&pulled, false}}}, - {"one deleted and written again turns again", []write{{&stands, true}, {nil, false}, {&stands, true}}}, - } { - t.Run(c.name, func(t *testing.T) { - turns := &assignmentTurns{} - for i, w := range c.writes { - var value []byte - if w.value != nil { - value = anAssignmentValue(t, *w.value) - } - a, turned := turns.heard("ace.mesh-build-node", value) - if turned != w.turning { - t.Fatalf("write %d: turned %v, want %v", i, turned, w.turning) - } - if turned && (a.Node != "ace" || a.Module != "mesh-build-node") { - t.Fatalf("write %d: the assignment turned is %s on %s", i, a.Module, a.Node) - } - } - }) - } - - type handed struct { - key string - fresh bool - } - for _, first := range []bool{true, false} { - t.Run(fmt.Sprintf("the watch, first %v", first), func(t *testing.T) { - bucket := chain.NewMemory() - put := func(a chain.Assignment) { - t.Helper() - value := anAssignmentValue(t, a) - if e, held, _ := bucket.Get(t.Context(), a.Key()); held { - if _, err := bucket.Update(t.Context(), a.Key(), value, e.Revision); err != nil { - t.Fatal(err) - } - return - } - if _, err := bucket.Create(t.Context(), a.Key(), value); err != nil { - t.Fatal(err) - } - } - put(chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at}) - got := make(chan handed, 8) - read := make(chan struct{}, 8) - ctx, stop := context.WithCancel(t.Context()) - done := make(chan error, 1) - go func() { - done <- watchAssignmentsOnce(ctx, bucket.Watch, &assignmentTurns{}, first, - func(a chain.Assignment, fresh bool) { got <- handed{a.Key(), fresh} }, - func() { read <- struct{}{} }) - }() - next := func() handed { - t.Helper() - select { - case h := <-got: - return h - case <-time.After(5 * time.Second): - t.Fatal("nothing handed") - return handed{} - } - } - // Standing when the watch opened: the start's look on the first watch, fresh on any other. - if h := next(); h != (handed{"g14.mesh-build-go", !first}) { - t.Fatalf("the standing assignment was handed %+v", h) - } - put(chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at, Provisioning: json.RawMessage(`{"plan":2}`)}) - put(chain.Assignment{Node: "ace", Module: "mesh-build-node", Assigned: at}) - if h := next(); h != (handed{"ace.mesh-build-node", true}) { - t.Fatalf("handed %+v, want ace.mesh-build-node as fresh (and nothing for the rewrite before it)", h) - } - stop() - if err := <-done; err != nil { - t.Fatal(err) - } - if want := map[bool]int{true: 1, false: 0}[first]; len(read) != want { - t.Fatalf("the start's look was called %d time(s), want %d", len(read), want) - } - }) - } - // A first watch that fails to open, or ends before its Caught mark, is no start's look: the next watch is, and hands - // every standing assignment to it once, none as fresh (the review of issue 1945: a burst of fresh settles, or keys - // handed and dropped). - for _, firstWatch := range []string{"fails to open", "ends before its Caught mark"} { - t.Run("a first watch that "+firstWatch, func(t *testing.T) { - bucket := chain.NewMemory() - standing := chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at} - if _, err := bucket.Create(t.Context(), standing.Key(), anAssignmentValue(t, standing)); err != nil { - t.Fatal(err) - } - watches := 0 - watch := func(ctx context.Context) (<-chan chain.Entry, error) { - watches++ - if watches > 1 { - return bucket.Watch(ctx) - } - if firstWatch == "fails to open" { - return nil, errors.New("the chain is not open yet") - } - cut := make(chan chain.Entry, 1) - cut <- chain.Entry{Key: standing.Key(), Value: anAssignmentValue(t, standing)} - close(cut) - return cut, nil - } - var looks [][]chain.Assignment - var fresh []chain.Assignment - ctx, stop := context.WithCancel(t.Context()) - defer stop() - runAssignmentWatches(ctx, watch, func(a chain.Assignment) { fresh = append(fresh, a) }, - func(look []chain.Assignment) { looks = append(looks, look); stop() }, - func() bool { return ctx.Err() == nil }) - if len(looks) != 1 || len(looks[0]) != 1 || looks[0][0].Key() != standing.Key() || len(fresh) != 0 { - t.Fatalf("after %d watch(es): looks %v, fresh %v; want one look at %s and nothing fresh", - watches, looks, fresh, standing.Key()) - } - }) - } -} - -// settle issues what is missing and then puts it on the bus: the node holding the bus composed (its user list) and the -// grants written, even when every account was minted before (a module assigned again keeps its account). The start's -// look reads the catalogue once, issues for each standing assignment, and composes the bus's node and writes the grants -// once when it looked at any. Neither acts for a controller that does not hold the lease, nor for the controller's own -// module. -func TestSettlingAndTheStartsLook(t *testing.T) { - ace := func(m string) chain.Assignment { return chain.Assignment{Node: "ace", Module: m} } - cases := []struct { - name string - look []chain.Assignment // nil: settle mesh-build-node on ace - module string - notHolding bool - issued bool - issueErr, composeE error - holder string - wantIssues int - wantComposed []string - wantGrants bool - wantErr string - stale []string // what forgetting the accounts of modules no longer on their nodes says - }{ - {name: "an account not minted", module: "mesh-build-node", issued: true, holder: "novox", wantIssues: 1, - wantComposed: []string{"novox"}, wantGrants: true}, - {name: "an account minted before: the bus and the grants still", module: "mesh-build-go", holder: "novox", - wantIssues: 1, wantComposed: []string{"novox"}, wantGrants: true}, - {name: "no lease: nothing, and tried again", module: "mesh-build-node", notHolding: true, issued: true, - holder: "novox", wantErr: errNotHolding.Error()}, - {name: "the controller's own module is the terminal's", module: controllerModule, issued: true, holder: "novox"}, - {name: "no bus holder: the grants still", module: "mesh-build-node", issued: true, wantIssues: 1, wantGrants: true}, - {name: "an issue that fails is the error, nothing composed", module: "mesh-build-node", - issueErr: errors.New("the bus cannot be reached"), holder: "novox", wantIssues: 1, wantErr: "the bus cannot be reached"}, - {name: "a composition that fails is the error, no grant", module: "mesh-build-node", issued: true, holder: "novox", - composeE: errors.New("store refused"), wantIssues: 1, wantComposed: []string{"novox"}, wantErr: "store refused"}, - {name: "the start's look: each issued, the bus and the grants once", - look: []chain.Assignment{ace("mesh-build-node"), ace(controllerModule), ace("mesh-build-go")}, issued: true, - holder: "novox", wantIssues: 2, wantComposed: []string{"novox"}, wantGrants: true}, - {name: "the start's look with every account minted still covers once", - look: []chain.Assignment{ace("mesh-build-node"), ace("mesh-build-go")}, holder: "novox", wantIssues: 2, - wantComposed: []string{"novox"}, wantGrants: true}, - {name: "the start's look at nothing does nothing", look: []chain.Assignment{ace(controllerModule)}, holder: "novox"}, - {name: "the start's look forgets the accounts of modules no longer on their nodes first", - look: []chain.Assignment{ace("mesh-build-node")}, holder: "novox", wantIssues: 1, wantComposed: []string{"novox"}, - wantGrants: true, stale: []string{"mesh-build-go no longer runs on ace: its bus accounts forgotten: ace.mesh-build-go"}}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - var composed []string - issues, catalogues, grants := 0, 0, 0 - forgets := 0 - steps := assignmentAccountSteps{ - holds: func(context.Context) bool { return !c.notHolding }, - forgetStale: func(context.Context) ([]string, error) { - forgets++ - return c.stale, nil - }, - catalogue: func(context.Context) (map[string]catalogue.Manifest, error) { - catalogues++ - return map[string]catalogue.Manifest{}, nil - }, - issue: func(context.Context, map[string]catalogue.Manifest, string, string) (bool, string, error) { - issues++ - if c.issued { - return true, "its bus credential issued", c.issueErr - } - return false, "", c.issueErr - }, - busHolder: func(context.Context) (string, error) { return c.holder, nil }, - compose: func(_ context.Context, nodes []string) error { - composed = append(composed, nodes...) - return c.composeE - }, - grants: func(context.Context) error { grants++; return nil }, - } - var said string - var err error - if c.look != nil { - said, err = steps.look(t.Context(), c.look) - } else { - said, err = steps.settle(t.Context(), "ace", c.module) - } - if c.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), c.wantErr) { - t.Fatalf("error %v, want one saying %q", err, c.wantErr) - } - } else if err != nil { - t.Fatal(err) - } - if issues != c.wantIssues || catalogues > 1 { - t.Fatalf("%d issue(s) from %d reading(s) of the catalogue, want %d from at most one", issues, catalogues, c.wantIssues) - } - if strings.Join(composed, ",") != strings.Join(c.wantComposed, ",") { - t.Fatalf("composed %v, want %v", composed, c.wantComposed) - } - if want := map[bool]int{true: 1}[c.wantGrants]; grants != want { - t.Fatalf("grants written %d time(s), want %d", grants, want) - } - if want := map[bool]int{true: 1}[len(c.look) > 0 && !c.notHolding]; forgets != want { - t.Fatalf("forgot stale accounts %d time(s), want %d (only from a reading that holds assignments)", forgets, want) - } - for _, line := range c.stale { - if !strings.Contains(said, line) { - t.Fatalf("said %q, want it to say %q", said, line) - } - } - if c.wantGrants && c.wantErr == "" && !strings.Contains(said, "the grants written") { - t.Fatalf("said %q, want it to name the grants", said) - } - }) - } -} - -// The accounts a module speaks with on a node: the module's own when it reads `broker`, its provisioner client's when -// it runs one alone, none when its accounts are its processes' or it does not speak on the bus. A process's account is -// held in the provisioner client's form, named for the process and carrying no claims, and may be issued only when -// the process declares its own secret broker- (ADR 0436 §2; issue 1181). -func TestTheAccountsAModuleSpeaksWithOnANode(t *testing.T) { - client := aProvisioner() - delete(client.OwnSecrets, "broker") - for _, c := range []struct { - name string - m catalogue.Manifest - user string - own bool - }{ - {"a module reading broker", catalogue.Manifest{Module: "mesh-build-node", - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh-build-node/broker"}}}, - broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-build-node"}.Username(), true}, - {"a module reading broker and running the client", aProvisioner(), - broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-provisioner"}.Username(), true}, - {"the provisioner client alone", client, - broker.Principal{Kind: broker.KindProvisioner, Node: "ace", Module: "mesh-provisioner"}.Username(), true}, - {"no broker, no client", catalogue.Manifest{Module: "hello-web"}, "", false}, - } { - t.Run(c.name, func(t *testing.T) { - user, own := ownAccountOf(c.m, "ace") - if user != c.user || own != c.own { - t.Fatalf("got %q %v, want %q %v", user, own, c.user, c.own) - } - }) - } - - t.Run("a process's account", func(t *testing.T) { - user := broker.Principal{Kind: broker.KindProcess, Node: "novox", Module: "mesh-mail", Process: "send"}.Username() - if user != "novox.mesh-mail.send" { - t.Fatalf("the process's username: %q", user) - } - held, err := heldProcessAccount(broker.Broker{Fingerprint: "abc"}, "novox.example:4222", "novox", "mesh-mail", - user, "a-password") - if err != nil { - t.Fatal(err) - } - var got map[string]any - if err := json.Unmarshal(held, &got); err != nil { - t.Fatal(err) - } - want := map[string]any{"url": "nats://novox.example:4222", "fingerprint": "abc", "node": "novox", - "module": "mesh-mail", "user": "novox.mesh-mail.send", "password": "a-password"} - for k, v := range want { - if got[k] != v { - t.Errorf("%s: %v, want %v", k, got[k], v) - } - } - if len(got) != len(want) { - t.Errorf("the held account carries %v; want exactly %v (never claims — a process holds no seat)", got, want) - } - }) - - // An issue mints and seals an account only when it is not issued: no row, or a row with no sealed credential (issue - // 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go had a row from 17:19 and no credential), under the lock on - // the account; a seal that fails leaves it unsealed for the next issue. A rotation always mints. - for _, c := range []struct { - name string - only bool - minted, sealed bool - sealErr error - want string - }{ - {"no row: minted unsealed, sealed, marked", true, false, false, nil, "lock mint-unsealed seal mark release"}, - {"a row sealed: nothing", true, true, true, nil, "lock release"}, - {"a row with no sealed credential: minted and sealed again", true, true, false, nil, "lock mint-unsealed seal mark release"}, - {"a seal that fails: not marked", true, false, false, errors.New("the vault did not take it"), "lock mint-unsealed seal release"}, - {"a rotation mints whatever stands", false, true, true, nil, "mint seal"}, - } { - t.Run("issuing an account: "+c.name, func(t *testing.T) { - acts := &anAccountStore{minted: c.minted, sealed: c.sealed} - ctx, got := withIssued(t.Context()) - if c.only { - ctx = onlyIfAbsent(ctx) - } - err := issueAccount(ctx, acts, inventory.BusUser{Username: "ace.mesh-build-go", Kind: inventory.BusModule}, - func(password string) error { - acts.did("seal") - if password == "" { - t.Error("sealed no password") - } - return c.sealErr - }, func() error { acts.did("after"); return nil }) - if (err != nil) != (c.sealErr != nil) { - t.Fatalf("error %v, want %v", err, c.sealErr) - } - if strings.Join(acts.acts, " ") != c.want { - t.Fatalf("acts %q, want %q", strings.Join(acts.acts, " "), c.want) - } - if issued := len(got.list()) == 1; issued != (c.sealErr == nil && strings.Contains(c.want, "seal")) { - t.Fatalf("said issued %v", got.list()) - } - }) - } - - // Issue 774: where a vault is installed the controller mints nothing; `module issue` and an assignment's issue are - // forwarded to the vault's issue-account, and what the account needs beside its value still runs. A refusal fails - // `module issue` and only says itself on an assignment's issue: the vault mints on the want when it hears it. - for _, c := range []struct { - name string - only bool - answer string - refuse error - want string - fails bool - }{ - {"module issue: forwarded, minted by the vault", false, `{"minted":true,"account":{"secret":"own/ace/mesh-build-go/broker","generation":1}}`, nil, "after", false}, - {"an assignment's issue: forwarded, held already", true, `{"minted":false,"why":"held already"}`, nil, "after", false}, - {"an assignment's issue: refused, said", true, "", errors.New("nobody wants it yet"), "after", false}, - {"module issue: refused, failed", false, "", errors.New("nobody wants it"), "", true}, - } { - t.Run("forwarded to the vault: "+c.name, func(t *testing.T) { - was := vaultIssuer - t.Cleanup(func() { vaultIssuer = was }) - var asked []string - vaultIssuer = func(context.Context, accountIssuer) (vaultAsker, bool, error) { - return func(verb string, args map[string]any) (json.RawMessage, error) { - asked = append(asked, verb+" "+fmt.Sprint(args["user"])) - return json.RawMessage(c.answer), c.refuse - }, true, nil - } - acts := &anAccountStore{} - ctx, got := withIssued(t.Context()) - if c.only { - ctx = onlyIfAbsent(ctx) - } - err := issueAccount(ctx, acts, inventory.BusUser{Username: "ace.mesh-build-go", Kind: inventory.BusModule}, - func(string) error { acts.did("seal"); return nil }, func() error { acts.did("after"); return nil }) - if (err != nil) != c.fails || strings.Join(acts.acts, " ") != c.want { - t.Fatalf("error %v, acts %q; want failed %v, acts %q (nothing minted, locked or sealed here)", err, - strings.Join(acts.acts, " "), c.fails, c.want) - } - if strings.Join(asked, ",") != "issue-account ace.mesh-build-go" { - t.Errorf("the vault was asked %v", asked) - } - if minted := len(got.list()) == 1; minted != strings.Contains(c.answer, `"minted":true`) { - t.Errorf("said issued %v", got.list()) - } - }) - } - - // The accounts of a module that no longer resolves on its node are stale, its provisioner client's among them; a - // module still resolved (assigned or required), an account of another kind, and a node whose resolution is unknown - // are left alone (issue 1945: ace's mesh-build-go rows from 17:19, marked sealed by migration 0108). - t.Run("stale accounts", func(t *testing.T) { - u := func(kind, node, module string) inventory.BusUser { - return inventory.BusUser{Username: node + "." + module + "." + kind, Kind: kind, Node: node, Module: module} - } - users := map[string]inventory.BusUser{} - for _, b := range []inventory.BusUser{u(inventory.BusModule, "ace", "mesh-build-go"), - u(inventory.BusProcess, "ace", "mesh-build-go"), u(inventory.BusProvisioner, "ace", "mesh-provisioner"), - u(inventory.BusModule, "ace", "mesh-build-node"), u(inventory.BusNode, "ace", ""), - u(inventory.BusModule, "g14", "mesh-build-go")} { - users[b.Username] = b - } - got := staleAccounts(users, map[string]map[string]bool{"ace": {"mesh-build-node": true}}) - if want := map[string][]string{"ace": {"mesh-build-go", "mesh-provisioner"}}; !reflect.DeepEqual(got, want) { - t.Fatalf("stale %v, want %v", got, want) - } - }) - - t.Run("a process may be issued only with its own broker secret", func(t *testing.T) { - // mesh-mail's real shape: no `broker` of its own, its process declaring bus (review round 1 of issue 1181). - if err := mayIssue(mailSenderWithProcess()); err != nil { - t.Fatalf("a module with no broker of its own, whose process declares bus, was refused: %v", err) - } - m := mailSenderWithProcess() - delete(m.OwnSecrets, "broker-send") - err := mayIssue(m) - if err == nil { - t.Fatal("a process with no broker- own secret was issued an account") - } - for _, want := range []string{"mesh-mail", "send", "broker-send", "own-secrets"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not say %q: %v", want, err) - } - } - }) -} - -// anAccountStore is an accountIssuer that writes down what was asked of it. -type anAccountStore struct { - minted, sealed bool - acts []string -} - -func (a *anAccountStore) did(act string) { a.acts = append(a.acts, act) } -func (a *anAccountStore) LockBusAccount(context.Context, string) (func(), error) { - a.did("lock") - return func() { a.did("release") }, nil -} -func (a *anAccountStore) BusAccountIssued(context.Context, string) (bool, bool, error) { - return a.minted, a.sealed, nil -} -func (a *anAccountStore) MintBusPassword(context.Context, inventory.BusUser) (string, error) { - a.did("mint") - return "a-password", nil -} -func (a *anAccountStore) MintBusPasswordUnsealed(context.Context, inventory.BusUser) (string, error) { - a.did("mint-unsealed") - return "a-password", nil -} -func (a *anAccountStore) MarkBusAccountSealed(context.Context, string) error { - a.did("mark") - return nil -} diff --git a/cmd/mesh-controller/bindings.go b/cmd/mesh-controller/bindings.go deleted file mode 100644 index 64563d50..00000000 --- a/cmd/mesh-controller/bindings.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" -) - -// A binding to data moves only by a person (novox/hq ADR 0232, issue 273). -// -// The resolver keeps each consumer of a provision that keeps its data at the provider it was last -// sent (catalogue/bound.go) and says what it would have moved. This is where that is said: on the -// push that composed it, and by the self-check's D12 every run, as an urgent condition naming the -// consumer, both providers and the pin that confirms the move. - -// The condition kinds of D12. -const ( - // kindBindingKept is a move the resolver refused: the consumer is still where its data is. - kindBindingKept = "binding-kept" - // kindBindingMoved is a consumer about to be sent another provider than the one on record with - // no pin naming it — what the resolver exists to make impossible, said if it ever is not. - kindBindingMoved = "binding-moved" - // kindBindingMoving is a move a pin asked for, not yet sent: a person's act, said so that the - // data is moved before the push that carries it. - kindBindingMoving = "binding-moving" -) - -// probeBindingsID is the self-check's id for this probe, and the source of what it raises. -const probeBindingsID = "D12" - -// keptObservation is the urgent condition for one refused move. -func keptObservation(k catalogue.KeptBinding) conditions.Observation { - return conditions.Observation{Scope: conditions.ScopeMachine, ID: bindingID(k.Machine, k.Consumer, k.Provision), - Token: kindBindingKept, Kind: kindBindingKept, Machine: k.Machine, Also: otherMachines(k.Machine, k.Bound.Node, k.Would.Node), - Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, - Summary: fmt.Sprintf("on %s, the mesh %s", k.Machine, k.String())} -} - -func bindingID(machine, consumer, provision string) string { - return machine + "." + consumer + "." + provision -} - -func otherMachines(machine string, nodes ...string) []string { - var out []string - seen := map[string]bool{machine: true} - for _, n := range nodes { - if n != "" && !seen[n] { - seen[n] = true - out = append(out, n) - } - } - return out -} - -// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last -// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin -// moves it (said, so the data goes first), and never anything else. -func probeBindings(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - // One reading of the mesh for every machine this looks at. - ctx = withInputCache(ctx) - inv := d.open.inventory - nodes, err := inv.Nodes(ctx) - if err != nil { - return nil, err - } - var out []conditions.Observation - for _, n := range nodes { - plan, _, err := resolveNode(ctx, d.open, n.Name) - if err != nil { - if unresolvable(err) { - // D1 says it, with the binding that refused it when that is why. - continue - } - return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err) - } - bound, err := inv.BindingsFor(ctx, n.Name) - if err != nil { - return nil, err - } - pins, err := inv.PinsFor(ctx, n.Name) - if err != nil { - return nil, err - } - out = append(out, bindingFindings(plan, bound, pins)...) - } - return out, nil -} - -// bindingFindings is what one machine's resolution says against its record. -func bindingFindings(plan catalogue.Resolution, bound map[string]map[string]catalogue.Chosen, - pins map[string]catalogue.Chosen) []conditions.Observation { - var out []conditions.Observation - for _, k := range plan.Kept { - out = append(out, keptObservation(k)) - } - said := map[string]bool{} - for _, need := range plan.Needs { - if !need.KeepsData || need.ByRecord { - continue - } - was, recorded := bound[need.For][need.Name] - now := catalogue.Chosen{Node: need.From, Module: need.Module} - if !recorded || was == now || (was.Module == "" && was.Node == now.Node) { - continue - } - id := bindingID(plan.Node, need.For, need.Name) - if said[id] { - continue - } - said[id] = true - o := conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Machine: plan.Node, - Also: otherMachines(plan.Node, was.Node, now.Node), Resolver: conditions.ResolverOperator} - if pin, pinned := pins[need.Name]; pinned && pin.Node == now.Node && (pin.Module == "" || pin.Module == now.Module) { - o.Token, o.Kind, o.Severity = kindBindingMoving, kindBindingMoving, conditions.Warning - o.Summary = fmt.Sprintf("on %s, %s's %s moves from %s to %s at the next push, by the pin — its data "+ - "is on %s: move it first", plan.Node, need.For, need.Name, was, now, was) - } else { - o.Token, o.Kind, o.Severity = kindBindingMoved, kindBindingMoved, conditions.Urgent - o.Summary = fmt.Sprintf("on %s, %s's %s would be sent %s, and it is bound to %s, where its data is, "+ - "with no pin naming %s — a move nothing asked for", plan.Node, need.For, need.Name, now, was, now) - } - out = append(out, o) - } - return out -} diff --git a/cmd/mesh-controller/bindings_integration_test.go b/cmd/mesh-controller/bindings_integration_test.go deleted file mode 100644 index da08a10a..00000000 --- a/cmd/mesh-controller/bindings_integration_test.go +++ /dev/null @@ -1,102 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" -) - -func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - for _, m := range storeManifests() { - register(t, open, m) - } - assignAll := func(pairs ...[2]string) { - for _, a := range pairs { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - } - // The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each. - assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) - for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { - if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { - t.Fatal(err) - } - } - assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, - [2]string{"laptop", "board"}) - - plan, _, err := resolveNode(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData { - t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData) - } - if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData { - t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData) - } - - // Sent, and recorded: only the binding to data. - bindings := boundToData(plan, nil) - if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) { - t.Fatalf("recorded %+v", bindings) - } - if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil { - t.Fatal(err) - } - if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { - t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err) - } - - // The laptop's store taken away: refused, not moved to the anchor's empty one. - if err := inv.Unassign(ctx, "laptop", "store"); err != nil { - t.Fatal(err) - } - if _, _, err := resolveNode(ctx, open, "laptop"); err == nil || !unresolvable(err) || - !strings.Contains(err.Error(), "board on laptop is bound to laptop/store") || - !strings.Contains(err.Error(), "pin laptop postgres-database anchor store") { - t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err) - } - - // A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was. - if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { - t.Fatal(err) - } - plan, _, err = resolveNode(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { - t.Fatalf("pinned to the anchor and bound to %s", n.From) - } - found, err := probeBindings(ctx, &doctor{open: open}) - if err != nil { - t.Fatal(err) - } - if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning || - !strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") { - t.Fatalf("a pinned move is not said before it is sent: %+v", found) - } - if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { - t.Fatal(err) - } - all, err := inv.Bindings(ctx) - if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" { - t.Fatalf("%+v, %v", all, err) - } - if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { - t.Fatalf("a move sent is still said: %+v, %v", found, err) - } -} diff --git a/cmd/mesh-controller/bindings_test.go b/cmd/mesh-controller/bindings_test.go deleted file mode 100644 index 8cd06d61..00000000 --- a/cmd/mesh-controller/bindings_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" -) - -// novox/hq issue 273, ADR 0232: a consumer of a provision that keeps its data moves only by a pin. - -func storeManifests() []catalogue.Manifest { - return []catalogue.Manifest{ - {Module: "store", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}, - Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}}, - {Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}, - {Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}}, - {Module: "board", Version: "1", Requires: []string{"postgres-database"}}, - } -} - -func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) catalogue.Needed { - t.Helper() - for _, n := range plan.Needs { - if n.For == consumer && n.Name == provision { - return n - } - } - t.Fatalf("no %s for %s: %+v", provision, consumer, plan.Needs) - return catalogue.Needed{} -} - -// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's -// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the -// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it. - -// What one machine's resolution says against its record. -func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) { - home, anchor := catalogue.Chosen{Node: "home", Module: "store"}, catalogue.Chosen{Node: "anchor", Module: "store"} - plan := catalogue.Resolution{Node: "laptop", - Kept: []catalogue.KeptBinding{{Machine: "laptop", Consumer: "board", Provision: "postgres-database", - Bound: home, Would: anchor}}, - Needs: []catalogue.Needed{ - {Name: "postgres-database", For: "board", From: "home", Module: "store", KeepsData: true}, - {Name: "postgres-database", For: "game", From: "anchor", Module: "store", KeepsData: true}, - {Name: "wildcard-resolution", For: "network", From: "anchor", Module: "resolver"}, - }} - bound := map[string]map[string]catalogue.Chosen{ - "board": {"postgres-database": home}, - "game": {"postgres-database": home}, - "network": {"wildcard-resolution": {Node: "home", Module: "resolver"}}, - } - found := linted(bindingFindings(plan, bound, nil)) - if len(found) != 2 { - t.Fatalf("found %+v", found) - } - kept, moved := found[0], found[1] - if kept.Kind != kindBindingKept || kept.Severity != conditions.Urgent || kept.Machine != "laptop" || - !strings.Contains(kept.Summary, "would move board's postgres-database from home/store to anchor/store") || - !strings.Contains(kept.Summary, "its data is on home/store") || - !strings.Contains(kept.Summary, "`pin laptop postgres-database anchor store` to confirm a move (and move the data first)") { - t.Errorf("kept: %+v", kept) - } - if moved.Kind != kindBindingMoved || moved.Severity != conditions.Urgent || - !strings.Contains(moved.Summary, "game's postgres-database would be sent anchor/store") { - t.Errorf("moved: %+v", moved) - } - if kept.Key() == moved.Key() { - t.Error("two consumers, one condition") - } -} diff --git a/cmd/mesh-controller/board.go b/cmd/mesh-controller/board.go deleted file mode 100644 index 3557b8dc..00000000 --- a/cmd/mesh-controller/board.go +++ /dev/null @@ -1,324 +0,0 @@ -package main - -import ( - "context" - "errors" - "flag" - "fmt" - "html/template" - "net/http" - "sort" - "strings" - "time" -) - -// boardCommand serves the three questions as a page. -// -// **It reads through the same functions everything else does and holds nothing** -// (novox/hq 03-DESIGN/01-to-be/11-a-board.md). The board being replaced is one service that reads -// every context's database directly — [ADR 0008](novox/hq) violated by the one component with a -// reason to violate it, and the cost is that a boundary nothing may cross can move, while one -// thing crossing it is enough to freeze it. A board that reads the provisioning tables is a board -// that breaks when provisioning changes its tables, and the change then gets weighed against the -// board. -// -// **It stores nothing of its own.** No cache that can disagree, no table of what the mesh looked -// like last time. Every request reads the mesh now; if that is slow, the answer belongs in the -// context that owns it, where everything else asking gets it too. -// -// **Reading is the whole of it.** Every action a board could offer already exists as a command, -// and a button that does something no command does is a second implementation of a decision. -func boardCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("board", flag.ContinueOnError) - // The private network, not everything. A board says which machines are broken and what they - // are running, which is exactly the map somebody attacking this would like — and there is no - // reason for it to be reachable from further away than the mesh. - listen := set.String("listen", "127.0.0.1:8080", "where to serve it") - if _, err := parseAround(set, args); err != nil { - return err - } - - server := &http.Server{ - Addr: *listen, - ReadHeaderTimeout: 10 * time.Second, - Handler: board(), - } - fmt.Printf("the board is on http://%s\n", *listen) - fmt.Printf(" it reads the mesh on every request and keeps nothing\n") - - go func() { - <-ctx.Done() - closing, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _ = server.Shutdown(closing) - }() - if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - return err - } - return nil -} - -// board is the handler, separate so a test can drive it without a listener. -func board() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.NotFound(w, r) - return - } - asked, err := ask(r.Context()) - if err != nil { - // **Said, not blank.** A board that cannot reach the mesh and renders an empty page - // says "nothing is wrong" in the one situation where nobody can know that. - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.WriteHeader(http.StatusServiceUnavailable) - _ = page.Execute(w, view{Unreachable: err.Error()}) - return - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - if err := page.Execute(w, asked); err != nil { - // The page is half-written by now; there is nothing useful left to say to the - // browser, and saying it here is what stops the failure being silent. - fmt.Printf("the board could not render: %v\n", err) - } - }) - // The same answers for something that is not a person, from the same read. A board and a - // script disagreeing about which machine is broken would be worse than either alone. - mux.HandleFunc("/mesh.json", func(w http.ResponseWriter, r *http.Request) { - open, err := openStores(r.Context()) - if err != nil { - http.Error(w, err.Error(), http.StatusServiceUnavailable) - return - } - defer open.Close() - asked, err := theThreeQuestions(r.Context(), open) - if err != nil { - http.Error(w, err.Error(), http.StatusServiceUnavailable) - return - } - body, err := statusAsJSON(asked) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(append(body, '\n')) - }) - return mux -} - -// ask reads the mesh for one request. -func ask(ctx context.Context) (view, error) { - open, err := openStores(ctx) - if err != nil { - return view{}, err - } - defer open.Close() - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - return view{}, err - } - return viewOf(asked), nil -} - -// view is what the page is given. Nothing is derived here that the reader could not derive. -type view struct { - Unreachable string - Machines int - Broken []brokenMachine - Quiet []quietMachine - Behind []staleModule - Waiting []waitingMachine - // Unresolved is every machine that cannot be worked out at all. Shown above everything else, - // because a machine here is in none of the other lists: nothing was computed for it, so it is - // not broken, not quiet and not behind — and a page without this said "all well" about a mesh - // where nothing could be sent anywhere. - Unresolved []blockedMachine - // Network is why the private network could not be computed, when it could not. - Network string - // Adopted is every node still adopted (novox/hq ADR 0100). Not broken: nothing forces the - // flip, so a node left adopted is shown rather than read as converged. - Adopted []string - At string -} - -type blockedMachine struct { - Node string - // Said is the mesh's own words, a line at a time. Every unmet requirement, not the first: - // a machine is usually blocked by more than one and fixing one of them changes nothing. - Said []string -} - -type waitingMachine struct { - Node string - // Never told is not out of date: nobody has ever asked this machine to be anything. Same - // remedy, different situation, and the page says which. - Never bool -} - -type brokenMachine struct { - Node string - // Outcome is refused or failed, and stays distinct all the way to the page. **Refused means - // the machine is exactly as it was and what is wrong is in what was sent; failed means it is - // in a state nobody declared and what is wrong is on the machine.** They are fixed in - // different places, so one word for both would send half the readers to the wrong one. - Outcome string - Said []string - When string -} - -type quietMachine struct { - Node string - // Heard is "never" or how long ago. Never heard from is not the same as quiet for a while: - // one may be a machine that was never sent anything. - Heard string -} - -type staleModule struct { - Module string - Holds string - Source string - Running []string -} - -func viewOf(asked answers) view { - out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"), - Network: asked.network, Adopted: adoptedNodes(asked.nodes)} - var blocked []string - for name := range asked.refused { - blocked = append(blocked, name) - } - sort.Strings(blocked) - for _, name := range blocked { - one := blockedMachine{Node: name} - for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") { - one.Said = append(one.Said, strings.TrimSpace(line)) - } - out.Unresolved = append(out.Unresolved, one) - } - for _, d := range asked.wrong { - one := brokenMachine{Node: d.Node, Outcome: d.Outcome, - When: d.At.Local().Format("2006-01-02 15:04")} - if d.Refused != "" { - // The host's own words. It says exactly what it could not accept, and nothing - // written here would say it better. - one.Said = append(one.Said, firstLine(d.Refused)) - } - for _, f := range d.Failed { - one.Said = append(one.Said, f.ID+": "+firstLine(f.Error)) - } - out.Broken = append(out.Broken, one) - } - for _, n := range asked.quiet { - out.Quiet = append(out.Quiet, quietMachine{Node: n.Name, Heard: heardFrom(n)}) - } - for _, m := range asked.waiting { - out.Waiting = append(out.Waiting, waitingMachine{Node: m.Node, Never: m.Never}) - } - for module, on := range asked.behind { - from := asked.sources[module] - out.Behind = append(out.Behind, staleModule{ - Module: module, Holds: short(from.BuiltFrom), Source: short(from.Head), Running: on, - }) - } - return out -} - -// The page. Deliberately one file with no assets: a board that cannot render without fetching -// something is a board that is blank exactly when the mesh is unwell. -var page = template.Must(template.New("board").Parse(` - - -the mesh - -{{if .Unreachable}} -

the mesh cannot be read

-

{{.Unreachable}}

-

This says nothing about whether the mesh is well — only that this page could not -find out.

-{{else}} -

{{.Machines}} machine{{if ne .Machines 1}}s{{end}}

- -{{if .Unresolved}} -

Can everything be worked out?

-
    - {{range .Unresolved}} -
  • blocked {{.Node}} - {{range .Said}}
    {{.}}
    {{end}} -
    • - {{end}} -
-

Nothing can be sent to a machine here, and it appears in none of the lists below: -nothing was computed for it, so there is nothing it can be behind.

-{{end}} -{{if .Network}} -

The private network could not be computed: {{.Network}}

-{{end}} - -

Is anything broken?

-{{if .Broken}} -
    - {{range .Broken}} -
  • - {{.Outcome}} - {{.Node}} {{.When}} - {{range .Said}}
    {{.}}
    {{end}} -
    • - {{end}} -
-{{else}}

No. Every machine is doing what it was told.

{{end}} - -

Is anything not answering?

-{{if .Quiet}} -
    {{range .Quiet}}
  • {{.Node}} {{.Heard}}
    • {{end}}
-

Not heard from is not the same as tried and could not — a machine here may be -new, switched off, or unreachable.

-{{else}}

No. Every machine has been heard from.

{{end}} - -

Is anything out of date?

-{{if .Behind}} -
    - {{range .Behind}} -
  • {{.Module}} holds {{.Holds}}, source has {{.Source}} - {{if .Running}}
    running on {{range $i, $n := .Running}}{{if $i}}, {{end}}{{$n}}{{end}}
    - {{else}}
    assigned to nothing
    {{end}} -
    • - {{end}} -
-{{else}}

No. Every module is what its source last had.

{{end}} - -{{if .Waiting}} -
    - {{range .Waiting}} -
  • {{.Node}} - {{if .Never}}has never been sent anything - {{else}}is not running what the mesh would send it{{end}} -
    • - {{end}} -
-

Never told is not out of date: nobody has asked that machine to be anything yet. -Both are sent by push --behind.

-{{else}}

Every machine is running what the mesh would send it.

{{end}} -{{if .Adopted}} -

Which machines are adopted?

-
    {{range .Adopted}}
  • {{.}} adopted — what was found on it is kept until each module is taken
    • {{end}}
-{{end}} -{{end}} - -
Read at {{.At}}. This page holds nothing and changes nothing.
-`)) diff --git a/cmd/mesh-controller/board_test.go b/cmd/mesh-controller/board_test.go deleted file mode 100644 index ea8029dd..00000000 --- a/cmd/mesh-controller/board_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// Refused and failed stay distinct all the way to the page. -// -// **Refused means the machine is exactly as it was and what is wrong is in what was sent; failed -// means it is in a state nobody declared and what is wrong is on the machine.** They are fixed in -// different places, so a page saying "error" for both sends half its readers to the wrong one. -func TestRefusedAndFailedReachThePageAsDifferentThings(t *testing.T) { - rendered := render(t, viewOf(answers{ - nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}}, - wrong: []inventory.Doing{ - {Node: "anchor", Outcome: "refused", Refused: "not a declaration this host speaks", - At: time.Now()}, - {Node: "laptop", Outcome: "failed", At: time.Now(), - Failed: []inventory.FailedResource{{ID: "web.container", Error: "no such image"}}}, - }, - })) - // The rendered outcome, not the word anywhere on the page: both words appear in the - // stylesheet, so a plain Contains passes whatever the machine actually said. It did. - for _, want := range []string{`refused`, - `failed`} { - if !strings.Contains(rendered, want) { - t.Fatalf("the page does not distinguish the two — missing %s:\n%s", want, rendered) - } - } - // And the host's own words, which say exactly what it could not accept. - for _, want := range []string{"not a declaration this host speaks", "web.container", "no such image"} { - if !strings.Contains(rendered, want) { - t.Fatalf("the page does not say %q, so a reader must go and ask:\n%s", want, rendered) - } - } -} - -// Nothing wrong is said, not left blank. An empty page and a well mesh must not look alike. -func TestAWellMeshSaysSoRatherThanShowingNothing(t *testing.T) { - rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}})) - for _, want := range []string{ - "Every machine is doing what it was told", - "Every machine has been heard from", - "Every module is what its source last had", - } { - if !strings.Contains(rendered, want) { - t.Fatalf("a well mesh does not say %q:\n%s", want, rendered) - } - } -} - -// A board that cannot reach the mesh must not render an empty page: that says "nothing is wrong" -// in the one situation where nobody can know it. -func TestABoardThatCannotReadTheMeshSaysSo(t *testing.T) { - rendered := render(t, view{Unreachable: "the store did not answer"}) - if !strings.Contains(rendered, "the store did not answer") { - t.Fatalf("the reason is not on the page:\n%s", rendered) - } - if strings.Contains(rendered, "Every machine is doing what it was told") { - t.Fatal("a board that could not read the mesh reported that the mesh is well") - } -} - -// Quiet is not broken, and the page says which it is. -func TestQuietIsNotReportedAsBroken(t *testing.T) { - rendered := render(t, viewOf(answers{ - nodes: []inventory.Node{{Name: "laptop"}}, - quiet: []inventory.Node{{Name: "laptop"}}, - })) - if !strings.Contains(rendered, "not the same as tried and could not") { - t.Fatalf("the page does not separate quiet from broken:\n%s", rendered) - } - if !strings.Contains(rendered, "Every machine is doing what it was told") { - t.Fatalf("a quiet machine was counted as broken:\n%s", rendered) - } -} - -// The page renders what a machine said, and a hostile string in it is not markup. -// -// What is on this page comes from machines, and a machine's own words are the whole reason the -// page is useful. They are also the one thing here that nobody in this repository wrote. -func TestWhatAMachineSaidIsNotMarkup(t *testing.T) { - rendered := render(t, viewOf(answers{ - nodes: []inventory.Node{{Name: "anchor"}}, - wrong: []inventory.Doing{{Node: "anchor", Outcome: "refused", - Refused: ``, At: time.Now()}}, - })) - if strings.Contains(rendered, "