Compare commits
269
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bf7026d97 | ||
|
|
0d2fd2c5bb | ||
|
|
a011743c69 | ||
|
|
72d7802415 | ||
|
|
b7d9f44936 | ||
|
|
75213b9091 | ||
|
|
487aa040de | ||
|
|
ba26ba2772 | ||
|
|
3d05d74400 | ||
|
|
58ebe590a5 | ||
|
|
b24bb030ec | ||
|
|
327654e57b | ||
|
|
14127d4878 | ||
|
|
d0580a17e5 | ||
|
|
d6e0a8250a | ||
|
|
9b6b0c5686 | ||
|
|
792352dfad | ||
|
|
b9e0cd34c3 | ||
|
|
1c26235bc1 | ||
|
|
bbd442cdf4 | ||
|
|
042874e0ce | ||
|
|
be92762969 | ||
|
|
9c714f00d6 | ||
|
|
cbce8f96ef | ||
|
|
068283137b | ||
|
|
0090bf6af7 | ||
|
|
58924dc920 | ||
|
|
b1016e5c66 | ||
|
|
b1e02aca1b | ||
|
|
4635341a9d | ||
|
|
dcec6a4db3 | ||
|
|
2bfa6ae4a0 | ||
|
|
41f7b2c152 | ||
|
|
37229b4db5 | ||
|
|
d7bf1bae83 | ||
|
|
c6f3d8cdfa | ||
|
|
d9289ef6d4 | ||
|
|
81f497e5dd | ||
|
|
8e8712e352 | ||
|
|
7aa98e64ce | ||
|
|
48581af35c | ||
|
|
4d05385819 | ||
|
|
dcee8cb5bf | ||
|
|
2b5060789f | ||
|
|
abf9125689 | ||
|
|
52af210e47 | ||
|
|
4b25af2aa3 | ||
|
|
fc65215c25 | ||
|
|
c988d6d7be | ||
|
|
8bfaf1523e | ||
|
|
b037c73fd5 | ||
|
|
18da8b37e9 | ||
|
|
4f4d365360 | ||
|
|
08e11ad761 | ||
|
|
3b2adda1c8 | ||
|
|
68009b16fe | ||
|
|
298daa6fbe | ||
|
|
751e39186c | ||
|
|
20c147ffdb | ||
|
|
2eb9a22c24 | ||
|
|
070ecafc07 | ||
|
|
e51c6a2cb9 | ||
|
|
722682f1c4 | ||
|
|
b853439792 | ||
|
|
9cf47f4429 | ||
|
|
8ddc019cd2 | ||
|
|
fab6b0059e | ||
|
|
e1f5d4fdf0 | ||
|
|
bb1607e424 | ||
|
|
cf4834a36c | ||
|
|
9d8cbe7b81 | ||
|
|
e74c32ed50 | ||
|
|
146c48fd96 | ||
|
|
1f3abd3e0e | ||
|
|
6d620f77c3 | ||
|
|
f8286c063d | ||
|
|
e096b4595a | ||
|
|
09c0c6b367 | ||
|
|
d1fc25f682 | ||
|
|
eda457f415 | ||
|
|
59f4d486b1 | ||
|
|
801552c0eb | ||
|
|
ede9bce6ef | ||
|
|
0f0028785c | ||
|
|
6fdcfad8d3 | ||
|
|
8d9d33ae85 | ||
|
|
cc25baa563 | ||
|
|
68a2ebdcc3 | ||
|
|
69b99eec68 | ||
|
|
09bd0eec4f | ||
|
|
222a38e050 | ||
|
|
ee99a24f77 | ||
|
|
f68521da28 | ||
|
|
296064c799 | ||
|
|
df9231c734 | ||
|
|
843b709b59 | ||
|
|
f506fb34ec | ||
|
|
c34b937dd3 | ||
|
|
d84c9699b3 | ||
|
|
002d5e578c | ||
|
|
2421b82ad2 | ||
|
|
0ebd48a6a8 | ||
|
|
67e291c02a | ||
|
|
8a400d165e | ||
|
|
53d3cd7ce9 | ||
|
|
6648e4a5c8 | ||
|
|
7fa2568ce7 | ||
|
|
4b382ceffd | ||
|
|
ce86d09d22 | ||
|
|
853be00ebe | ||
|
|
e0ce2236dd | ||
|
|
9873b3bf13 | ||
|
|
541603c15c | ||
|
|
106507b1d3 | ||
|
|
e610f2d92c | ||
|
|
f80b6cdbd1 | ||
|
|
5dcf33db45 | ||
|
|
6abce7e956 | ||
|
|
0d2b304f08 | ||
|
|
bdfbd0254f | ||
|
|
16c5e78fa8 | ||
|
|
ed90771382 | ||
|
|
672d1f4ca1 | ||
|
|
208901c6cc | ||
|
|
4ac5cfe3a7 | ||
|
|
22660dc274 | ||
|
|
d7f359c498 | ||
|
|
ed25fd68e2 | ||
|
|
01c5ab2aab | ||
|
|
bb3cd6437b | ||
|
|
0b07e68cb8 | ||
|
|
625d02862c | ||
|
|
e8478e208b | ||
|
|
5761737687 | ||
|
|
89ec48b9a9 | ||
|
|
869fb6d6bf | ||
|
|
d25b69178b | ||
|
|
a7bb1e0b1c | ||
|
|
5eaed84271 | ||
|
|
326b1aec14 | ||
|
|
134d039ff8 | ||
|
|
d90c6ab93a | ||
|
|
6b7d2ec49b | ||
|
|
4ed1057df3 | ||
|
|
1f4c67a01b | ||
|
|
5474ea2d41 | ||
|
|
b7912172af | ||
|
|
b36babcd7d | ||
|
|
49cf0aa562 | ||
|
|
5a4f73f761 | ||
|
|
6af891e358 | ||
|
|
568f55fd2e | ||
|
|
35314175f2 | ||
|
|
ec2e6255a9 | ||
|
|
f3f34a170e | ||
|
|
e2622fd031 | ||
|
|
3ee32970ef | ||
|
|
11b654499b | ||
|
|
d69e19103c | ||
|
|
10f948e970 | ||
|
|
f421d4588c | ||
|
|
74b0dab34c | ||
|
|
41b20b2782 | ||
|
|
912e9f4e85 | ||
|
|
babd7b2f47 | ||
|
|
892dfd1d08 | ||
|
|
cfac579392 | ||
|
|
fe0d295490 | ||
|
|
d8a0238e02 | ||
|
|
dcf710a8d5 | ||
|
|
a2003ab616 | ||
|
|
1363a2fe27 | ||
|
|
f19a2254ac | ||
|
|
7d46e48b26 | ||
|
|
78915f9f7a | ||
|
|
17b8f14fe1 | ||
|
|
42c394acc2 | ||
|
|
b9ad7a2948 | ||
|
|
cde22ff627 | ||
|
|
79993fb498 | ||
|
|
aec55b7072 | ||
|
|
c7884f5a72 | ||
|
|
580c4d66a7 | ||
|
|
63bfc5fda5 | ||
|
|
02e3482eb5 | ||
|
|
294e83dab1 | ||
|
|
b5438bb331 | ||
|
|
c23be73d4d | ||
|
|
d2d171f2d2 | ||
|
|
73fa64ea68 | ||
|
|
1a13dbeb17 | ||
|
|
e11caecdad | ||
|
|
7bb9e55d0b | ||
|
|
00037608ae | ||
|
|
cea59428b1 | ||
|
|
5c832f2d19 | ||
|
|
cdebb7d1a5 | ||
|
|
6a803ea5b3 | ||
|
|
9745c1ab31 | ||
|
|
c0c3c3fed4 | ||
|
|
c1449fffe9 | ||
|
|
f873c97db5 | ||
|
|
b0b3d87fe2 | ||
|
|
ba189e6943 | ||
|
|
cadf74a176 | ||
|
|
74efe8e2e7 | ||
|
|
68af9eff44 | ||
|
|
518eeb7941 | ||
|
|
bf2da878a0 | ||
|
|
50cf253a43 | ||
|
|
980a0dee93 | ||
|
|
ac9c2d57be | ||
|
|
8b016cc62b | ||
|
|
cf2bb3b87d | ||
|
|
473376259b | ||
|
|
e1293fb0ad | ||
|
|
6784efae75 | ||
|
|
d86baebe9a | ||
|
|
82481099b7 | ||
|
|
b127f005c3 | ||
|
|
58b4fcb8c8 | ||
|
|
796f6410c1 | ||
|
|
e67c58cd98 | ||
|
|
85873b19e1 | ||
|
|
2ebbb79937 | ||
|
|
9204190445 | ||
|
|
06ea2168d8 | ||
|
|
2b149dd43e | ||
|
|
17dba2a34c | ||
|
|
11e4bc0ba1 | ||
|
|
e56f3aa1cb | ||
|
|
f966693583 | ||
|
|
5acc763992 | ||
|
|
4f009fff83 | ||
|
|
f27e31954f | ||
|
|
62650cd48c | ||
|
|
408f6dbad9 | ||
|
|
eae0577567 | ||
|
|
de26918c52 | ||
|
|
e01d18e548 | ||
|
|
59166b1031 | ||
|
|
c294949f2a | ||
|
|
28853a251b | ||
|
|
76a8b8df9e | ||
|
|
f86f6a74f0 | ||
|
|
a3e8a4185b | ||
|
|
78de54381b | ||
|
|
ff5ef0ab60 | ||
|
|
a5d6a1187c | ||
|
|
06d0a4bfe8 | ||
|
|
d293a0deaf | ||
|
|
11a44e1ec4 | ||
|
|
28c7f05b39 | ||
|
|
93a0c6202e | ||
|
|
7d82751862 | ||
|
|
905f3363c9 | ||
|
|
9f9d9b3b25 | ||
|
|
bde4b61b3b | ||
|
|
d07018f3c5 | ||
|
|
729a5f9e6c | ||
|
|
773b561f5e | ||
|
|
ca7e81e964 | ||
|
|
08bb56f1d3 | ||
|
|
1a44d281c2 | ||
|
|
1c8fe65601 | ||
|
|
bea1a1c513 | ||
|
|
21d38c9b0e | ||
|
|
689dd060b0 | ||
|
|
99c4c4ef04 |
+13
-5
@@ -1,5 +1,11 @@
|
||||
ARG GO_BASE=golang:1.25-alpine
|
||||
# The control plane's image.
|
||||
# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq
|
||||
# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how
|
||||
# `make image` broke once before (issue 146).
|
||||
ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||
# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go
|
||||
# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it.
|
||||
# Genesis builds this file and raises it as the container the process replaces on the first push
|
||||
# (mesh-host internal/bootstrap, novox/hq issue 223).
|
||||
#
|
||||
# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
|
||||
# machine where no mesh exists yet, and run before there is anything to check it against. So it
|
||||
@@ -15,13 +21,15 @@ ARG GO_BASE=golang:1.25-alpine
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
|
||||
# Dependencies first, so a change to the source does not refetch them.
|
||||
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
|
||||
# the image builds from this repository alone — the host's validator among them, whose module no
|
||||
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY vendor/ vendor/
|
||||
|
||||
COPY . .
|
||||
ARG VERSION=development
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o /mesh-controller ./cmd/mesh-controller
|
||||
|
||||
|
||||
@@ -27,17 +27,21 @@ build:
|
||||
IMAGE ?= mesh-controller:$(VERSION)
|
||||
DEV_TAG ?= mesh-controller:development
|
||||
|
||||
# The base the module declares, read from the manifest rather than written here twice.
|
||||
# The Go base the image is built on.
|
||||
#
|
||||
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
|
||||
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
|
||||
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
|
||||
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
|
||||
# what it cost).
|
||||
GO_BASE ?= $(shell python3 -c "import json;print(next(o['image'] for o in json.load(open('module.json'))['build']['on'] if o['arg']=='GO_BASE'))" 2>/dev/null)
|
||||
#
|
||||
# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds
|
||||
# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab —
|
||||
# still needs a Go base. The digest is the one the manifest declared until then.
|
||||
GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||
|
||||
image:
|
||||
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
|
||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -48,7 +52,7 @@ BUILDER_IMAGE ?= mesh-builder:$(VERSION)
|
||||
BUILDER_DEV_TAG ?= mesh-builder:development
|
||||
|
||||
builder-image:
|
||||
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
|
||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -98,18 +102,18 @@ proxy-image:
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
#
|
||||
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
|
||||
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
|
||||
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
|
||||
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
|
||||
# was reading, as "no response from stream". That reads as a bug in the code under test.
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: fmt vet postgres
|
||||
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
|
||||
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
test:
|
||||
go test -p 1 ./...
|
||||
go test -timeout 15m ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
@@ -193,6 +193,13 @@ passes every check that only looks at the message.
|
||||
|
||||
## The image
|
||||
|
||||
**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go
|
||||
bundle, `controller`, which the host on the controller's machine unpacks and runs as the process
|
||||
`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what
|
||||
still runs a container of the controller: genesis, which raises the first controller from it and
|
||||
installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the
|
||||
mesh's own build any more — `make image` builds it.
|
||||
|
||||
`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
|
||||
manager, no libc, no CA certificates.
|
||||
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
|
||||
//
|
||||
// **The queue is the controller's; the build running here is this machine's.** The controller can
|
||||
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
|
||||
// on this machine and containers in this machine's runtime, and only this machine can end them. So
|
||||
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
|
||||
// it, pause, resume.
|
||||
//
|
||||
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
|
||||
// set per build — so "the build running here" is one or none, and kill names it by id so a call
|
||||
// that arrives as one build ends and the next begins cannot end the wrong one.
|
||||
|
||||
// holder is this machine's state as a holder of the build seat.
|
||||
type holder struct {
|
||||
on, seat string
|
||||
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
|
||||
// rather than silently taking work again.
|
||||
workspace string
|
||||
// say publishes this machine's state: whether it takes work (link.HolderState).
|
||||
say func(link.HolderState) error
|
||||
// remove runs what a kill needs outside the build: the containers left behind.
|
||||
remove builder.Runner
|
||||
|
||||
mu sync.Mutex
|
||||
paused bool
|
||||
running *running
|
||||
}
|
||||
|
||||
// running is the build this machine is doing.
|
||||
type running struct {
|
||||
request link.BuildRequest
|
||||
step string
|
||||
started time.Time
|
||||
cancel context.CancelFunc
|
||||
killed bool
|
||||
// returned is the build's own work having ended, before a kill or not; outcome is what was then
|
||||
// announced, and announced whether it went out.
|
||||
returned bool
|
||||
outcome string
|
||||
announced bool
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// pausedFile is where the flag lives in the workspace.
|
||||
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
|
||||
|
||||
// newHolder reads the paused flag the workspace keeps.
|
||||
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
|
||||
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
|
||||
if _, err := os.Stat(pausedFile(workspace)); err == nil {
|
||||
h.paused = true
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Paused is asked by the taking loop before every fetch.
|
||||
func (h *holder) Paused() bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.paused
|
||||
}
|
||||
|
||||
// setPaused records the flag in the workspace first and then in memory, so what this holder says
|
||||
// it is and what it would be after a restart never differ.
|
||||
func (h *holder) setPaused(paused bool) error {
|
||||
path := pausedFile(h.workspace)
|
||||
if paused {
|
||||
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
|
||||
}
|
||||
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.paused = paused
|
||||
h.mu.Unlock()
|
||||
h.announce()
|
||||
return nil
|
||||
}
|
||||
|
||||
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
|
||||
// controller reading an older state is a plan read as late rather than a build lost.
|
||||
func (h *holder) announce() {
|
||||
if h.say == nil {
|
||||
return
|
||||
}
|
||||
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
|
||||
// a pause outlives the events stream's retention.
|
||||
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||
h.announce()
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if h.Paused() {
|
||||
h.announce()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// begin records the build this machine took, with the cancel that ends it.
|
||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||
h.mu.Lock()
|
||||
h.running = r
|
||||
h.mu.Unlock()
|
||||
return r
|
||||
}
|
||||
|
||||
// end clears it, and lets a kill waiting on it know it is over.
|
||||
func (h *holder) end(r *running) {
|
||||
h.mu.Lock()
|
||||
if h.running == r {
|
||||
h.running = nil
|
||||
}
|
||||
h.mu.Unlock()
|
||||
close(r.done)
|
||||
}
|
||||
|
||||
// stepped records the step a build is at, for `current`.
|
||||
func (h *holder) stepped(r *running, step string) {
|
||||
h.mu.Lock()
|
||||
r.step = step
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// currentBuild is what `current` answers.
|
||||
type currentBuild struct {
|
||||
On string `json:"on"`
|
||||
Paused bool `json:"paused"`
|
||||
Running *struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
} `json:"running,omitempty"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
func (h *holder) current() currentBuild {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out := currentBuild{On: h.on, Paused: h.paused}
|
||||
taking := "taking builds"
|
||||
if h.paused {
|
||||
taking = "paused, taking no new build"
|
||||
}
|
||||
if h.running == nil {
|
||||
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
|
||||
return out
|
||||
}
|
||||
r := h.running
|
||||
elapsed := time.Since(r.started).Round(time.Second)
|
||||
out.Running = &struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
|
||||
r.started.UTC().Format(time.RFC3339), elapsed.String()}
|
||||
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
|
||||
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
|
||||
return out
|
||||
}
|
||||
|
||||
// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
|
||||
// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
|
||||
// (killAnswer in the controller's queue.go, 75s).
|
||||
var (
|
||||
killRemoves = 15 * time.Second
|
||||
killWaits = 20 * time.Second
|
||||
)
|
||||
|
||||
// kill ends the build with this id, if it is the one running here and still working: its context
|
||||
// cancelled — which kills each command's process group — and the containers it started removed by
|
||||
// their label, once at once and again after the build has ended, so one created while it was being
|
||||
// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
|
||||
// ask so it is not redelivered; the answer says whether that happened.
|
||||
func (h *holder) kill(id string) (string, error) {
|
||||
h.mu.Lock()
|
||||
r := h.running
|
||||
if r == nil || r.request.ID != id {
|
||||
doing := "nothing"
|
||||
if r != nil {
|
||||
doing = r.request.ID
|
||||
}
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
|
||||
}
|
||||
if r.returned {
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
|
||||
}
|
||||
r.killed = true
|
||||
cancel, done := r.cancel, r.done
|
||||
h.mu.Unlock()
|
||||
|
||||
cancel()
|
||||
removed, removeErr := h.removeContainers(id)
|
||||
ended := false
|
||||
select {
|
||||
case <-done:
|
||||
ended = true
|
||||
case <-time.After(killWaits):
|
||||
}
|
||||
again, againErr := h.removeContainers(id)
|
||||
removed += again
|
||||
if removeErr == nil {
|
||||
removeErr = againErr
|
||||
}
|
||||
containers := fmt.Sprintf("%d container(s) it started removed", removed)
|
||||
if removeErr != nil {
|
||||
containers = "its containers could not all be listed or removed: " + removeErr.Error()
|
||||
}
|
||||
if !ended {
|
||||
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
|
||||
"its outcome is in", id, h.on, containers), nil
|
||||
}
|
||||
h.mu.Lock()
|
||||
outcome, announced := r.outcome, r.announced
|
||||
h.mu.Unlock()
|
||||
if !announced {
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
|
||||
"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
|
||||
containers), nil
|
||||
}
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
|
||||
"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
|
||||
}
|
||||
|
||||
// removeContainers is one pass of removing what the build left, bounded.
|
||||
func (h *holder) removeContainers(id string) (int, error) {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
||||
defer stop()
|
||||
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||
}
|
||||
|
||||
// returned records that the build's work ended, and says whether a kill came first — only then is
|
||||
// the build killed; an error it ended with on its own is its own outcome.
|
||||
func (h *holder) returned(r *running) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
r.returned = true
|
||||
return r.killed
|
||||
}
|
||||
|
||||
// said records the outcome announced, and whether it went out, for a kill to answer with.
|
||||
func (h *holder) said(r *running, outcome string, announced bool) {
|
||||
h.mu.Lock()
|
||||
r.outcome, r.announced = outcome, announced
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// handlers are the seat's verbs, as this machine answers them.
|
||||
func (h *holder) handlers() map[string]link.ToolHandler {
|
||||
return map[string]link.ToolHandler{
|
||||
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
|
||||
return nil, errors.New("kill needs the build's id")
|
||||
}
|
||||
said, err := h.kill(strings.TrimSpace(args.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": said}, nil
|
||||
},
|
||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
said := h.on + " is paused: it takes no new build until resumed"
|
||||
if c := h.current(); c.Running != nil {
|
||||
said += "; " + c.Running.ID + " runs on and finishes"
|
||||
}
|
||||
return map[string]any{"said": said, "paused": true}, nil
|
||||
},
|
||||
"resume": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func orNothing(s string) string {
|
||||
if s == "" {
|
||||
return "its start"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
|
||||
// log it would be is the one being ended.
|
||||
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
|
||||
// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
|
||||
// the module answering, the seat, scope node, the machine, its description and argument schema — so
|
||||
// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
|
||||
//
|
||||
// One service per machine, named for the seat and identified by the machine, so the answer is this
|
||||
// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
|
||||
// store, and what it serves is what this binary was built to serve.
|
||||
func announcement(seat, module, node string) micro.Info {
|
||||
s, _ := catalogue.SeatNamed(seat)
|
||||
var endpoints []micro.EndpointInfo
|
||||
for _, v := range s.Serves {
|
||||
schema, _ := json.Marshal(v.Input)
|
||||
endpoints = append(endpoints, micro.EndpointInfo{
|
||||
Name: seat + "__" + v.Name,
|
||||
Subject: link.NodeSeatToolSubject(seat, v.Name, node),
|
||||
// The queue group the verbs are served in, as every runtime announces its own.
|
||||
QueueGroup: "seat." + seat,
|
||||
Metadata: map[string]string{
|
||||
"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
|
||||
"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
|
||||
},
|
||||
})
|
||||
}
|
||||
return micro.Info{
|
||||
ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
|
||||
Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
|
||||
Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
}
|
||||
|
||||
// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
|
||||
func moduleOf(user string) string {
|
||||
if _, module, ok := strings.Cut(user, "."); ok && module != "" {
|
||||
return module
|
||||
}
|
||||
return "build-agent"
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
||||
// 0219), and what it says about itself follows.
|
||||
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
var said []link.HolderState
|
||||
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
||||
said = append(said, s)
|
||||
return nil
|
||||
})
|
||||
if h.Paused() {
|
||||
t.Fatal("a new holder starts paused")
|
||||
}
|
||||
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
||||
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
||||
}
|
||||
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
||||
if !again.Paused() {
|
||||
t.Fatal("restarted, the holder forgot it was paused")
|
||||
}
|
||||
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
||||
t.Fatal("resumed, the holder came back paused")
|
||||
}
|
||||
}
|
||||
|
||||
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
||||
// label, and refuses an id it is not building.
|
||||
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
var removed []string
|
||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "ps" {
|
||||
return "c1\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
kill := h.handlers()["kill"]
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
||||
t.Fatal("killed a build while none ran")
|
||||
}
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
||||
h.stepped(r, "image")
|
||||
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
||||
t.Fatalf("current says %+v", c)
|
||||
}
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "build-1") {
|
||||
t.Fatalf("killed another id: %v", err)
|
||||
}
|
||||
// The build's own goroutine: it ends when its context does, as a build's commands do, and
|
||||
// announces what came of it.
|
||||
go func() {
|
||||
<-building.Done()
|
||||
if h.returned(r) {
|
||||
h.said(r, link.KilledByHand, true)
|
||||
}
|
||||
h.end(r)
|
||||
}()
|
||||
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if building.Err() == nil {
|
||||
t.Fatal("the build's context was not cancelled")
|
||||
}
|
||||
if !r.killed {
|
||||
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
||||
}
|
||||
said := answer.(map[string]any)["said"].(string)
|
||||
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
|
||||
t.Errorf("kill said %q", said)
|
||||
}
|
||||
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
||||
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
||||
t.Errorf("removed %v", removed)
|
||||
}
|
||||
if c := h.current(); c.Running != nil {
|
||||
t.Errorf("after the kill current says %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
|
||||
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
|
||||
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
|
||||
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
|
||||
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
|
||||
}
|
||||
kill := info.Endpoints[1]
|
||||
md := kill.Metadata
|
||||
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
|
||||
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
|
||||
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
|
||||
t.Fatalf("kill is announced as %+v", kill)
|
||||
}
|
||||
body, err := json.Marshal(info)
|
||||
if err != nil || len(body) > 8*1024 {
|
||||
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
|
||||
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
|
||||
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
|
||||
_, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
|
||||
if killed := h.returned(r); killed {
|
||||
t.Fatal("a build nobody killed reads as killed")
|
||||
}
|
||||
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
|
||||
t.Fatalf("killed a build that had ended: %v", err)
|
||||
}
|
||||
h.end(r)
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
|
||||
go func() {
|
||||
<-building.Done()
|
||||
if h.returned(r) {
|
||||
h.said(r, link.KilledByHand, false)
|
||||
}
|
||||
h.end(r)
|
||||
}()
|
||||
said, err := h.kill("build-2")
|
||||
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
|
||||
t.Fatalf("kill said %q (%v)", said, err)
|
||||
}
|
||||
}
|
||||
+156
-13
@@ -19,11 +19,13 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
@@ -107,43 +109,96 @@ func run() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
machine, err := takeWorkFrom(credential, on)
|
||||
js, seat, err := dialFor(credential)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
|
||||
// paused flag is read from the workspace before anything is taken, so a holder restarted while
|
||||
// paused takes nothing.
|
||||
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
|
||||
body, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
|
||||
return err
|
||||
})
|
||||
if h.Paused() {
|
||||
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||
}
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
// one serves none, and a subscription its holder has no grant for would be refused for ever.
|
||||
if seat == link.TheBuildMachine {
|
||||
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
|
||||
log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopServing()
|
||||
// And says so, for the console to find (novox/hq ADR 0197).
|
||||
stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
|
||||
announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopAnnouncing()
|
||||
go h.stateWhilePaused(ctx, time.Hour)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
answer(ctx, publisher, on, workspace, work)
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
|
||||
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
|
||||
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
|
||||
// holds.
|
||||
//
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||
// machine told about both would take work from one and answer on the other, and every log line would
|
||||
// say it was fine.
|
||||
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
|
||||
func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
||||
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
||||
// and that is enough to dial it, pinned.
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
return link.MachineOverNATS(js, on), nil
|
||||
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
||||
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
||||
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
||||
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
||||
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
||||
return js, seat, nil
|
||||
}
|
||||
|
||||
// answer does one build and says what happened, whichever way it went.
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
|
||||
request := work.Request()
|
||||
|
||||
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
|
||||
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
|
||||
// reaches it through the parent, and that keeps today's meaning below.
|
||||
building, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
var mine *running
|
||||
if h != nil {
|
||||
mine = h.begin(request, cancel)
|
||||
defer h.end(mine)
|
||||
}
|
||||
|
||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||
@@ -157,6 +212,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
say := func(step, message string) {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
h.stepped(mine, step)
|
||||
}
|
||||
}
|
||||
builder.Said = say
|
||||
defer func() { builder.Said = nil }()
|
||||
@@ -166,7 +224,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
|
||||
result := link.BuildResult{
|
||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||
Ref: request.Ref, On: on, Source: request.Source,
|
||||
Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
|
||||
}
|
||||
what := "building " + request.Repository
|
||||
if request.Path != "" {
|
||||
@@ -179,19 +237,48 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
|
||||
npmrc, err := packagesFrom()
|
||||
var built builder.Result
|
||||
if err == nil {
|
||||
if request.Check != nil {
|
||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
||||
// published or registered; the verdict is the outcome.
|
||||
result.Checked = request.Check
|
||||
registry := ""
|
||||
if r, ok := publisher.(builder.Registry); ok {
|
||||
registry = r.Address
|
||||
}
|
||||
var v builder.CheckVerdict
|
||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
||||
if err == nil {
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||
// after a clone that then fails at npm ci.
|
||||
built, err = builder.Build(ctx, builder.Command, publisher,
|
||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||
// docker client (ADR 0219).
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
if err != nil {
|
||||
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
||||
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
||||
killed := false
|
||||
if mine != nil {
|
||||
killed = h.returned(mine) && err != nil
|
||||
}
|
||||
if killed {
|
||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||
// error it ended with is the kill's consequence, not a fault of the source.
|
||||
result.Failed = link.KilledByHand
|
||||
say("failed", link.KilledByHand)
|
||||
} else if err != nil {
|
||||
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
||||
// builder that is not running, and those want completely different responses.
|
||||
result.Failed = err.Error()
|
||||
say("failed", err.Error())
|
||||
} else if request.Check != nil {
|
||||
say("checked", result.Check.Verdict+": "+result.Check.Summary)
|
||||
} else {
|
||||
manifest, marshalErr := json.Marshal(built.Manifest)
|
||||
if marshalErr != nil {
|
||||
@@ -205,6 +292,8 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
})
|
||||
}
|
||||
result.Against = built.Against
|
||||
result.SourceFingerprint = built.Source
|
||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
||||
for _, r := range built.Read {
|
||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
@@ -212,7 +301,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
}
|
||||
|
||||
if err := work.Announce(ctx, result); err != nil {
|
||||
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
|
||||
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
|
||||
// to be built again. A machine being stopped is the other case and keeps its meaning: the
|
||||
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
|
||||
announcing := ctx
|
||||
if killed {
|
||||
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
announcing = fresh
|
||||
}
|
||||
announceErr := work.Announce(announcing, result)
|
||||
if mine != nil {
|
||||
h.said(mine, result.Failed, announceErr == nil)
|
||||
}
|
||||
if err := announceErr; err != nil {
|
||||
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||
// nothing was said at all, so another machine can try.
|
||||
@@ -227,6 +330,31 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
}
|
||||
|
||||
// checkSpecOf is a check request as the builder runs it.
|
||||
func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
c := request.Check
|
||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
}
|
||||
for _, m := range c.Members {
|
||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||
Ref: m.Ref, Paths: m.Paths})
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
// layerOf is one layer of a check as the outcome carries it.
|
||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules}
|
||||
}
|
||||
|
||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||
// (novox/hq ADR 0076, issue 053).
|
||||
//
|
||||
@@ -453,6 +581,21 @@ type Credential struct {
|
||||
// as two fields and this machine joins them once, here, to dial.
|
||||
User string `json:"user,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
// Claims are the seats the module this credential was issued for claims, as the mesh writes
|
||||
// them beside the credential (novox/hq ADR 0159). The first is the build role this machine
|
||||
// serves; a credential naming none is from before claims travelled in it.
|
||||
Claims []struct {
|
||||
Seat string `json:"seat"`
|
||||
} `json:"claims,omitempty"`
|
||||
}
|
||||
|
||||
// seatsClaimed is the seats the credential names, in order.
|
||||
func (c Credential) seatsClaimed() []string {
|
||||
out := make([]string, 0, len(c.Claims))
|
||||
for _, claim := range c.Claims {
|
||||
out = append(out, claim.Seat)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
|
||||
|
||||
@@ -105,6 +105,7 @@ func buildOnce(ctx context.Context, args []string) error {
|
||||
Ref: *ref,
|
||||
Manifest: built.Manifest,
|
||||
Against: built.Against,
|
||||
Source: built.Source,
|
||||
}
|
||||
for _, r := range built.Read {
|
||||
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
@@ -135,6 +136,7 @@ type onceResult struct {
|
||||
Made []madeArtifact `json:"made"`
|
||||
Against []string `json:"against,omitempty"`
|
||||
Read []readRepository `json:"read,omitempty"`
|
||||
Source string `json:"source-fingerprint,omitempty"`
|
||||
}
|
||||
|
||||
// readRepository is a repository this build read source from besides the module's own.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover).
|
||||
func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) {
|
||||
var held Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x",
|
||||
"claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" {
|
||||
t.Errorf("the old builder's credential serves %q", got)
|
||||
}
|
||||
var bare Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine {
|
||||
t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
|
||||
//
|
||||
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
|
||||
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
|
||||
// not happen. Three ways a process stands to the lease:
|
||||
//
|
||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
||||
// its service manager restarts it as a candidate (serve, in push.go).
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
|
||||
// between makes it stale and refused like any other. **When nobody holds the lease, the command
|
||||
// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
|
||||
// it, as it would for another controller.
|
||||
// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
|
||||
// refused nothing: there is nothing to order against, and nothing it does reaches a machine.
|
||||
//
|
||||
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
|
||||
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
|
||||
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
|
||||
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
|
||||
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
|
||||
// instead would be a controller that can never send the user list that lets it act.
|
||||
|
||||
// actor is this process's standing to the lease.
|
||||
type actor struct {
|
||||
mu sync.Mutex
|
||||
// held is the lease this process holds: the serving controller's, or a command's own.
|
||||
held *lease.Lease
|
||||
// unleased is why a serving controller acts without the lease; empty while it holds it or is not
|
||||
// serving.
|
||||
unleased string
|
||||
// serving is a serving controller, which never borrows another's epoch.
|
||||
serving bool
|
||||
// kv is the lease bucket, for a command to read the holder's epoch from.
|
||||
kv jetstream.KeyValue
|
||||
close func()
|
||||
// noBus is a process with no bus configured.
|
||||
noBus bool
|
||||
// reset is when the lease bucket was found raised again from nothing and its revisions moved past
|
||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
var theLease = &actor{}
|
||||
|
||||
// instance names this process among controller instances: its machine, its process and when it
|
||||
// started. The lease's holder and every call this process keeps carry it.
|
||||
var instance = func() string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||
}()
|
||||
|
||||
// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
|
||||
func (a *actor) epoch(ctx context.Context) (uint64, error) {
|
||||
a.mu.Lock()
|
||||
held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
|
||||
a.mu.Unlock()
|
||||
switch {
|
||||
case held != nil:
|
||||
return held.Epoch()
|
||||
case serving && unleased != "":
|
||||
return 0, nil
|
||||
case serving:
|
||||
return 0, lease.ErrNotHeld
|
||||
case noBus:
|
||||
return 0, nil
|
||||
}
|
||||
return a.forACommand(ctx)
|
||||
}
|
||||
|
||||
// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
|
||||
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if a.held != nil {
|
||||
return a.held.Epoch()
|
||||
}
|
||||
if a.kv == nil {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
// No bus: this process reaches no machine, and has nothing to order against.
|
||||
a.noBus = true
|
||||
return 0, nil
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
|
||||
"read and nothing is done: %w", err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := broker.EnsureLeaseBucket(reading, api); err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
kv, err := api.KeyValue(reading, broker.LeaseBucket)
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
a.kv, a.close = kv, js.Close
|
||||
if _, found, err := lease.Current(reading, kv); err == nil && !found {
|
||||
// Nobody: this command takes it for as long as it runs.
|
||||
l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
epoch, err := l.TryTake(reading)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
|
||||
}
|
||||
keeping, stop := context.WithCancel(context.Background())
|
||||
go l.Keep(keeping)
|
||||
a.held = l
|
||||
closeBus := a.close
|
||||
a.close = func() {
|
||||
stop()
|
||||
l.Release(context.Background())
|
||||
closeBus()
|
||||
}
|
||||
return epoch, nil
|
||||
}
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
holder, found, err := lease.Current(reading, a.kv)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
|
||||
"more is done under an epoch nobody holds — run the command again")
|
||||
}
|
||||
return holder.Epoch, nil
|
||||
}
|
||||
|
||||
// release gives back what this process holds, at its end.
|
||||
func (a *actor) release() {
|
||||
a.mu.Lock()
|
||||
closing := a.close
|
||||
a.close = nil
|
||||
a.mu.Unlock()
|
||||
if closing != nil {
|
||||
closing()
|
||||
}
|
||||
}
|
||||
|
||||
// holderOf is this process as the lease's holder.
|
||||
func holderOf(instance string) lease.Holder {
|
||||
host, _ := os.Hostname()
|
||||
return lease.Holder{Instance: instance, Host: host, Build: version}
|
||||
}
|
||||
|
||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
||||
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
|
||||
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
|
||||
a.mu.Lock()
|
||||
a.serving = true
|
||||
a.mu.Unlock()
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.EnsureLeaseBucket(asserting, api)
|
||||
cancel()
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
|
||||
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
|
||||
"no machine refuses the next epoch", was, floor, floor)
|
||||
}})
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
gone := make(chan struct{})
|
||||
epoch, err := l.Take(ctx)
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
js.Close()
|
||||
return nil, ctx.Err()
|
||||
case err != nil && !errors.Is(err, lease.ErrUnwritable):
|
||||
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
|
||||
js.Close()
|
||||
return nil, err
|
||||
case err != nil:
|
||||
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
|
||||
a.mu.Lock()
|
||||
a.unleased = err.Error()
|
||||
a.mu.Unlock()
|
||||
say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
|
||||
"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
|
||||
go a.takeWhenGranted(ctx, l, inv, gone)
|
||||
default:
|
||||
a.took(ctx, l, inv, epoch, gone)
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.close = func() {
|
||||
if held, err := l.Epoch(); err == nil {
|
||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
|
||||
say("how epoch %d ended could not be recorded: %v", held, err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
l.Release(context.Background())
|
||||
js.Close()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
return gone, nil
|
||||
}
|
||||
|
||||
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
|
||||
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
|
||||
a.mu.Lock()
|
||||
a.held, a.unleased = l, ""
|
||||
a.mu.Unlock()
|
||||
h := holderOf(instance)
|
||||
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
|
||||
Build: h.Build, Taken: time.Now()})
|
||||
cancel()
|
||||
if err != nil {
|
||||
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
|
||||
}
|
||||
for _, e := range expired {
|
||||
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
|
||||
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
|
||||
}
|
||||
go l.Keep(ctx)
|
||||
go func() {
|
||||
<-l.Lost()
|
||||
if ctx.Err() == nil {
|
||||
why := l.LostWhy()
|
||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
|
||||
cancel()
|
||||
fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
|
||||
"be started again as a candidate\n", epoch, why)
|
||||
}
|
||||
close(gone)
|
||||
}()
|
||||
}
|
||||
|
||||
// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
|
||||
// controller if another took it meanwhile: two serving at once is what the lease is for.
|
||||
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
|
||||
tick := time.NewTicker(lease.RenewEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
epoch, err := l.TryTake(ctx)
|
||||
if errors.Is(err, lease.ErrTaken) {
|
||||
fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
|
||||
"stops and exits\n", err)
|
||||
close(gone)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
// Still not written, or not readable this time: unleased, said by S12, tried again.
|
||||
a.mu.Lock()
|
||||
a.unleased = err.Error()
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
a.took(ctx, l, inv, epoch, gone)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// standing is what `status` and the self-check say of this process and the lease.
|
||||
type standing struct {
|
||||
Epoch uint64
|
||||
Held bool
|
||||
Renewed time.Time
|
||||
Unleased string
|
||||
// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
|
||||
Reset time.Time
|
||||
ResetSaid string
|
||||
}
|
||||
|
||||
func (a *actor) standing() standing {
|
||||
a.mu.Lock()
|
||||
held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
|
||||
a.mu.Unlock()
|
||||
st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
|
||||
if held == nil {
|
||||
return st
|
||||
}
|
||||
epoch, err := held.Epoch()
|
||||
st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
|
||||
return st
|
||||
}
|
||||
|
||||
// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
|
||||
func init() {
|
||||
link.ActingGate = func(ctx context.Context) error {
|
||||
_, err := theLease.epoch(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("this controller may not send: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
+187
-20
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -38,7 +40,10 @@ import (
|
||||
//
|
||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||
// machines this just blocked is worth more than the milliseconds.
|
||||
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("assign %s names no module", node)
|
||||
}
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
@@ -46,6 +51,16 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
||||
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
||||
// resources are applied through a seat nothing on the node holds is refused, because that order
|
||||
// — the service manager, the package manager and the runtime before anything that installs,
|
||||
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
|
||||
// holders that depend on each other go on in one command.
|
||||
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
||||
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
||||
// assignment resolves against a record rather than against a coincidence.
|
||||
@@ -53,58 +68,180 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
var lines []string
|
||||
var added []string
|
||||
for _, module := range modules {
|
||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||
if err != nil {
|
||||
return strings.Join(lines, "\n"), err
|
||||
}
|
||||
if !fresh {
|
||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||
lines = append(lines, fmt.Sprintf(
|
||||
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
|
||||
continue
|
||||
}
|
||||
added = append(added, module)
|
||||
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
|
||||
}
|
||||
if !fresh {
|
||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||
return fmt.Sprintf("%s already runs %s — one node runs one of each (ADR 0115); nothing changed",
|
||||
node, module), nil
|
||||
if len(added) == 0 {
|
||||
return strings.Join(lines, "\n"), nil
|
||||
}
|
||||
said := fmt.Sprintf("%s is assigned %s", node, module)
|
||||
answer := strings.Join(lines, "\n")
|
||||
for _, line := range settled {
|
||||
said += "\n " + line
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||
// node's list, and every other node's, is `status`'s.
|
||||
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||
for _, module := range added {
|
||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||
answer += "\n " + line
|
||||
}
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||
return said + blockedElsewhere(ctx, open, node), err
|
||||
return answer + blockedElsewhere(ctx, open, node), err
|
||||
}
|
||||
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
||||
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
||||
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
||||
isAdded := map[string]bool{}
|
||||
for _, m := range added {
|
||||
isAdded[m] = true
|
||||
}
|
||||
for _, u := range plan.Unhostable {
|
||||
if u.Module != module {
|
||||
if !isAdded[u.Module] {
|
||||
continue
|
||||
}
|
||||
for _, c := range u.Missing {
|
||||
said += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||
}
|
||||
}
|
||||
return said + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||
blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// unassign takes a module off a node. What it leaves behind is the host's business: a directory
|
||||
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
||||
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
|
||||
// and are not judged again.
|
||||
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
|
||||
map[string]catalogue.Manifest, []string, error) {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
already := map[string]bool{}
|
||||
for _, a := range assigned {
|
||||
already[a] = true
|
||||
}
|
||||
var adding []string
|
||||
for _, m := range modules {
|
||||
if !already[m] {
|
||||
adding = append(adding, m)
|
||||
}
|
||||
}
|
||||
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
|
||||
// with everything else this act changed, so they are not said twice.
|
||||
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Two modules declaring one package, path or unit is refused before anything is recorded
|
||||
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
||||
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return shelf, assigned, nil
|
||||
}
|
||||
|
||||
// unassign takes modules off a node. What they leave behind is the host's business: a directory
|
||||
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
||||
//
|
||||
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
||||
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
||||
// about the command's own output would ever have said so.
|
||||
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
//
|
||||
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
|
||||
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, module, node) + blockedElsewhere(ctx, open, node), nil
|
||||
assigned, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Every one checked before any is taken off, so a refusal leaves the node as it was.
|
||||
runs := map[string]bool{}
|
||||
for _, a := range assigned {
|
||||
runs[a] = true
|
||||
}
|
||||
for _, module := range modules {
|
||||
if !runs[module] {
|
||||
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
||||
}
|
||||
}
|
||||
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, module := range modules {
|
||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, strings.Join(modules, ", "), node)
|
||||
var left []string
|
||||
for _, a := range assigned {
|
||||
if !slices.Contains(modules, a) {
|
||||
left = append(left, a)
|
||||
}
|
||||
}
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||
// command API and the controller seat's verbs as they do from the command line.
|
||||
func splitModules(field string) []string {
|
||||
var out []string
|
||||
for _, m := range strings.Split(field, ",") {
|
||||
if m = strings.TrimSpace(m); m != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
||||
@@ -152,3 +289,33 @@ func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
||||
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
||||
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
|
||||
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
|
||||
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
|
||||
// module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
m, known := shelf[module]
|
||||
if !known || mayIssue(m) != nil {
|
||||
return ""
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||
return ""
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err == nil {
|
||||
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
|
||||
"`push %s` refuses to send %s until it is", err, module, node, node, module)
|
||||
}
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
@@ -111,6 +111,14 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
@@ -145,7 +153,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// container is told so beside the sealed connection genesis wrote.
|
||||
// process is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -157,8 +165,8 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
|
||||
Reference: "registry.example/control@" + aDigest}})
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -175,6 +183,12 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -194,12 +208,12 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.server" {
|
||||
if r["id"] == "mesh-controller.controller" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's container is not in its own node's declaration")
|
||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
@@ -232,7 +246,7 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
out := m
|
||||
out.Resources = nil
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
if r["type"] != "container" && r["type"] != "process" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||
|
||||
@@ -95,7 +95,7 @@ func showFiltering(f inventory.Filtering, adopted bool) {
|
||||
case fw.Active:
|
||||
fmt.Printf(" found firewall %s is ACTIVE on this converged machine; the next apply retires it again\n", fw.Kind)
|
||||
case fw.RetiredBy == "removed":
|
||||
fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0175)\n", fw.Kind)
|
||||
fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0180)\n", fw.Kind)
|
||||
case fw.RetiredBy == inventory.FilterMesh || fw.RetiredBy == "mesh":
|
||||
fmt.Printf(" found firewall %s, retired by the mesh; its configuration stays on disk\n", fw.Kind)
|
||||
case fw.RetiredBy != "":
|
||||
|
||||
@@ -95,10 +95,10 @@ func commands(who Authenticator) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return assign(ctx, open, in.Node, in.Module)
|
||||
return assign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||
}))
|
||||
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return unassign(ctx, open, in.Node, in.Module)
|
||||
return unassign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||
}))
|
||||
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
||||
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
||||
|
||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
||||
func aKeeper() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
||||
// are made — without a bus to carry it.
|
||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
||||
t.Helper()
|
||||
digest := pushedBy(t, open, node)
|
||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// d1 runs D1 once.
|
||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
t.Helper()
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-time.Hour)
|
||||
cases := []struct {
|
||||
name string
|
||||
send lastSend
|
||||
want bool
|
||||
}{
|
||||
{"never sent", lastSend{}, false},
|
||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.send.settled("keeper", now); got != c.want {
|
||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A binding to data moves only by a person (novox/hq ADR 0232, issue 273).
|
||||
//
|
||||
// The resolver keeps each consumer of a provision that keeps its data at the provider it was last
|
||||
// sent (catalogue/bound.go) and says what it would have moved. This is where that is said: on the
|
||||
// push that composed it, and by the self-check's D12 every run, as an urgent condition naming the
|
||||
// consumer, both providers and the pin that confirms the move.
|
||||
|
||||
// The condition kinds of D12.
|
||||
const (
|
||||
// kindBindingKept is a move the resolver refused: the consumer is still where its data is.
|
||||
kindBindingKept = "binding-kept"
|
||||
// kindBindingMoved is a consumer about to be sent another provider than the one on record with
|
||||
// no pin naming it — what the resolver exists to make impossible, said if it ever is not.
|
||||
kindBindingMoved = "binding-moved"
|
||||
// kindBindingMoving is a move a pin asked for, not yet sent: a person's act, said so that the
|
||||
// data is moved before the push that carries it.
|
||||
kindBindingMoving = "binding-moving"
|
||||
)
|
||||
|
||||
// probeBindingsID is the self-check's id for this probe, and the source of what it raises.
|
||||
const probeBindingsID = "D12"
|
||||
|
||||
// keptObservation is the urgent condition for one refused move.
|
||||
func keptObservation(k catalogue.KeptBinding) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: bindingID(k.Machine, k.Consumer, k.Provision),
|
||||
Token: kindBindingKept, Kind: kindBindingKept, Machine: k.Machine, Also: otherMachines(k.Machine, k.Bound.Node, k.Would.Node),
|
||||
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("on %s, the mesh %s", k.Machine, k.String())}
|
||||
}
|
||||
|
||||
func bindingID(machine, consumer, provision string) string {
|
||||
return machine + "." + consumer + "." + provision
|
||||
}
|
||||
|
||||
func otherMachines(machine string, nodes ...string) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{machine: true}
|
||||
for _, n := range nodes {
|
||||
if n != "" && !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
|
||||
// condition at once where this process keeps the conditions: a push is when a person is looking.
|
||||
func reportKept(ctx context.Context, plan catalogue.Resolution) {
|
||||
for _, k := range plan.Kept {
|
||||
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
|
||||
if conditionsFrom != nil {
|
||||
o := keptObservation(k)
|
||||
o.Source = probeBindingsID
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
|
||||
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
|
||||
// moves it (said, so the data goes first), and never anything else.
|
||||
func probeBindings(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, d.open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
// D1 says it, with the binding that refused it when that is why.
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
bound, err := inv.BindingsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pins, err := inv.PinsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, bindingFindings(plan, bound, pins)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// bindingFindings is what one machine's resolution says against its record.
|
||||
func bindingFindings(plan catalogue.Resolution, bound map[string]map[string]catalogue.Chosen,
|
||||
pins map[string]catalogue.Chosen) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, k := range plan.Kept {
|
||||
out = append(out, keptObservation(k))
|
||||
}
|
||||
said := map[string]bool{}
|
||||
for _, need := range plan.Needs {
|
||||
if !need.KeepsData || need.ByRecord {
|
||||
continue
|
||||
}
|
||||
was, recorded := bound[need.For][need.Name]
|
||||
now := catalogue.Chosen{Node: need.From, Module: need.Module}
|
||||
if !recorded || was == now || (was.Module == "" && was.Node == now.Node) {
|
||||
continue
|
||||
}
|
||||
id := bindingID(plan.Node, need.For, need.Name)
|
||||
if said[id] {
|
||||
continue
|
||||
}
|
||||
said[id] = true
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Machine: plan.Node,
|
||||
Also: otherMachines(plan.Node, was.Node, now.Node), Resolver: conditions.ResolverOperator}
|
||||
if pin, pinned := pins[need.Name]; pinned && pin.Node == now.Node && (pin.Module == "" || pin.Module == now.Module) {
|
||||
o.Token, o.Kind, o.Severity = kindBindingMoving, kindBindingMoving, conditions.Warning
|
||||
o.Summary = fmt.Sprintf("on %s, %s's %s moves from %s to %s at the next push, by the pin — its data "+
|
||||
"is on %s: move it first", plan.Node, need.For, need.Name, was, now, was)
|
||||
} else {
|
||||
o.Token, o.Kind, o.Severity = kindBindingMoved, kindBindingMoved, conditions.Urgent
|
||||
o.Summary = fmt.Sprintf("on %s, %s's %s would be sent %s, and it is bound to %s, where its data is, "+
|
||||
"with no pin naming %s — a move nothing asked for", plan.Node, need.For, need.Name, now, was, now)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// novox/hq issue 273, ADR 0232: a consumer of a provision that keeps its data moves only by a pin.
|
||||
|
||||
func storeManifests() []catalogue.Manifest {
|
||||
return []catalogue.Manifest{
|
||||
{Module: "store", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
|
||||
{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
|
||||
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
|
||||
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
|
||||
}
|
||||
}
|
||||
|
||||
func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) catalogue.Needed {
|
||||
t.Helper()
|
||||
for _, n := range plan.Needs {
|
||||
if n.For == consumer && n.Name == provision {
|
||||
return n
|
||||
}
|
||||
}
|
||||
t.Fatalf("no %s for %s: %+v", provision, consumer, plan.Needs)
|
||||
return catalogue.Needed{}
|
||||
}
|
||||
|
||||
// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
|
||||
// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
|
||||
// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
|
||||
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
assignAll := func(pairs ...[2]string) {
|
||||
for _, a := range pairs {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
|
||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
||||
[2]string{"laptop", "board"})
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
|
||||
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
|
||||
}
|
||||
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
|
||||
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
|
||||
}
|
||||
|
||||
// Sent, and recorded: only the binding to data.
|
||||
bindings := boundToData(plan, nil)
|
||||
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
|
||||
t.Fatalf("recorded %+v", bindings)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
|
||||
}
|
||||
|
||||
// The laptop's store taken away: refused, not moved to the anchor's empty one.
|
||||
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
|
||||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
|
||||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
|
||||
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
|
||||
}
|
||||
|
||||
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
found, err := probeBindings(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
|
||||
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := inv.Bindings(ctx)
|
||||
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
|
||||
t.Fatalf("%+v, %v", all, err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a move sent is still said: %+v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What one machine's resolution says against its record.
|
||||
func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
|
||||
home, anchor := catalogue.Chosen{Node: "home", Module: "store"}, catalogue.Chosen{Node: "anchor", Module: "store"}
|
||||
plan := catalogue.Resolution{Node: "laptop",
|
||||
Kept: []catalogue.KeptBinding{{Machine: "laptop", Consumer: "board", Provision: "postgres-database",
|
||||
Bound: home, Would: anchor}},
|
||||
Needs: []catalogue.Needed{
|
||||
{Name: "postgres-database", For: "board", From: "home", Module: "store", KeepsData: true},
|
||||
{Name: "postgres-database", For: "game", From: "anchor", Module: "store", KeepsData: true},
|
||||
{Name: "wildcard-resolution", For: "network", From: "anchor", Module: "resolver"},
|
||||
}}
|
||||
bound := map[string]map[string]catalogue.Chosen{
|
||||
"board": {"postgres-database": home},
|
||||
"game": {"postgres-database": home},
|
||||
"network": {"wildcard-resolution": {Node: "home", Module: "resolver"}},
|
||||
}
|
||||
found := linted(bindingFindings(plan, bound, nil))
|
||||
if len(found) != 2 {
|
||||
t.Fatalf("found %+v", found)
|
||||
}
|
||||
kept, moved := found[0], found[1]
|
||||
if kept.Kind != kindBindingKept || kept.Severity != conditions.Urgent || kept.Machine != "laptop" ||
|
||||
!strings.Contains(kept.Summary, "would move board's postgres-database from home/store to anchor/store") ||
|
||||
!strings.Contains(kept.Summary, "its data is on home/store") ||
|
||||
!strings.Contains(kept.Summary, "`pin laptop postgres-database anchor store` to confirm a move (and move the data first)") {
|
||||
t.Errorf("kept: %+v", kept)
|
||||
}
|
||||
if moved.Kind != kindBindingMoved || moved.Severity != conditions.Urgent ||
|
||||
!strings.Contains(moved.Summary, "game's postgres-database would be sent anchor/store") {
|
||||
t.Errorf("moved: %+v", moved)
|
||||
}
|
||||
if kept.Key() == moved.Key() {
|
||||
t.Error("two consumers, one condition")
|
||||
}
|
||||
}
|
||||
|
||||
// A push says the move it refused, and raises its condition at once.
|
||||
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
|
||||
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
|
||||
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
|
||||
open[0].Source != probeBindingsID {
|
||||
t.Fatalf("raised %+v", open)
|
||||
}
|
||||
}
|
||||
+210
-18
@@ -6,7 +6,9 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -147,6 +149,13 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
||||
// rebuild the graph rather than a list of names.
|
||||
Path: result.Path,
|
||||
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
||||
SourceFingerprint: result.SourceFingerprint,
|
||||
}
|
||||
// When it was asked, which is what orders it against another build of the same module
|
||||
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
||||
if asked, ok := link.BuildAskedAt(result.ID); ok {
|
||||
kept.Asked = asked
|
||||
}
|
||||
for _, ref := range result.Against {
|
||||
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
||||
@@ -387,34 +396,47 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
//
|
||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
||||
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
||||
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
||||
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
||||
dryRun bool) (string, error) {
|
||||
if dryRun && wait != 0 {
|
||||
return "", errors.New("a dry run waited for is `build --dry-run`")
|
||||
}
|
||||
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
||||
// the reason, rather than sent to a machine to fail at `git clone`.
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
// Correlated by something the control plane makes, not by the module's name: two builds of one
|
||||
// module can be in flight, and the second answer is not the first one's.
|
||||
request := link.BuildRequest{
|
||||
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Path: path,
|
||||
Ref: ref,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
DryRun: dryRun,
|
||||
}
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
@@ -430,11 +452,13 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
ask, err := askOver(server)
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
fmt.Printf(" of %s\n", seat)
|
||||
|
||||
if wait == 0 {
|
||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||
@@ -442,26 +466,31 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
||||
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
|
||||
// follows the build by its id instead.
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
||||
"its outcome is not taken in\n", request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
|
||||
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
|
||||
return nil
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
result, err := ask.Submit(ctx, request, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
return request.ID, err
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return request.ID, err
|
||||
}
|
||||
defer open.Close()
|
||||
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
||||
if err != nil {
|
||||
return err
|
||||
return request.ID, err
|
||||
}
|
||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||
for _, made := range kept.Made {
|
||||
@@ -471,7 +500,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
||||
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
|
||||
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
||||
return nil
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
|
||||
@@ -524,20 +553,108 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
BuiltFrom: result.Commit, Head: result.Commit,
|
||||
// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
|
||||
Against: kept.Against,
|
||||
// When it was asked, so an older request heard later does not replace a newer one
|
||||
// (novox/hq 04-ISSUES/219).
|
||||
Asked: kept.Asked,
|
||||
}
|
||||
if result.Source != nil && result.Source.Seat != "" {
|
||||
recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
|
||||
}
|
||||
// **A build at a commit does not change the branch a module follows** (novox/hq 04-ISSUES/215):
|
||||
// the commit is built and recorded as what it was built from, and the module keeps following
|
||||
// what it followed before — the repository's default branch for one new to the catalogue.
|
||||
if followedBranch(result.Ref) == "" && result.Ref != "" {
|
||||
recorded.Ref = ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
recorded.Ref = followedBranch(was.Ref)
|
||||
}
|
||||
}
|
||||
if err := namesNoInstallation(manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
result.On, result.Repository, short(result.Commit), err)
|
||||
}
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
|
||||
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
|
||||
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
|
||||
// The branch the module already follows is its trunk — never the branch a build was asked at, or a
|
||||
// build of a feature branch by name would make that branch its trunk.
|
||||
follows := ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
follows = followedBranch(was.Ref)
|
||||
}
|
||||
if err := publishable(result, follows); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
||||
return manifest, kept, err
|
||||
} else if failed {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
||||
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
|
||||
short(result.Commit))
|
||||
}
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
||||
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
||||
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
||||
// byte-reproducible, and the bus rebuilt for another module's merge demanded a planned upgrade.
|
||||
if kept.SourceFingerprint != "" {
|
||||
stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID)
|
||||
if err != nil {
|
||||
return manifest, kept, err
|
||||
}
|
||||
if stands != "" && stands != kept.ID && len(raw) > 0 {
|
||||
if same, err := catalogue.ParseManifest(raw); err == nil && same.Module == manifest.Module {
|
||||
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
|
||||
manifest.Module, short(result.Commit), stands)
|
||||
manifest = same
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||
if errors.Is(err, inventory.ErrSuperseded) {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
|
||||
result.On, manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||
// worked and the module is registered.
|
||||
collect(ctx, inv)
|
||||
return manifest, kept, nil
|
||||
}
|
||||
|
||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
||||
|
||||
// publishable says whether a build's outcome may become a module's version: its commit on the module's
|
||||
// trunk, as the build seat read the forge at the build — the branch the module follows when its source
|
||||
// names one, else its repository's default branch. A build seat that could not say — one older than the
|
||||
// rule, building its own successor — is let through and said, so the rule can reach the mesh.
|
||||
func publishable(result link.BuildResult, follows string) error {
|
||||
if result.Check != nil || result.Checked != nil || result.DryRun {
|
||||
return errors.New("a check or a dry run is never published")
|
||||
}
|
||||
if result.Trunk == "" {
|
||||
fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+
|
||||
"registered as before\n", result.ID, short(result.Commit))
|
||||
return nil
|
||||
}
|
||||
trunk := result.Trunk
|
||||
if follows != "" {
|
||||
trunk = follows
|
||||
}
|
||||
on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk)
|
||||
if !on {
|
||||
return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildAndShow builds and prints the manifest without recording anything.
|
||||
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
@@ -555,16 +672,17 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
ask, err := askOver(server)
|
||||
ask, err := askOverOn(buildSeatHeld(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
|
||||
result, err := ask.Submit(ctx, link.BuildRequest{
|
||||
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository, Path: path, Ref: ref,
|
||||
Held: heldBy(ctx), Seats: seatBases(ctx),
|
||||
DryRun: true,
|
||||
}, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -600,6 +718,8 @@ type answers struct {
|
||||
reported []inventory.Reported
|
||||
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
||||
plans []inventory.Plan
|
||||
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||
paused pauseView
|
||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||
// other answer here: those are about a machine that was told something, and this is about one
|
||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||
@@ -623,6 +743,29 @@ type answers struct {
|
||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||
untaken map[string]map[string]int
|
||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||
unheld []catalogue.Unheld
|
||||
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
|
||||
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
|
||||
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
|
||||
// not be read when they could not — never read as none.
|
||||
conditions []conditions.Condition
|
||||
conditionsUnread string
|
||||
// overflowing is every module whose identity overflows the bound of a provision it requires
|
||||
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||
// this is the one place it is said across the mesh. Not well while there is any.
|
||||
overflowing []catalogue.Overflow
|
||||
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||
handActs *int
|
||||
handActsUnread string
|
||||
// heals is what the healers did in the last seven days (novox/hq to-be 45 §7): acts, and conditions
|
||||
// handed to the operator; healsUnread why it could not be read.
|
||||
heals *healsCount
|
||||
healsUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
@@ -639,12 +782,14 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
|
||||
"base will be told that base is missing: %v\n", err)
|
||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.inventory.Held(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
@@ -668,12 +813,56 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOver(_ *link.Server) (link.Builders, error) {
|
||||
func askOverOn(seat string) (link.Builders, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.BuildsOverNATS(address)
|
||||
return link.BuildsOverNATSOn(address, seat)
|
||||
}
|
||||
|
||||
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
|
||||
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
|
||||
// moment the first build-agent is assigned — and a controller that asked the new role before then
|
||||
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
|
||||
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
|
||||
func buildSeatHeld(ctx context.Context) string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
return buildSeatAmong(entries)
|
||||
}
|
||||
|
||||
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
|
||||
// assigned module claims it; else the retired role while an assigned module still claims that; else
|
||||
// the current role, which is where every ask goes once the handover is done.
|
||||
func buildSeatAmong(entries []inventory.Entry) string {
|
||||
heldBefore := false
|
||||
for _, e := range entries {
|
||||
if len(e.On) == 0 {
|
||||
continue
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
|
||||
heldBefore = true
|
||||
}
|
||||
}
|
||||
if heldBefore {
|
||||
return link.TheBuildMachineBefore
|
||||
}
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
|
||||
// buildLog prints everything a build machine said about one build, read back from the bus.
|
||||
@@ -693,10 +882,13 @@ func buildLog(ctx context.Context, id string) error {
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
sub, err := js.Context().PullSubscribe(link.BuildLog(id), "",
|
||||
// Under whichever build role did it: a build asked of the retired role during the handover
|
||||
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
|
||||
lines := link.BuildLogOf("*", id)
|
||||
sub, err := js.Context().PullSubscribe(lines, "",
|
||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read %s from the bus: %w", link.BuildLog(id), err)
|
||||
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func claiming(module, seat string, on ...string) inventory.Entry {
|
||||
return inventory.Entry{
|
||||
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
|
||||
On: on,
|
||||
}
|
||||
}
|
||||
|
||||
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
|
||||
// one while only the builder is assigned, the current one from the first build-agent on, and the
|
||||
// current one when nothing holds either — where every ask goes once the handover is done.
|
||||
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
|
||||
onlyTheBuilder := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
|
||||
}
|
||||
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
|
||||
t.Errorf("with only the builder assigned, asked %q", got)
|
||||
}
|
||||
bothHeld := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine, "home-server"),
|
||||
}
|
||||
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
|
||||
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
|
||||
}
|
||||
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
|
||||
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
|
||||
t.Errorf("with no holder of either, asked %q, want the current role", got)
|
||||
}
|
||||
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
|
||||
t.Errorf("an empty catalogue asks %q", got)
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -63,3 +66,87 @@ func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the
|
||||
// module keeps following the branch it followed — a new one, the default branch.
|
||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||
result := func(id, ref, commit string) link.BuildResult {
|
||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-1", "main", "1111111aaaa")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||
}
|
||||
|
||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||
r.Manifest, r.Path = other, "modules/letta"
|
||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered,
|
||||
// so a push sends what the newer request built.
|
||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
result := func(asked time.Time, image string) link.BuildResult {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result(newer, "4bcd5f73")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||
t.Errorf("the late build was not recorded: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||
at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC)
|
||||
if got, ok := link.BuildAskedAt(link.NewBuildID(at)); !ok || !got.Equal(at) {
|
||||
t.Errorf("read back %v %v; want %v", got, ok, at)
|
||||
}
|
||||
if got, ok := link.BuildAskedAt("build-1791064317392539762"); !ok || got.Format(time.TimeOnly) != "21:51:57" {
|
||||
t.Errorf("the incident's id reads as %v %v", got, ok)
|
||||
}
|
||||
for _, id := range []string{"b-1", "build-2", "build-", "build-x", ""} {
|
||||
if _, ok := link.BuildAskedAt(id); ok {
|
||||
t.Errorf("%q read as a request time", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236).
|
||||
//
|
||||
// **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the
|
||||
// controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything,
|
||||
// and a version whose data format moved (2.10 → 2.11) cannot be undone by putting the old one back.
|
||||
// So nothing sends a new bus build on its own: its policy is `record` whatever anyone says (the
|
||||
// catalogue's DerivedUpgrade, and `upgrade` refuses a roll-out), a plan builds it and sends nothing, a
|
||||
// cascade holds the machine (ADR 0221), and a push naming that machine is refused while a new bus build
|
||||
// waits for it (busHeld). The one way is this verb: a person, with why, the streams snapshotted first,
|
||||
// whether it can be reverted said before it starts, the step said as `bus-maintenance` while it runs,
|
||||
// and every stream, durable consumer and a round trip checked after (H-bus) — or the step said failed,
|
||||
// with its snapshot as the way back.
|
||||
|
||||
// busStepProbe is the registry's row for the step; its kinds.
|
||||
const (
|
||||
busStepProbe = "DB"
|
||||
kindBusMaintenance = "bus-maintenance"
|
||||
kindBusUpgradeFailed = "bus-upgrade-failed"
|
||||
)
|
||||
|
||||
// busStepBound is how long after its start a bus upgrade must be followed by a healthy bus.
|
||||
var busStepBound = 15 * time.Minute
|
||||
|
||||
// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is: the bus
|
||||
// machine's backup holder backs the bus module up now, whose dump is the streams' snapshot (novox/hq ADR
|
||||
// 0235). A variable so a test takes none. A person who took one by hand says where with --snapshot-taken,
|
||||
// and then none is taken — for a bus whose module does not yet carry the snapshot program.
|
||||
var takeBusSnapshot = snapshotTheBusNow
|
||||
|
||||
// busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per
|
||||
// machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds
|
||||
// no bus module.
|
||||
type busPending struct {
|
||||
module string
|
||||
machines []string
|
||||
from map[string]string
|
||||
to string
|
||||
// same are the commits whose build was made from the same source as the build the mesh holds, or
|
||||
// made the same artifacts and manifest (novox/hq issue 280).
|
||||
same map[string]bool
|
||||
}
|
||||
|
||||
// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the
|
||||
// two builds were made from the same source, or made the same artifacts from the same manifest — a
|
||||
// rebuild of the same source for another module's merge changes nothing the machine runs, whatever
|
||||
// image digest it made (novox/hq issue 280).
|
||||
func (b busPending) moves(machine string) bool {
|
||||
from, known := b.from[machine]
|
||||
if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) {
|
||||
return false
|
||||
}
|
||||
return !known || !b.same[from]
|
||||
}
|
||||
|
||||
// pendingBus reads what a bus upgrade would do.
|
||||
func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, error) {
|
||||
var b busPending
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for name, m := range shelf {
|
||||
if catalogue.ProvidesBus(m) {
|
||||
b.module = name
|
||||
}
|
||||
}
|
||||
if b.module == "" {
|
||||
return b, nil
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.to = current[b.module].Commit
|
||||
if b.machines, err = inv.Running(ctx, b.module); err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.from, b.same = map[string]string{}, map[string]bool{}
|
||||
made, err := inv.BuildFingerprints(ctx, b.module)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for commit, refs := range made {
|
||||
b.same[commit] = refs != "" && refs == made[b.to]
|
||||
}
|
||||
sources, err := inv.BuildSourceFingerprints(ctx, b.module)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for commit, src := range sources {
|
||||
if src != "" && src == sources[b.to] {
|
||||
b.same[commit] = true
|
||||
}
|
||||
}
|
||||
for _, n := range b.machines {
|
||||
sent, known, err := inv.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
if known {
|
||||
if c, carried := sent[b.module]; carried {
|
||||
b.from[n] = c
|
||||
}
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// busHeld names the machines a push may not send because sending them would replace the bus: the
|
||||
// planned step's, not a push's (ADR 0236). Said with the remedy.
|
||||
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
|
||||
b, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, n := range machines {
|
||||
for _, holder := range b.machines {
|
||||
if n == holder && b.moves(n) {
|
||||
out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+
|
||||
"`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)",
|
||||
n, b.module, short(orNotKnown(b.from[n])), short(b.to))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busCommand is `bus` — what a bus upgrade would do and how the last went — and `bus upgrade`.
|
||||
func busCommand(ctx context.Context, args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
||||
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
||||
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
||||
irreversible := set.Bool("irreversible", false, "the new version cannot be undone by putting the old one back, "+
|
||||
"and this is the person's explicit word that it runs anyway")
|
||||
why := addHandActFlags(set)
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>]]")
|
||||
}
|
||||
switch sub {
|
||||
case "":
|
||||
return busStatus(ctx)
|
||||
case "upgrade":
|
||||
default:
|
||||
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub)
|
||||
}
|
||||
// Everything refused before anything is done.
|
||||
if err := why.require("bus upgrade"); err != nil {
|
||||
return err
|
||||
}
|
||||
if *reversible == *irreversible {
|
||||
return errors.New("bus upgrade says, before it starts, whether the new version can be undone by putting the " +
|
||||
"old one back: --reversible, or --irreversible as your explicit word that it runs anyway (to-be 45 §8). " +
|
||||
"Nothing was done")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
b, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.module == "" {
|
||||
return errors.New("the mesh holds no module that provides its bus: there is nothing to upgrade")
|
||||
}
|
||||
var moving []string
|
||||
for _, n := range b.machines {
|
||||
if b.moves(n) {
|
||||
moving = append(moving, n)
|
||||
}
|
||||
}
|
||||
if len(moving) == 0 {
|
||||
fmt.Printf("every machine running %s runs the build the mesh holds (%s): nothing to upgrade\n", b.module, short(b.to))
|
||||
return nil
|
||||
}
|
||||
where := strings.TrimSpace(*snapshot)
|
||||
if where == "" {
|
||||
for _, n := range moving {
|
||||
fmt.Printf("snapshotting the bus's streams on %s first (its backup holder, ADR 0235)…\n", n)
|
||||
if where, err = takeBusSnapshot(ctx, b.module, n); err != nil {
|
||||
return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w — a snapshot "+
|
||||
"taken by hand is said with --snapshot-taken <where>", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
from := map[string]bool{}
|
||||
for _, n := range moving {
|
||||
from[orNotKnown(b.from[n])] = true
|
||||
}
|
||||
step, err := inv.StartBusStep(ctx, inventory.BusStep{Module: b.module, Machines: moving,
|
||||
From: strings.Join(sortedKeys(from), ", "), To: b.to, Snapshot: where, Reversible: *reversible,
|
||||
By: link.Caller(), Why: strings.TrimSpace(*why.why)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cause := "bus-upgrade"
|
||||
if strings.TrimSpace(*why.cause) == "" {
|
||||
why.cause = &cause
|
||||
}
|
||||
why.record(ctx, "bus upgrade", append([]string{b.module}, moving...))
|
||||
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
|
||||
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
|
||||
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
|
||||
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
|
||||
if err != nil {
|
||||
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
|
||||
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
|
||||
}
|
||||
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
||||
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
|
||||
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busStatus is `bus`: what an upgrade would do, and the last step.
|
||||
func busStatus(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
b, err := pendingBus(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.module == "" {
|
||||
fmt.Println("the mesh holds no module that provides its bus")
|
||||
} else {
|
||||
fmt.Printf("the bus is %s, built %s, on %s; never rolled out — a planned step (`bus upgrade`)\n", b.module,
|
||||
short(b.to), orNone(strings.Join(b.machines, ", ")))
|
||||
for _, n := range b.machines {
|
||||
state := "runs it"
|
||||
if b.moves(n) {
|
||||
state = "runs " + short(orNotKnown(b.from[n])) + ": `bus upgrade` replaces it"
|
||||
}
|
||||
fmt.Printf(" %-10s %s\n", n, state)
|
||||
}
|
||||
}
|
||||
fmt.Println(" `bus upgrade` has the bus machine's backup holder snapshot the streams first (ADR 0235)")
|
||||
s, found, err := open.inventory.LatestBusStep(ctx)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
state := "running since " + s.Started.Local().Format("2006-01-02 15:04")
|
||||
if s.Ended != nil {
|
||||
state = s.Outcome + " at " + s.Ended.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
fmt.Printf("last step %d: %s → %s on %s by %s (%s): %s; snapshot %s\n", s.ID, s.From, short(s.To),
|
||||
strings.Join(s.Machines, ", "), orNone(s.By), s.Why, state, s.Snapshot)
|
||||
if s.Found != "" {
|
||||
fmt.Printf(" %s\n", s.Found)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeBusStep is DB: a bus upgrade running is said as `bus-maintenance`; the bus healthy again after
|
||||
// the machines reported the new build ends it done; past its bound, unhealthy, it ends failed and is
|
||||
// said — urgent, with its snapshot — while the bus is still not healthy.
|
||||
func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found {
|
||||
return nil, err
|
||||
}
|
||||
if s.Ended != nil && s.Outcome != "failed" {
|
||||
return nil, nil
|
||||
}
|
||||
problems, err := busHealth(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applied := true
|
||||
for _, r := range reports {
|
||||
for _, n := range s.Machines {
|
||||
if r.Node == n && (!r.Current || r.Outcome != inventory.OutcomeApplied || r.At == nil || r.At.Before(s.Started)) {
|
||||
applied = false
|
||||
problems = append(problems, n+" has not reported the new bus applied")
|
||||
}
|
||||
}
|
||||
}
|
||||
id := s.Module
|
||||
if s.Ended == nil {
|
||||
switch {
|
||||
case applied && len(problems) == 0:
|
||||
return nil, inv.EndBusStep(ctx, s.ID, "done", "the bus answered healthy after the upgrade")
|
||||
case time.Since(s.Started) > busStepBound:
|
||||
found := strings.Join(problems, "; ")
|
||||
if err := inv.EndBusStep(ctx, s.ID, "failed", found); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Found = found
|
||||
default:
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "maintenance",
|
||||
Kind: kindBusMaintenance, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus is being upgraded (step %d, %s → %s on %s, by %s: %s); its snapshot is %s",
|
||||
s.ID, s.From, short(s.To), strings.Join(s.Machines, ", "), orNone(s.By), s.Why, s.Snapshot),
|
||||
Said: orNone(strings.Join(problems, "; "))}}, nil
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil // failed, and healthy since: nothing wrong now
|
||||
}
|
||||
way := "put the old build back"
|
||||
if !s.Reversible {
|
||||
way = "restore the snapshot"
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "upgrade-failed",
|
||||
Kind: kindBusUpgradeFailed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the bus upgrade (step %d, %s → %s) did not end healthy within %s: %s — the way back is to %s (%s)",
|
||||
s.ID, s.From, short(s.To), busStepBound, strings.Join(problems, "; "), way, s.Snapshot)}}, nil
|
||||
}
|
||||
|
||||
func sortedKeys(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// busSnapshotWithin is how long the bus machine's backup holder is given to take the bus's snapshot.
|
||||
var busSnapshotWithin = 15 * time.Minute
|
||||
|
||||
// snapshotTheBusNow asks the bus machine's backup holder to back the bus module up now — its dump is
|
||||
// the streams' snapshot (novox/hq ADR 0235) — and waits until it says a backup newer than the ask:
|
||||
// where the snapshot is, as a person reads it. The serving controller's connection, or one of its own.
|
||||
func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) {
|
||||
var where string
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
asked := time.Now()
|
||||
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", node,
|
||||
map[string]any{"module": module}, 30*time.Second)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s's backup holder would not take the bus's snapshot: %s", node, answer.Error)
|
||||
}
|
||||
deadline := time.Now().Add(busSnapshotWithin)
|
||||
for {
|
||||
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", node, map[string]any{}, 10*time.Second)
|
||||
if err == nil && answer.Error == "" {
|
||||
if measured, err := readHolder(answer.Result); err == nil {
|
||||
for item, m := range measured[module] {
|
||||
if m.LastBackup != nil && m.LastBackup.After(asked) && m.Error == "" {
|
||||
where = fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, item,
|
||||
m.LastBackup.UTC().Format(time.RFC3339))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s; the bus is "+
|
||||
"not replaced", node, busSnapshotWithin)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
}
|
||||
})
|
||||
return where, err
|
||||
}
|
||||
|
||||
// errBusWaits is a send refused because it would replace the bus outside its planned step.
|
||||
var errBusWaits = errors.New("a new bus build waits for its planned step")
|
||||
|
||||
type busStepKey struct{}
|
||||
|
||||
// withBusStep marks a send as the bus's planned step: the one send that may replace the bus.
|
||||
func withBusStep(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, busStepKey{}, true)
|
||||
}
|
||||
|
||||
func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on }
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
|
||||
// D6 and D7).
|
||||
//
|
||||
// **What the controller asserts at its start and what the self-check expects to find are one
|
||||
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
|
||||
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
|
||||
// the drift rather than the fault.
|
||||
|
||||
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
|
||||
// can now hear a declaration.
|
||||
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
if err := broker.Raise(r, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||
// consumer nothing had created (2026-09-28).
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return nil, errors.Join(failed...)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// What raises the condition a send says when the objects it implies could not be asserted, and its kind.
|
||||
const (
|
||||
sourceBusObjects = "bus-objects"
|
||||
kindBusObjectsUnasserted = "bus-objects-unasserted"
|
||||
)
|
||||
|
||||
// assertOnSend asserts, on the bus a send is about to use, every object assertBusObjects derives —
|
||||
// **whenever a declaration is sent, not only when the controller starts** (novox/hq issue 208).
|
||||
//
|
||||
// A module assigned after the controller started was sent its declaration and found no consumer to
|
||||
// bind (`consumer not found`, messenger on 2026-10-06), and a seat holder assigned after it found no
|
||||
// worker: the objects a declaration implies were asserted at start and nowhere else, so they existed
|
||||
// only for what was assigned before the last restart. The same derivation, not a second list of what
|
||||
// a send needs: what start asserts, the self-check expects and a send asserts are one answer. Every
|
||||
// part is idempotent, so asserting the whole of it again is the no-op a restart already relies on.
|
||||
//
|
||||
// **A failure is said and raised, and the send goes on.** The objects are the mesh's, not the
|
||||
// machines' being sent: holding every machine back for one consumer that none of them may use would
|
||||
// turn one fault into all of them, and the declarations are not what is wrong. It is never silent —
|
||||
// said in the send's own output and raised as a condition, which the next send that asserts them
|
||||
// clears — and the start-time raise still refuses to serve without them.
|
||||
func assertOnSend(ctx context.Context, inv *inventory.Inventory, r broker.Raiser, indent string) error {
|
||||
_, err := assertBusObjects(ctx, inv, r)
|
||||
var observed []conditions.Observation
|
||||
if err != nil {
|
||||
fmt.Printf("%sTHE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES: %v\n", indent, err)
|
||||
fmt.Printf("%s a module may find no consumer to bind, or a holder no worker; sent anyway, raised as "+
|
||||
"condition %s, and asserted again by the next send\n", indent, unassertedObservation(err).Key())
|
||||
observed = append(observed, unassertedObservation(err))
|
||||
}
|
||||
// Observed when it failed, cleared when it did not: a send that asserted everything is the
|
||||
// observation that the bus holds what it should.
|
||||
if kerr := withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
return k.Reconcile(ctx, sourceBusObjects, observed)
|
||||
}); kerr != nil {
|
||||
fmt.Printf("%sand whether the bus holds what this send implies could not be kept as a condition: %v\n",
|
||||
indent, kerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// unassertedObservation is a send's failure to assert the bus's objects, as a condition.
|
||||
func unassertedObservation(err error) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "objects", Token: "unasserted",
|
||||
Kind: kindBusObjectsUnasserted, Severity: conditions.Warning, Source: sourceBusObjects,
|
||||
Summary: "the bus's streams and consumers could not be asserted when a declaration was sent: " +
|
||||
"a module may find no consumer to bind, or a seat's holder no worker",
|
||||
Said: err.Error()}
|
||||
}
|
||||
|
||||
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return broker.ConsumersOf(users), nil
|
||||
}
|
||||
|
||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
return len(consumers), err
|
||||
}
|
||||
|
||||
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
|
||||
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||
var rec recordingRaiser
|
||||
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
|
||||
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
|
||||
}
|
||||
return rec.streams, rec.consumers, nil
|
||||
}
|
||||
|
||||
// recordingRaiser keeps what it was asked to assert and asserts nothing.
|
||||
type recordingRaiser struct {
|
||||
streams []broker.Stream
|
||||
consumers []broker.Consumer
|
||||
}
|
||||
|
||||
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
|
||||
r.streams = append(r.streams, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||
r.consumers = append(r.consumers, c)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
// only when the controller starts.
|
||||
|
||||
// aCarriedConsumer is the runtime on laptop and, carried by it, a module that consumes an event: the
|
||||
// shape of messenger on 2026-10-06, assigned after the controller started.
|
||||
func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}})
|
||||
register(t, open, catalogue.Manifest{Module: "messenger", Version: "1",
|
||||
Consumes: []string{"billing.order.placed"}})
|
||||
for _, m := range []string{catalogue.RuntimeModule, "messenger"} {
|
||||
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
consumers, err := moduleConsumers(t.Context(), open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if c.Module == "messenger" && c.Node == "laptop" {
|
||||
return c.Consumer
|
||||
}
|
||||
}
|
||||
t.Fatalf("messenger on laptop is derived no consumer: %+v", consumers)
|
||||
return broker.Consumer{}
|
||||
}
|
||||
|
||||
// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
|
||||
// started, and the worker its seat's queue should have for it.
|
||||
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
|
||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range inventory.MeshSeats() {
|
||||
if s.Name == "node-build-agent" {
|
||||
c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
|
||||
if !needed {
|
||||
t.Fatal("the build agent's seat needs no worker")
|
||||
}
|
||||
return c
|
||||
}
|
||||
}
|
||||
t.Fatal("the mesh declares no build agent's seat")
|
||||
return broker.Consumer{}
|
||||
}
|
||||
|
||||
// asserted says whether a recording holds a consumer by stream and name.
|
||||
func asserted(rec *recordingRaiser, want broker.Consumer) bool {
|
||||
for _, c := range rec.consumers {
|
||||
if c.Stream == want.Stream && c.Name == want.Name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// What a send asserts includes what was assigned after the start's assertion: a carried module's
|
||||
// consumer and a late holder's worker. The derivation is the start's own, so this holds without a bus.
|
||||
func TestASendAssertsWhatWasAssignedAfterStart(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
var atStart recordingRaiser
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, &atStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
if asserted(&atStart, consumer) || asserted(&atStart, worker) {
|
||||
t.Fatal("the start asserted what was not yet assigned; the test proves nothing")
|
||||
}
|
||||
var onSend recordingRaiser
|
||||
if err := assertOnSend(t.Context(), open.inventory, &onSend, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !asserted(&onSend, consumer) {
|
||||
t.Fatalf("messenger's consumer %s on %s is not asserted by the send: %+v", consumer.Name, consumer.Stream, onSend.consumers)
|
||||
}
|
||||
if !asserted(&onSend, worker) {
|
||||
t.Fatalf("the late holder's worker %s on %s is not asserted by the send: %+v", worker.Name, worker.Stream, onSend.consumers)
|
||||
}
|
||||
}
|
||||
|
||||
// failingRaiser refuses one consumer by name and records everything it was asked.
|
||||
type failingRaiser struct {
|
||||
recordingRaiser
|
||||
refuse string
|
||||
}
|
||||
|
||||
func (f *failingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||
f.consumers = append(f.consumers, c)
|
||||
if c.Name == f.refuse {
|
||||
return errors.New("nats: API error: code=503 description=insufficient resources")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A send whose objects cannot be asserted says so in its output and raises a condition — and the next
|
||||
// send that asserts them clears it. The consumers after the refused one are still asked for.
|
||||
func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
failing := &failingRaiser{refuse: consumer.Name}
|
||||
|
||||
var sendErr error
|
||||
out := stdoutOf(t, func() error {
|
||||
sendErr = assertOnSend(t.Context(), open.inventory, failing, " ")
|
||||
return nil
|
||||
})
|
||||
if sendErr == nil || !strings.Contains(sendErr.Error(), "how messenger on laptop hears what it consumes") {
|
||||
t.Fatalf("the failure is not answered: %v", sendErr)
|
||||
}
|
||||
if !strings.Contains(out, "THE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES") ||
|
||||
!strings.Contains(out, "insufficient resources") || !strings.Contains(out, "bus.objects.unasserted") {
|
||||
t.Fatalf("the failure is not said in the send's output:\n%s", out)
|
||||
}
|
||||
if !asserted(&failing.recordingRaiser, worker) {
|
||||
t.Fatal("the seat's worker was not asked for")
|
||||
}
|
||||
c, open1, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted")
|
||||
if err != nil || !open1 {
|
||||
t.Fatalf("no condition raised: %v", err)
|
||||
}
|
||||
if c.Kind != kindBusObjectsUnasserted || c.Source != sourceBusObjects ||
|
||||
len(c.Evidence) == 0 || !strings.Contains(c.Evidence[0].Said, "insufficient resources") {
|
||||
t.Fatalf("the condition does not say what failed: %+v", c)
|
||||
}
|
||||
|
||||
// The next send asserts them, and that observation clears it.
|
||||
if err := assertOnSend(t.Context(), open.inventory, &recordingRaiser{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, stillOpen, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); err != nil || stillOpen {
|
||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
// The controller starting, before either was assigned.
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, nil)
|
||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
||||
t.Fatal("a send that asserted everything raised a condition")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
|
||||
// RequestTimeout) — the shortest wait of a caller the mesh ships.
|
||||
const consoleWaits = 30 * time.Second
|
||||
|
||||
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
|
||||
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
|
||||
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
|
||||
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
||||
if link.AnswerWithin >= consoleWaits/2 {
|
||||
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
|
||||
}
|
||||
if link.AnswerWithin >= broker.ResponseTTL {
|
||||
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
|
||||
}
|
||||
}
|
||||
|
||||
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
||||
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
||||
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want bool
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
||||
}
|
||||
if got := answersFirst(argv); got != c.want {
|
||||
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
|
||||
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
|
||||
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
|
||||
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
|
||||
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
|
||||
// planner, so the two can be compared.
|
||||
|
||||
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
|
||||
type policyOf func(module string) (inventory.Upgrade, bool)
|
||||
|
||||
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
|
||||
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
|
||||
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
|
||||
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
machines := map[string]*link.MachinePlan{}
|
||||
machine := func(name string) *link.MachinePlan {
|
||||
if machines[name] == nil {
|
||||
machines[name] = &link.MachinePlan{Machine: name}
|
||||
}
|
||||
return machines[name]
|
||||
}
|
||||
for _, tier := range r.Plan.Tiers {
|
||||
for _, name := range tier {
|
||||
e, held := byName[name]
|
||||
if !held {
|
||||
continue
|
||||
}
|
||||
u, known := policy(name)
|
||||
waits := known && !u.RollOut
|
||||
for _, on := range e.On {
|
||||
if waits {
|
||||
machine(on).Waits = append(machine(on).Waits, name)
|
||||
} else {
|
||||
machine(on).Receives = append(machine(on).Receives, name)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case name == "nats":
|
||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
||||
case waits && len(e.On) > 0:
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
||||
orNone(u.From)))
|
||||
}
|
||||
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
|
||||
var offers []string
|
||||
for _, o := range e.Manifest.Provides {
|
||||
offers = append(offers, o.Name)
|
||||
}
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
|
||||
name, strings.Join(offers, ", ")))
|
||||
}
|
||||
if e.Manifest.Data != nil && len(e.On) > 0 {
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
|
||||
}
|
||||
}
|
||||
}
|
||||
var names []string
|
||||
for name := range machines {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
p.Machines = append(p.Machines, *machines[name])
|
||||
}
|
||||
p.Summary = summaryOf(p)
|
||||
return p
|
||||
}
|
||||
|
||||
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
|
||||
func summaryOf(p link.ChangePlan) string {
|
||||
if len(p.Moved) == 0 && len(p.New) == 0 {
|
||||
return "builds nothing: the change touches no module of the mesh's graph"
|
||||
}
|
||||
var parts []string
|
||||
what := strings.Join(p.Moved, ", ")
|
||||
if len(p.Dependents) > 0 {
|
||||
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
|
||||
}
|
||||
if len(p.New) > 0 {
|
||||
if what != "" {
|
||||
what += ", "
|
||||
}
|
||||
what += "new: " + strings.Join(p.New, ", ")
|
||||
}
|
||||
var to []string
|
||||
for _, m := range p.Machines {
|
||||
if len(m.Receives) > 0 {
|
||||
to = append(to, m.Machine)
|
||||
}
|
||||
}
|
||||
if len(to) > 0 {
|
||||
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
|
||||
} else {
|
||||
parts = append(parts, "builds "+what+", sent nowhere")
|
||||
}
|
||||
bus := "no bus step"
|
||||
for _, s := range p.Steps {
|
||||
if strings.HasPrefix(s, "a planned bus step") {
|
||||
bus = "a bus step"
|
||||
}
|
||||
}
|
||||
parts = append(parts, bus)
|
||||
waiting := 0
|
||||
for _, m := range p.Machines {
|
||||
waiting += len(m.Waits)
|
||||
}
|
||||
if waiting > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// planText is a change plan as a person reads it, for the pull request's comment.
|
||||
func planText(p link.ChangePlan) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "change plan of %s at %.8s into %s: %s\n", p.Repository, p.Head, p.Base, p.Summary)
|
||||
for i, tier := range p.Tiers {
|
||||
fmt.Fprintf(&b, " tier %d: %s\n", i, strings.Join(tier, ", "))
|
||||
}
|
||||
for _, m := range p.Machines {
|
||||
line := " " + m.Machine + ": "
|
||||
if len(m.Receives) > 0 {
|
||||
line += "receives " + strings.Join(m.Receives, ", ")
|
||||
}
|
||||
if len(m.Waits) > 0 {
|
||||
if len(m.Receives) > 0 {
|
||||
line += "; "
|
||||
}
|
||||
line += "waits for a person: " + strings.Join(m.Waits, ", ")
|
||||
}
|
||||
b.WriteString(line + "\n")
|
||||
}
|
||||
for _, s := range p.Steps {
|
||||
b.WriteString(" - " + s + "\n")
|
||||
}
|
||||
if len(p.Unread) > 0 {
|
||||
fmt.Fprintf(&b, " read by no module's build: %s\n", strings.Join(p.Unread, ", "))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
|
||||
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
|
||||
// person, a provider whose consumers follow it.
|
||||
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name, path string, on ...string) inventory.Entry {
|
||||
e := fromRepo(name, catalogue_, path)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = on
|
||||
return e
|
||||
}
|
||||
nats := entry("nats", "modules/nats", "anchor")
|
||||
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
gitea := entry("gitea", "modules/gitea", "anchor")
|
||||
held := entry("photos", "modules/photos", "anchor", "laptop")
|
||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
||||
tools.On = []string{"anchor", "laptop"}
|
||||
entries := []inventory.Entry{nats, gitea, held, tools}
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
||||
policy := func(module string) (inventory.Upgrade, bool) {
|
||||
if module == "photos" {
|
||||
return inventory.Upgrade{RollOut: false, From: "a person"}, true
|
||||
}
|
||||
return inventory.Upgrade{RollOut: true}, true
|
||||
}
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
||||
}
|
||||
|
||||
p := plan("modules/gitea/index.ts")
|
||||
if p.Summary != "builds gitea → anchor; no bus step" {
|
||||
t.Errorf("one module's change reads %q", p.Summary)
|
||||
}
|
||||
|
||||
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
|
||||
if !strings.Contains(p.Summary, "a bus step") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
||||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
|
||||
t.Errorf("the bus and a held module read %q", p.Summary)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, m := range p.Machines {
|
||||
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
|
||||
}
|
||||
// The bus first, what stands on it after: the build plan's order, per machine.
|
||||
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
|
||||
t.Errorf("the deploy plan reads %v", got)
|
||||
}
|
||||
text := strings.Join(p.Steps, "\n")
|
||||
for _, want := range []string{"a planned bus step", "photos waits for a person", "nats provides mesh-bus"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("the steps do not say %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
p = plan("merge-check.sh")
|
||||
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
|
||||
t.Errorf("a root file's plan reads %+v", p)
|
||||
}
|
||||
if !strings.Contains(planText(p), "read by no module's build: merge-check.sh") {
|
||||
t.Errorf("the plan's text does not say why nothing is built:\n%s", planText(p))
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
@@ -24,7 +25,17 @@ import (
|
||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||
// refused what it could not see would teach people to ignore it.
|
||||
//
|
||||
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
||||
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||
// provider's machine when a real machine's name first meets the module's.
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||
}
|
||||
|
||||
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
if len(paths) == 0 {
|
||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||
"manifest of a repository together so the rules between them are checked too")
|
||||
@@ -73,6 +84,24 @@ func moduleCheck(paths []string, out io.Writer) error {
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||
// only the provider's manifest states.
|
||||
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
||||
sort.Strings(identities)
|
||||
for _, p := range identities {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(identities)
|
||||
|
||||
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
|
||||
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
|
||||
data := catalogue.DataProblems(shelf)
|
||||
sort.Strings(data)
|
||||
for _, p := range data {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(data)
|
||||
|
||||
var names []string
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
@@ -90,6 +119,25 @@ func moduleCheck(paths []string, out io.Writer) error {
|
||||
if len(m.Invokes) > 0 {
|
||||
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
||||
}
|
||||
// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
|
||||
if len(m.State) > 0 {
|
||||
kept := make([]string, 0, len(m.State))
|
||||
for _, s := range m.State {
|
||||
kept = append(kept, s.Name)
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||
}
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
|
||||
if items := m.DataItems(); len(items) > 0 {
|
||||
kept := make([]string, 0, len(items))
|
||||
for _, it := range items {
|
||||
kept = append(kept, it.ID+" ("+it.Class+")")
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if failed > 0 {
|
||||
@@ -97,7 +145,8 @@ func moduleCheck(paths []string, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||
"module not given here reads as unknown\n", len(paths))
|
||||
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
||||
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
|
||||
// runs it** (novox/hq issue 286): the same code (builder.Check), the same ask the controller would make of
|
||||
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
|
||||
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
|
||||
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
|
||||
// the mesh runs.
|
||||
//
|
||||
// **The build seat is the reference.** A merge-check.sh that passed on an agent's machine failed on the
|
||||
// seat for three reasons that were each the agent's environment, never the change: a newer Go whose gofmt
|
||||
// lays a file out differently, a sibling checkout at the agent's feature branch where the seat had the
|
||||
// commit the mesh runs, and a different user. Run here, a check sees what the seat will.
|
||||
//
|
||||
// check-here [--tree <checkout>] [--base main] [--registry <artifact store>] [--forge <url of the owner>]
|
||||
// [--number <n>] [--user uid:gid] [--facts store|<file>] [--keep]
|
||||
//
|
||||
// The checkout's HEAD is what is checked, and it must be committed: the seat checks a commit, never a
|
||||
// working tree.
|
||||
func checkHereCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("check-here", flag.ContinueOnError)
|
||||
tree := set.String("tree", ".", "the change's checkout; its HEAD is checked")
|
||||
base := set.String("base", "main", "the branch the change would merge into")
|
||||
registry := set.String("registry", os.Getenv("MESH_REGISTRY"), "the artifact store holding the facts and the toolchains")
|
||||
forge := set.String("forge", "", "where the repositories beside it are cloned from, as <forge>/<repository>.git; "+
|
||||
"the checkout's origin without its own name when not given")
|
||||
number := set.Int("number", 0, "the pull request's number, when there is one")
|
||||
// The build seat's service runs as root, and its check containers run as the builder does.
|
||||
user := set.String("user", "0:0", "the user the check's containers run as: the build seat's")
|
||||
keep := set.Bool("keep", false, "keep the workspace afterwards")
|
||||
factsFrom := set.String("facts", "store", "the facts snapshot: `store`, the one the artifact store holds — "+
|
||||
"what the seat reads — or a file")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *registry == "" {
|
||||
return errors.New("check-here reads the facts and the toolchains from the artifact store: --registry <host:port> or MESH_REGISTRY")
|
||||
}
|
||||
dir, err := filepath.Abs(*tree)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
git := func(args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.Output()
|
||||
return strings.TrimSpace(string(out)), err
|
||||
}
|
||||
if dirty, err := git("status", "--porcelain", "--untracked-files=no"); err != nil {
|
||||
return fmt.Errorf("%s is not a checkout: %w", dir, err)
|
||||
} else if dirty != "" {
|
||||
return errors.New("the checkout has changes not committed: the build seat checks a commit, so commit first")
|
||||
}
|
||||
head, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
origin, err := git("remote", "get-url", "origin")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the checkout has no origin to say which repository it is: %w", err)
|
||||
}
|
||||
owner, repo, prefix := ownerRepoOf(origin)
|
||||
if *forge == "" {
|
||||
*forge = prefix
|
||||
}
|
||||
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
|
||||
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
|
||||
}
|
||||
changedText, err := git("diff", "--name-only", "origin/"+*base+"...HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var paths, removed []string
|
||||
for _, p := range strings.Split(changedText, "\n") {
|
||||
if p = strings.TrimSpace(p); p == "" {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, p)
|
||||
if _, err := os.Stat(filepath.Join(dir, p)); os.IsNotExist(err) {
|
||||
removed = append(removed, p)
|
||||
}
|
||||
}
|
||||
|
||||
_ = os.Setenv("MESH_REGISTRY", *registry)
|
||||
f, err := readFacts(ctx, *factsFrom)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the facts snapshot cannot be read: %w", err)
|
||||
}
|
||||
entries, read, edges, err := graphOfFacts(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p := link.PullUpdated{Owner: owner, Repo: repo, Number: *number, Base: *base, Commit: head, Paths: paths,
|
||||
Removed: removed, ModuleDirs: moduleDirsIn(dir, paths), ModuleDirsSaid: true}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
_, scriptErr := os.Stat(filepath.Join(dir, builder.CheckScript))
|
||||
if !scope.gated() && !scope.Mesh {
|
||||
fmt.Printf("%s: %s, and the repository is not the mesh's: the build seat runs nothing for it\n",
|
||||
owner+"/"+repo, noModuleTouched)
|
||||
return nil
|
||||
}
|
||||
if !scope.gated() && scriptErr != nil {
|
||||
fmt.Printf("%s: %s; %s\n", owner+"/"+repo, noModuleTouched, noMergeCheck)
|
||||
return nil
|
||||
}
|
||||
|
||||
toolchains := map[string]string{}
|
||||
for language, reference := range f.Versions.Toolchains {
|
||||
toolchains[language] = catalogue.Rerouted(reference, *registry)
|
||||
}
|
||||
if len(toolchains) == 0 {
|
||||
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
|
||||
"issue 286, and the seat's toolchain cannot be known here")
|
||||
}
|
||||
beside := map[string]builder.Beside{}
|
||||
for d, ref := range f.Beside {
|
||||
from := d
|
||||
if d == "mesh-controller-main" {
|
||||
from = "mesh-controller"
|
||||
}
|
||||
beside[d] = builder.Beside{Repository: strings.TrimSuffix(*forge, "/") + "/" + from + ".git", Ref: ref}
|
||||
}
|
||||
id := fmt.Sprintf("check-here-%d", time.Now().UnixNano())
|
||||
spec := builder.CheckSpec{ID: id, Repository: dir, Ref: head, Owner: owner, Repo: repo, Number: *number,
|
||||
Paths: paths, Beside: beside, Modules: scope.Modules, New: scope.New, Manifests: scope.Manifests,
|
||||
Base: *base, Judge: scope.Judge, Toolchain: toolchains["go"], Toolchains: toolchains, User: *user}
|
||||
|
||||
workspace, err := os.MkdirTemp("", "mesh-check-here-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !*keep {
|
||||
defer removeWorkspace(spec.Toolchain, workspace, *user)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "checking %s/%s at %.8s as the build seat would, against the facts of %s, in %s\n",
|
||||
owner, repo, head, f.Taken.Format(time.RFC3339), workspace)
|
||||
v, err := builder.Check(ctx, builder.Command, spec, workspace, *registry, builder.GitCredential{},
|
||||
func(step, message string) {
|
||||
if step != "output" {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the check could not run — on the seat an error, never a pass: %w", err)
|
||||
}
|
||||
fmt.Println(v.Report)
|
||||
fmt.Println()
|
||||
fmt.Printf("mesh/merge-gate: %s — %s\n", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary)
|
||||
if v.Repo != nil {
|
||||
fmt.Printf("mesh/repo-check: %s — %s\n", strings.ToUpper(v.Repo.Verdict), v.Repo.Summary)
|
||||
}
|
||||
for _, l := range []*builder.Layer{v.Gate, v.Repo} {
|
||||
if l != nil && l.Verdict != "pass" && l.Verdict != "warning" {
|
||||
return errors.New("the build seat would not pass this change")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownerRepoOf reads owner, repository and the owner's URL from a remote: ssh://git@host:222/novox/mesh-host.git
|
||||
// → novox, mesh-host, ssh://git@host:222/novox.
|
||||
func ownerRepoOf(remote string) (string, string, string) {
|
||||
trimmed := strings.TrimSuffix(strings.TrimSuffix(remote, "/"), ".git")
|
||||
cut := strings.LastIndexAny(trimmed, "/:")
|
||||
if cut < 0 {
|
||||
return "", trimmed, ""
|
||||
}
|
||||
repo, prefix := trimmed[cut+1:], trimmed[:cut]
|
||||
owner := prefix
|
||||
if at := strings.LastIndexAny(prefix, "/:"); at >= 0 {
|
||||
owner = prefix[at+1:]
|
||||
}
|
||||
return owner, repo, prefix
|
||||
}
|
||||
|
||||
// removeWorkspace removes what the check left, written as the seat's user: by a container of that user
|
||||
// when it is not this one.
|
||||
func removeWorkspace(image, workspace, user string) {
|
||||
if image != "" && user != fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) {
|
||||
// Everything in it — the check's HOME is the workspace, so the toolchain's own files are there too.
|
||||
_ = exec.Command("docker", "run", "--rm", "--user", user, "--volume", workspace+":/workspace", image,
|
||||
"sh", "-c", "rm -rf /workspace/* /workspace/.[!.]*").Run()
|
||||
}
|
||||
_ = os.RemoveAll(workspace)
|
||||
}
|
||||
@@ -92,3 +92,14 @@ func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
|
||||
t.Fatalf("a name declared on purpose passes; got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged
|
||||
// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it —
|
||||
// refused it. Held here, so a verb added to the table without the manifest fails this repository's
|
||||
// own suite rather than its next pull request's gate.
|
||||
func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
|
||||
var out strings.Builder
|
||||
if err := moduleCheck([]string{"../../module.json"}, &out); err != nil {
|
||||
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A pull request's merge check (novox/hq to-be 45 §9, ADR 0237 as amended 2026-10-06): the forge
|
||||
// announces every pull request's new head, the controller decides what is checked, asks the build seat to
|
||||
// check it, and says the verdict as `checked`, which the forge's holder sets as the pull request's
|
||||
// statuses. **Before merge, never after**: every check the mesh had ran after a merge, on a machine.
|
||||
//
|
||||
// **The mesh's module graph decides, not the repository.** The controller holds the graph — every module,
|
||||
// the repository and directory it is built from — and maps the pull request's changed paths onto it by
|
||||
// the planner's own answer (reachOfMerge, touchedBy — the one place a changed file is mapped onto modules,
|
||||
// for the merge handler, the release planner, the merge gate and this check; issue 280): **a changed file
|
||||
// touches exactly the modules whose build reads it** — a module's own directory (the whole repository for
|
||||
// one built from its root), or a repository its recipe packages. A file no build reads — a script at the
|
||||
// root, a README — touches no module. A directory the change adds a module.json in, which the graph does
|
||||
// not hold yet (said by the head, issue 278), is a new module and is checked too.
|
||||
//
|
||||
// - touches a module: the build seat runs **the gate** — `mesh/merge-gate`, the touched manifests, every
|
||||
// machine composed with the change, the replays — and the repository's own merge-check.sh beside it;
|
||||
// - touches none, in a repository that is the mesh's (it sources a module on some branch, or shares the
|
||||
// core's owner): the gate is a pass that says so — a fact, not a missing check — and the repository's
|
||||
// own merge-check.sh runs as `mesh/repo-check`, a warning when it has none;
|
||||
// - touches none, anywhere else: the gate is a pass that says so, and nothing more is said.
|
||||
//
|
||||
// What is checked is decided here and run there (internal/builder/check.go).
|
||||
|
||||
// checkTimeout is how long one check may run on the build seat. Said here so the ask's watchdog (S6)
|
||||
// and the builder agree on what late means.
|
||||
const checkTimeout = 45 * time.Minute
|
||||
|
||||
// noModuleTouched is the gate's word for a change that touches nothing of the graph.
|
||||
const noModuleTouched = "the change touches no module of the mesh's graph"
|
||||
|
||||
// noMergeCheck is the repository layer's word for a repository of the mesh with no merge-check.sh.
|
||||
const noMergeCheck = "the repository declares no merge-check.sh: none of its own tests run before it merges"
|
||||
|
||||
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
|
||||
// each beside it — and whose owner is the mesh's own.
|
||||
var coreModules = map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host",
|
||||
"node-tools": "mesh-tools", "nats": "mesh-catalog"}
|
||||
|
||||
// checkScope is what a pull request reaches of the mesh's graph, as the planner reckons it.
|
||||
type checkScope struct {
|
||||
// Modules are the modules a merge of the change would move itself — built from the repository into
|
||||
// the pull request's base and reading a changed file, or packaging the repository's source — and
|
||||
// Dependents those the plan would build after them; New the directories it adds a module in.
|
||||
Modules []string
|
||||
Dependents []string
|
||||
New []string
|
||||
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
|
||||
Manifests []string
|
||||
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
|
||||
// module's build reads.
|
||||
Width, Tiers int
|
||||
Unread []string
|
||||
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
|
||||
// core's, or the change adds a module to it.
|
||||
Mesh bool
|
||||
// Judge is who judges the gate (link.JudgeSelf, link.JudgeValidator, or the running controller).
|
||||
Judge string
|
||||
// From is a module built from the repository, for how the mesh clones it; nil when none is.
|
||||
From *inventory.Entry
|
||||
// Reach is the planner's whole answer, which the change plan is made from.
|
||||
Reach mergeReach
|
||||
}
|
||||
|
||||
func (s checkScope) gated() bool { return len(s.Modules)+len(s.New) > 0 }
|
||||
|
||||
// pullScope is what a pull request reaches: **the planner's own answer** (reachOfMerge), asked as if the
|
||||
// head were merged into the base — never a mapping of its own, so a change to what a merge touches
|
||||
// changes what is checked with it (novox/hq ADR 0238).
|
||||
func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) checkScope {
|
||||
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL, Commit: p.Commit,
|
||||
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
|
||||
ModuleDirsSaid: p.ModuleDirsSaid}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
|
||||
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
|
||||
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
|
||||
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
|
||||
switch e.Manifest.Module {
|
||||
case "mesh-controller":
|
||||
s.Judge = link.JudgeSelf
|
||||
case "mesh-host":
|
||||
if s.Judge == "" {
|
||||
s.Judge = link.JudgeValidator
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, d := range r.Added {
|
||||
s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile))
|
||||
}
|
||||
sort.Strings(s.Manifests)
|
||||
s.Manifests = slices.Compact(s.Manifests)
|
||||
|
||||
// Whose repository it is, for how it is cloned and whether its own check is the mesh's to run.
|
||||
owners := map[string]bool{}
|
||||
for i, e := range entries {
|
||||
if e.Provided {
|
||||
continue
|
||||
}
|
||||
if _, core := coreModules[e.Manifest.Module]; core {
|
||||
if owner := sourceOwner(e.Source.Repository); owner != "" {
|
||||
owners[owner] = true
|
||||
}
|
||||
}
|
||||
if sameRepository(e.Source.Repository, m) && s.From == nil {
|
||||
s.From = &entries[i]
|
||||
}
|
||||
}
|
||||
s.Mesh = s.From != nil || owners[strings.ToLower(p.Owner)] || s.gated()
|
||||
return s
|
||||
}
|
||||
|
||||
// sourceOwner is the owner of a recorded repository, a path on the git seat or a URL: novox/mesh-host → novox.
|
||||
func sourceOwner(repository string) string {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(parts[len(parts)-2])
|
||||
}
|
||||
|
||||
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
|
||||
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted
|
||||
// with the verdict whatever the verdict is.
|
||||
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Print(planText(plan))
|
||||
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
|
||||
sayChecked(ctx, direct)
|
||||
return nil
|
||||
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
|
||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
|
||||
sayChecked(ctx, direct)
|
||||
return nil
|
||||
}
|
||||
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
||||
scope.Width, scope.Tiers)
|
||||
}
|
||||
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
|
||||
p.Commit, seat, what, request.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
|
||||
// and what is read beside it.
|
||||
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
|
||||
entries []inventory.Entry) (link.BuildRequest, error) {
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
clone := func(s inventory.Source) (string, error) {
|
||||
if s.Seat == "" {
|
||||
return s.Repository, nil
|
||||
}
|
||||
return clonedFromSeat(world, s.Seat, s.Repository)
|
||||
}
|
||||
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
|
||||
source := inventory.Source{Seat: gitSeat, Repository: p.Owner + "/" + p.Repo}
|
||||
if scope.From != nil {
|
||||
source = scope.From.Source
|
||||
}
|
||||
repository, err := clone(source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
if dir == "mesh-controller" {
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
"mesh-lab")}); err == nil {
|
||||
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Ref: p.Commit,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
||||
func besideRefs(dir, running string) map[string]string {
|
||||
switch dir {
|
||||
case "mesh-catalog":
|
||||
return map[string]string{dir: "main"}
|
||||
case "mesh-controller":
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
||||
}
|
||||
return map[string]string{dir: running}
|
||||
}
|
||||
|
||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||
func siblingOf(repository, name string) string {
|
||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||
return repository[:cut+1] + name
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// sourceOnSeat is a source's seat form, nil for one on no seat.
|
||||
func sourceOnSeat(s inventory.Source) *link.SourceOnSeat {
|
||||
if s.Seat == "" {
|
||||
return nil
|
||||
}
|
||||
return &link.SourceOnSeat{Seat: s.Seat, Repository: s.Repository}
|
||||
}
|
||||
|
||||
// checkEvents is where the serving controller says a check's verdict; nil in a command.
|
||||
var checkEvents link.Bus
|
||||
|
||||
// maxCheckReport is how much of a check's report travels in its verdict: enough for the failures and
|
||||
// the machines, never a log.
|
||||
const maxCheckReport = 60 << 10
|
||||
|
||||
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
||||
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
||||
func checked(ctx context.Context, result link.BuildResult) {
|
||||
sayChecked(ctx, checkedOf(result))
|
||||
}
|
||||
|
||||
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
|
||||
// could not run.
|
||||
func checkedOf(result link.BuildResult) link.Checked {
|
||||
c := link.Checked{ID: result.ID, On: result.On, Commit: result.Ref}
|
||||
if result.Checked != nil {
|
||||
c.Owner, c.Repo, c.Number = result.Checked.Owner, result.Checked.Repo, result.Checked.Number
|
||||
}
|
||||
switch {
|
||||
case result.Check != nil:
|
||||
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
|
||||
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
|
||||
if c.Gate == nil {
|
||||
// A build seat from before the layers: its verdict is the gate's.
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
}
|
||||
case result.Failed != "":
|
||||
// The check could not run: an error, never read as a pass — on both layers it was asked for.
|
||||
c.Verdict, c.Summary = "error", "the check could not run: "+firstLine(result.Failed)
|
||||
default:
|
||||
c.Verdict, c.Summary = "error", "the build seat answered the check with no verdict"
|
||||
}
|
||||
if c.Verdict == "" {
|
||||
c.Verdict = "error"
|
||||
}
|
||||
if result.Check == nil {
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
}
|
||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
|
||||
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
|
||||
}
|
||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Dependents) == 0 {
|
||||
c.Gate.Dependents = result.Checked.Dependents
|
||||
}
|
||||
if result.Checked != nil {
|
||||
c.Plan = result.Checked.Plan
|
||||
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
|
||||
if g := result.Checked; g.Group != "" {
|
||||
c.Group = g.Group
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
|
||||
Commit: result.Ref})
|
||||
for _, m := range g.Members {
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
|
||||
Commit: m.Ref})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, l := range []*link.CheckLayer{c.Gate, c.RepoCheck} {
|
||||
if l != nil && l.Verdict == "" {
|
||||
l.Verdict = "error"
|
||||
}
|
||||
}
|
||||
if len(c.Report) > maxCheckReport {
|
||||
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func prefixedAll(prefix string, items []string) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, i := range items {
|
||||
out = append(out, prefix+i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
|
||||
func sayChecked(ctx context.Context, c link.Checked) {
|
||||
repo := "none"
|
||||
if c.RepoCheck != nil {
|
||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||
}
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
if checkEvents == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
|
||||
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **The mesh's module graph decides what a pull request's check runs**, not the repository (novox/hq
|
||||
// ADR 0237 as amended): a change is mapped onto the graph by the rule a merge is (issue 278), the gate
|
||||
// runs when it touches a module — a new one included — and a repository that is the mesh's and touches
|
||||
// none still has its own merge-check.sh run.
|
||||
func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) {
|
||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
const controller = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
const host = "http://forge.internal:20000/novox/mesh-host.git"
|
||||
photos := fromRepo("photos", "http://forge.internal:20000/novox/photos.git", "")
|
||||
photos.Source.Ref = "nox-mesh"
|
||||
snake := fromRepo("snake", "jschoubben/snake", "")
|
||||
snake.Source.Seat = "git"
|
||||
entries := []inventory.Entry{
|
||||
fromRepo("gitea", catalogue, "modules/gitea"),
|
||||
fromRepo("keycloak", catalogue, "modules/keycloak"),
|
||||
fromRepo("nats", catalogue, "modules/nats"),
|
||||
fromRepo("mesh-controller", controller, ""),
|
||||
fromRepo("mesh-host", host, ""),
|
||||
photos, snake,
|
||||
}
|
||||
pull := func(owner, repo, base string, paths []string, dirs ...string) link.PullUpdated {
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Base: base, Commit: "abc", Paths: paths,
|
||||
ModuleDirs: dirs, ModuleDirsSaid: true}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
p link.PullUpdated
|
||||
modules, new, manifest string
|
||||
mesh bool
|
||||
judge string
|
||||
}{
|
||||
{"one module's own files", pull("novox", "mesh-catalog", "main", []string{"modules/gitea/index.ts"}, "modules/gitea"),
|
||||
"gitea", "", "modules/gitea/module.json", true, ""},
|
||||
{"a file no module's build reads touches no module (issue 280)",
|
||||
pull("novox", "mesh-catalog", "main", []string{"merge-check.sh", "README.md"}), "", "", "", true, ""},
|
||||
{"files the announcer could not list: everything built from it",
|
||||
link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", PathsTruncated: true,
|
||||
Paths: []string{"README.md"}}, "gitea,keycloak,nats", "",
|
||||
"modules/gitea/module.json,modules/keycloak/module.json,modules/nats/module.json", true, ""},
|
||||
{"a new module, said by the head", pull("novox", "mesh-catalog", "main",
|
||||
[]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, "modules/newmod"),
|
||||
"", "modules/newmod", "modules/newmod/module.json", true, ""},
|
||||
{"the controller judges itself", pull("novox", "mesh-controller", "main", []string{"cmd/x.go"}),
|
||||
"mesh-controller", "", "module.json", true, "self"},
|
||||
{"the node-engine is judged with its validator", pull("novox", "mesh-host", "main", []string{"validate/v.go"}),
|
||||
"mesh-host", "", "module.json", true, "validator"},
|
||||
{"the core's owner, no module: the mesh's, its own check alone", pull("novox", "hq", "main", []string{"README.md"}),
|
||||
"", "", "", true, ""},
|
||||
{"a branch nothing is built from: the mesh's repository, no module", pull("novox", "photos", "master",
|
||||
[]string{"server/x.js"}), "", "", "", true, ""},
|
||||
{"the branch a module is built from", pull("novox", "photos", "nox-mesh", []string{"server/x.js"}),
|
||||
"photos", "", "module.json", true, ""},
|
||||
{"a repository on the forge's seat", pull("jschoubben", "snake", "main", []string{"index.html"}),
|
||||
"snake", "", "module.json", true, ""},
|
||||
{"a repository of nobody's, touching nothing", pull("someone", "dotfiles", "main", []string{"x"}),
|
||||
"", "", "", false, ""},
|
||||
{"a repository adding a module at its root", pull("someone", "newapp", "main", []string{"module.json", "x.js"}),
|
||||
"", ".", "module.json", true, ""},
|
||||
} {
|
||||
s := pullScope(c.p, entries, nil, nil)
|
||||
got := []string{strings.Join(s.Modules, ","), strings.Join(s.New, ","), strings.Join(s.Manifests, ",")}
|
||||
want := []string{c.modules, c.new, c.manifest}
|
||||
for i, what := range []string{"modules", "new", "manifests"} {
|
||||
if got[i] != want[i] {
|
||||
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
|
||||
}
|
||||
}
|
||||
if s.Mesh != c.mesh || s.Judge != c.judge {
|
||||
t.Errorf("%s: the mesh's %v judged by %q, wanted %v by %q", c.what, s.Mesh, s.Judge, c.mesh, c.judge)
|
||||
}
|
||||
if s.gated() != (c.modules != "" || c.new != "") {
|
||||
t.Errorf("%s: gated %v", c.what, s.gated())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose build packages another repository's source is touched by a change to it.
|
||||
func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
|
||||
entries := []inventory.Entry{fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")}
|
||||
read := map[string][]inventory.ReadRepository{"node-tools": {{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}}
|
||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "abc", Paths: []string{"go/x.go"}}, entries, read, nil)
|
||||
if strings.Join(s.Modules, ",") != "node-tools" || len(s.Manifests) != 0 {
|
||||
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
// Each layer is said; a check that could not run is an error on both, never a pass; a build seat from
|
||||
// before the layers is read as the gate.
|
||||
func TestAChecksLayersAreEachSaidAndAnErrorIsNeverAPass(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3, Modules: []string{"gitea"}}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
||||
if c.Verdict != "error" || c.Gate == nil || c.Gate.Verdict != "error" || c.RepoCheck == nil || c.RepoCheck.Verdict != "error" {
|
||||
t.Fatalf("a check that could not run said %+v", c)
|
||||
}
|
||||
if strings.Join(c.Gate.Modules, ",") != "gitea" {
|
||||
t.Errorf("the gate names %v", c.Gate.Modules)
|
||||
}
|
||||
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "warning", Summary: "wide"}})
|
||||
if c.Gate == nil || c.Gate.Verdict != "warning" || c.RepoCheck != nil {
|
||||
t.Fatalf("an outcome without layers said %+v", c)
|
||||
}
|
||||
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "pass",
|
||||
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "its merge-check.sh failed"}}})
|
||||
if c.RepoCheck.Verdict != "fail" || c.Gate.Verdict != "pass" {
|
||||
t.Fatalf("the layers said %+v / %+v", c.Gate, c.RepoCheck)
|
||||
}
|
||||
}
|
||||
|
||||
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
|
||||
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
|
||||
// module in both, and what stands on a touched module is named as its dependents in both.
|
||||
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
|
||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
nats := fromRepo("nats", catalogue, "modules/nats")
|
||||
nats.Source.BuiltFrom = "old"
|
||||
gitea := fromRepo("gitea", catalogue, "modules/gitea")
|
||||
gitea.Source.BuiltFrom = "old"
|
||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
||||
entries := []inventory.Entry{nats, gitea, tools}
|
||||
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
||||
read := map[string][]inventory.ReadRepository{}
|
||||
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
paths []string
|
||||
moved, dependents string
|
||||
width int
|
||||
unread string
|
||||
}{
|
||||
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
|
||||
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
|
||||
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
|
||||
} {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
||||
s := pullScope(p, entries, read, edges)
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
|
||||
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
|
||||
}
|
||||
if got := strings.Join(s.Dependents, ","); got != c.dependents {
|
||||
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
|
||||
}
|
||||
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
|
||||
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
|
||||
}
|
||||
if got := strings.Join(s.Unread, ","); got != c.unread {
|
||||
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
|
||||
}
|
||||
}
|
||||
|
||||
// A repository whose build another module's recipe packages: that module is reached through the
|
||||
// planner's `read`, and what stands on it after it.
|
||||
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
|
||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
|
||||
Paths: []string{"src/index.ts"}}, entries, read, edges)
|
||||
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
|
||||
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a build of a commit off its repository's
|
||||
// default branch — a pull request's head, a branch built by hand, a rebuild or replay of one — is recorded
|
||||
// and never registered, so nothing can send it; a check or a dry run never is either.
|
||||
func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
|
||||
on := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main", OnTrunk: true}
|
||||
if err := publishable(on, ""); err != nil {
|
||||
t.Errorf("a build on the trunk was refused: %v", err)
|
||||
}
|
||||
off := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main"}
|
||||
if err := publishable(off, ""); !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "main") {
|
||||
t.Errorf("a build off the trunk was let through: %v", err)
|
||||
}
|
||||
for _, r := range []link.BuildResult{
|
||||
{ID: "c", Trunk: "main", OnTrunk: true, Check: &link.CheckOutcome{Verdict: "pass"}},
|
||||
{ID: "d", Trunk: "main", OnTrunk: true, Checked: &link.CheckRequest{}},
|
||||
{ID: "e", Trunk: "main", OnTrunk: true, DryRun: true},
|
||||
} {
|
||||
if publishable(r, "") == nil {
|
||||
t.Errorf("%s, a check or a dry run, was publishable", r.ID)
|
||||
}
|
||||
}
|
||||
// A module that follows a branch other than the default: that branch is its trunk, and the default
|
||||
// is not.
|
||||
follows := link.BuildResult{ID: "g", Commit: "abc", Trunk: "master", Branches: []string{"nox-mesh"}}
|
||||
if err := publishable(follows, "nox-mesh"); err != nil {
|
||||
t.Errorf("a commit on the branch a module follows was refused: %v", err)
|
||||
}
|
||||
if err := publishable(link.BuildResult{ID: "h", Commit: "abc", Trunk: "master", OnTrunk: true,
|
||||
Branches: []string{"master"}}, "nox-mesh"); err == nil {
|
||||
t.Error("a commit on the default but not on the branch the module follows was published")
|
||||
}
|
||||
if err := publishable(link.BuildResult{ID: "i", Commit: "abc", Trunk: "main", Branches: []string{"feat/x"}}, ""); err == nil {
|
||||
t.Error("a feature branch's commit was published")
|
||||
}
|
||||
// A build seat older than the rule says nothing: let through and said, so the rule can reach the mesh.
|
||||
if err := publishable(link.BuildResult{ID: "f", Commit: "abc"}, ""); err != nil {
|
||||
t.Errorf("a build seat that said nothing was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Letting the artifact store go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||
//
|
||||
// **Run where the records change.** A build is the moment new bytes landed in the store and the
|
||||
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
||||
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
||||
//
|
||||
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
||||
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
||||
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
||||
// having failed.
|
||||
|
||||
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
||||
// what it let go of. Says what it did and what it could not; returns nothing, because nothing
|
||||
// upstream should branch on it.
|
||||
func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||
references, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
|
||||
return
|
||||
}
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
|
||||
return
|
||||
}
|
||||
if len(references) == 0 && len(kept) == 0 {
|
||||
return
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue to find the artifact store: %v\n", err)
|
||||
return
|
||||
}
|
||||
// As the mesh reaches it from the network. Empty means the store is not on the network — on a
|
||||
// mesh being raised it is not yet, and there the store holds one build of anything and has
|
||||
// nothing to collect.
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil || address == "" {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not find the artifact store to collect from: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
|
||||
// never collected has the whole history to get through, and a person waiting on `build` should
|
||||
// not pay for it. Two bounds, and what is left over is simply offered again next time —
|
||||
// builds are frequent, and the point is that the store stops growing, not that it empties
|
||||
// tonight.
|
||||
within, stop := context.WithTimeout(ctx, sweepBudget)
|
||||
defer stop()
|
||||
store := artifacts.Store{Address: address}
|
||||
|
||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||
wrote, missing, err := holdKept(within, store, kept)
|
||||
if wrote > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
|
||||
}
|
||||
if missing > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||
"`collection` lists them\n", missing)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
var done []string
|
||||
var left, skipped int
|
||||
for i, reference := range references {
|
||||
if i >= mostPerSweep || within.Err() != nil {
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
err := store.LetGo(within, reference)
|
||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||
// again for ever.
|
||||
done = append(done, reference)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||
// reason to stop, because the store was never asked. One of these at the front of
|
||||
// the oldest-first order ended every sweep until this.
|
||||
skipped++
|
||||
if skipped == 1 {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"the sweep will not address %s and went on: %v\n", reference, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||
// front of whoever was building something.
|
||||
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
|
||||
reference, err)
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
|
||||
if len(done) > 0 {
|
||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||
// the sweep ran out of budget would mean asking about them again for ever.
|
||||
if err := inv.MarkCollected(ctx, done); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
|
||||
len(done), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
|
||||
len(done))
|
||||
}
|
||||
if left > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
|
||||
}
|
||||
}
|
||||
|
||||
// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
|
||||
// Answers how many holders it wrote and how many kept archives the store does not have.
|
||||
//
|
||||
// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
|
||||
// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
|
||||
// reference the store cannot be asked about is skipped as the deletion loop skips one
|
||||
// (novox/hq issue 226).
|
||||
func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
|
||||
for _, reference := range kept {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
|
||||
}
|
||||
did, err := store.Hold(ctx, reference)
|
||||
switch {
|
||||
case err == nil:
|
||||
if did {
|
||||
wrote++
|
||||
}
|
||||
case errors.Is(err, artifacts.Gone):
|
||||
missing++
|
||||
case errors.Is(err, artifacts.ErrNotOurs):
|
||||
default:
|
||||
return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
|
||||
}
|
||||
}
|
||||
return wrote, missing, nil
|
||||
}
|
||||
|
||||
// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
|
||||
// several times a day converges within days of this landing; small enough that no single build
|
||||
// waits on the whole backlog.
|
||||
const mostPerSweep = 200
|
||||
|
||||
// sweepBudget is the longest a sweep will keep a build waiting.
|
||||
const sweepBudget = 60 * time.Second
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
)
|
||||
|
||||
// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
|
||||
// (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The question to answer before the store's collector runs for real.** The collector deletes
|
||||
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
|
||||
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
|
||||
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
|
||||
// dry run go.
|
||||
//
|
||||
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
|
||||
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
|
||||
// new verb in the seat's row.
|
||||
|
||||
type collectionReport struct {
|
||||
// Store is the artifact store as this machine reached it; empty when it is not on the network.
|
||||
Store string `json:"store"`
|
||||
// KeptArchives is how many archives the mesh keeps, for either reason.
|
||||
KeptArchives int `json:"kept_archives"`
|
||||
// Held is how many of them the store holds by their manifest.
|
||||
Held int `json:"held"`
|
||||
// Unheld are the kept archives the collector would delete tonight if it ran for real.
|
||||
Unheld []string `json:"unheld"`
|
||||
// Missing are kept archives the store does not have at all.
|
||||
Missing []string `json:"missing"`
|
||||
// Unasked is how many could not be asked about, and why the asking stopped.
|
||||
Unasked int `json:"unasked"`
|
||||
Stopped string `json:"stopped,omitempty"`
|
||||
// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
|
||||
// collected — split by kind.
|
||||
Eligible int `json:"eligible"`
|
||||
EligibleImages int `json:"eligible_images"`
|
||||
EligibleArchives int `json:"eligible_archives"`
|
||||
// SafeToCollect is whether every kept archive was asked about and every one is held.
|
||||
SafeToCollect bool `json:"safe_to_collect"`
|
||||
}
|
||||
|
||||
func collectionCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("collection", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("collection [--json]")
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eligible, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
|
||||
for _, reference := range eligible {
|
||||
if strings.Contains(reference, "/blobs/") {
|
||||
report.EligibleArchives++
|
||||
} else {
|
||||
report.EligibleImages++
|
||||
}
|
||||
}
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if report.Store == "" {
|
||||
report.Unasked = len(kept)
|
||||
report.Stopped = "this mesh has no artifact store on its network"
|
||||
} else {
|
||||
report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
|
||||
}
|
||||
report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
|
||||
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(report)
|
||||
}
|
||||
printCollection(report)
|
||||
return nil
|
||||
}
|
||||
|
||||
// askHeld asks the store about each kept archive, stopping at the first answer that is not about
|
||||
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
|
||||
// identical failures says less than one line.
|
||||
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
|
||||
for i, reference := range kept {
|
||||
held, err := store.Held(ctx, reference)
|
||||
switch {
|
||||
case err == nil && held:
|
||||
report.Held++
|
||||
case err == nil:
|
||||
report.Unheld = append(report.Unheld, reference)
|
||||
case errors.Is(err, artifacts.Gone):
|
||||
report.Missing = append(report.Missing, reference)
|
||||
case errors.Is(err, artifacts.ErrNotOurs):
|
||||
// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
|
||||
report.Unasked++
|
||||
default:
|
||||
return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
|
||||
}
|
||||
}
|
||||
return report.Unasked, ""
|
||||
}
|
||||
|
||||
func printCollection(r collectionReport) {
|
||||
store := r.Store
|
||||
if store == "" {
|
||||
store = "(not on the network)"
|
||||
}
|
||||
fmt.Printf("artifact store %s\n", store)
|
||||
fmt.Printf("kept archives %d\n", r.KeptArchives)
|
||||
fmt.Printf(" held %d\n", r.Held)
|
||||
fmt.Printf(" unheld %d\n", len(r.Unheld))
|
||||
fmt.Printf(" missing %d\n", len(r.Missing))
|
||||
if r.Unasked > 0 {
|
||||
fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped)
|
||||
}
|
||||
fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
|
||||
if len(r.Unheld) > 0 {
|
||||
fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
|
||||
for _, reference := range r.Unheld {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
}
|
||||
if len(r.Missing) > 0 {
|
||||
fmt.Println("\nmissing — kept by the mesh, not in the store:")
|
||||
for _, reference := range r.Missing {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
if r.SafeToCollect {
|
||||
fmt.Println("every kept archive is held: the store's collector may run for real")
|
||||
} else {
|
||||
fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
||||
//
|
||||
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||
// while, with a reason, and that is recorded as a hand act.
|
||||
|
||||
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||
var conditionsFrom *conditions.Keeper
|
||||
|
||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||
// does nothing).
|
||||
Changed: statusFrom.nudge,
|
||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||
}
|
||||
|
||||
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
||||
// it was given before it returns.
|
||||
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||
if conditionsFrom != nil {
|
||||
return f(conditionsFrom)
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
k, err := keeperOn(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
k.Close(flushing)
|
||||
}()
|
||||
return f(k)
|
||||
})
|
||||
}
|
||||
|
||||
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
||||
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
||||
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||
if conditionsFrom != nil {
|
||||
return conditionsFrom.Open(ctx)
|
||||
}
|
||||
if _, err := broker.BusAddress(); err != nil {
|
||||
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||
}
|
||||
var out []conditions.Condition
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, err = conditions.Read(reading, store)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// conditionsUsage is how the verb is typed.
|
||||
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
||||
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
||||
"conditions history [--days N] [--key K] [--json]"
|
||||
|
||||
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
||||
func conditionsCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
return listConditions(ctx, args)
|
||||
case "show":
|
||||
return showCondition(ctx, args)
|
||||
case "silence":
|
||||
return silenceCondition(ctx, args)
|
||||
case "history":
|
||||
return conditionHistory(ctx, args)
|
||||
}
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
|
||||
func listConditions(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||
severity := set.String("severity", "", "only urgent, or only warning")
|
||||
machine := set.String("machine", "", "only those about this machine")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
||||
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
||||
}
|
||||
open, err := openConditions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, c := range open {
|
||||
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
||||
(*machine == "" || concerns(c, *machine)) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Condition{}
|
||||
}
|
||||
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if len(open) == 0 {
|
||||
fmt.Println("no open conditions")
|
||||
} else {
|
||||
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, line := range conditionLines(out, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||
func concerns(c conditions.Condition, machine string) bool {
|
||||
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||
return true
|
||||
}
|
||||
for _, part := range strings.Split(c.Key, ".") {
|
||||
if part == machine {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
||||
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||
var out []string
|
||||
for _, c := range list {
|
||||
times := ""
|
||||
if c.Count > 1 {
|
||||
times = fmt.Sprintf(", raised %d times", c.Count)
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
||||
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
||||
if c.SilencedAt(now) {
|
||||
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
||||
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func showCondition(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New("conditions show <key>")
|
||||
}
|
||||
key := rest[0]
|
||||
var c conditions.Condition
|
||||
var found bool
|
||||
if conditionsFrom != nil {
|
||||
c, found, err = conditionsFrom.Get(ctx, key)
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
||||
"history --key %s` what became of it", key, key)
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(c)
|
||||
}
|
||||
now := time.Now()
|
||||
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
if c.Subject.Machine != "" {
|
||||
fmt.Printf(", on %s", c.Subject.Machine)
|
||||
}
|
||||
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||
now.Sub(c.LastObserved).Round(time.Second))
|
||||
if c.Count > 1 {
|
||||
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
}
|
||||
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||
if c.Silenced != nil {
|
||||
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
c.Silenced.By, c.Silenced.Why)
|
||||
}
|
||||
if len(c.Tried) > 0 {
|
||||
fmt.Println("\n tried:")
|
||||
for _, t := range c.Tried {
|
||||
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n evidence, newest first:")
|
||||
for _, e := range c.Evidence {
|
||||
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolverWords(r string) string {
|
||||
switch r {
|
||||
case conditions.ResolverSelf:
|
||||
return "itself: it clears when observation says it is resolved"
|
||||
case conditions.ResolverOperator:
|
||||
return "the operator: nothing in the mesh will repair it"
|
||||
case conditions.ResolverAgent:
|
||||
return "an agent"
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
||||
// who and why before it is done, its cause the condition's kind unless one is given.
|
||||
func silenceCondition(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
||||
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
||||
acts := addHandActFlags(set)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
||||
}
|
||||
key := rest[0]
|
||||
if err := acts.require("conditions silence"); err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := parseFor(*forFlag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d > conditions.MaxSilence {
|
||||
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
||||
}
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
c, found, err := k.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
||||
}
|
||||
if strings.TrimSpace(*acts.cause) == "" {
|
||||
*acts.cause = c.Kind
|
||||
}
|
||||
*acts.condition = key
|
||||
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
||||
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
||||
"`status` still says it; it clears when observation says it is resolved\n",
|
||||
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// parseFor reads a duration, days included.
|
||||
func parseFor(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
||||
}
|
||||
if days, ok := strings.CutSuffix(s, "d"); ok {
|
||||
n, err := strconv.Atoi(days)
|
||||
if err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("%q is not a number of days", s)
|
||||
}
|
||||
return time.Duration(n) * 24 * time.Hour, nil
|
||||
}
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func conditionHistory(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||
days := set.Int("days", 7, "how many days back, at most 90")
|
||||
key := set.String("key", "", "only this condition")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("conditions history [--days N] [--key K] [--json]")
|
||||
}
|
||||
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
||||
var events []conditions.Event
|
||||
read := func(h conditions.History) error {
|
||||
var err error
|
||||
events, err = h.Since(ctx, since)
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if conditionsFrom != nil {
|
||||
events, err = conditionsFrom.HistorySince(ctx, since)
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
_, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return read(history)
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out []conditions.Event
|
||||
for _, e := range events {
|
||||
if *key == "" || e.Key == *key {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Event{}
|
||||
}
|
||||
return printJSON(map[string]any{"history": out, "days": *days})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for _, e := range out {
|
||||
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
||||
switch e.Change {
|
||||
case conditions.ChangeRaised, conditions.ChangeReopened:
|
||||
line += " — " + e.Summary
|
||||
case conditions.ChangeSeverity:
|
||||
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
||||
case conditions.ChangeResolver:
|
||||
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
||||
default:
|
||||
if e.Why != "" {
|
||||
line += " — " + e.Why
|
||||
}
|
||||
}
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
||||
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
||||
// is not "none open".
|
||||
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
||||
if unread != "" {
|
||||
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
urgent := 0
|
||||
for _, c := range list {
|
||||
if c.Severity == conditions.Urgent {
|
||||
urgent++
|
||||
}
|
||||
}
|
||||
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
||||
for _, line := range conditionLines(list, now) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
||||
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
||||
}
|
||||
|
||||
// orHealer is who tried, as an attempt names it.
|
||||
func orHealer(by string) string {
|
||||
if by == "" {
|
||||
return "a healer"
|
||||
}
|
||||
return by
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
|
||||
// by what is open.
|
||||
|
||||
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "conditions --json"},
|
||||
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
|
||||
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
|
||||
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
|
||||
"conditions history --json --days 3 --key machine.ace.silent"},
|
||||
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
|
||||
"conditions silence machine.ace.silent --for 2h --why on the train"},
|
||||
{map[string]any{"run": "true"}, "doctor run --json"},
|
||||
{map[string]any{}, "doctor --json"},
|
||||
} {
|
||||
verb := "conditions"
|
||||
if strings.HasPrefix(c.want, "doctor") {
|
||||
verb = "doctor"
|
||||
}
|
||||
argv, err := argvFor(verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
|
||||
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
|
||||
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
|
||||
} {
|
||||
if argv, err := argvFor(c.verb, c.args); err == nil {
|
||||
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
|
||||
t.Error("a silence is not a hand act")
|
||||
}
|
||||
}
|
||||
|
||||
// **A silence through the verb is recorded and bounded**; the condition stays open.
|
||||
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
|
||||
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
|
||||
t.Fatal("silenced without saying why")
|
||||
}
|
||||
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
|
||||
t.Fatal("silenced for more than a week")
|
||||
}
|
||||
said := printed(t, func() error {
|
||||
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
|
||||
})
|
||||
if !strings.Contains(said, "is silenced until") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
c, found, _ := k.Get(ctx, "machine.ace.silent")
|
||||
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
|
||||
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
|
||||
t.Fatalf("%s", listed)
|
||||
}
|
||||
}
|
||||
|
||||
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.Fail = errors.New("the bus is away")
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() || asked.conditionsUnread == "" {
|
||||
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.Fail = nil
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A finding one look can be wrong about is raised on the second look in a row** (novox/hq issue 277).
|
||||
//
|
||||
// D2 raised its resolver urgent on one unanswered question, asked once, while the resolver's machine
|
||||
// was loaded by a push and a build starting; thirty questions right after were answered at once. A
|
||||
// single sample of something that is answered over the network, or timed on a machine under load,
|
||||
// is not the invariant failing. So a source marks such a finding `Confirm`, and this holds it back:
|
||||
//
|
||||
// - raised when the source's previous look saw it too — two runs of the self-check in a row (five
|
||||
// minutes apart), or two ticks of the watchdogs (half a minute) — at the severity the source says;
|
||||
// - kept while it is already open, however it is seen, so a condition the next look still sees is
|
||||
// never cleared and raised again (flapping is not news; a reopening within ReopenWithin still is);
|
||||
// - cleared, as everything is, by the look that no longer sees it.
|
||||
//
|
||||
// A finding held back is not a pass: the self-check's verdict names it as unconfirmed. A finding that is
|
||||
// a definite answer — a resolver that answered wrongly, a stream that is not there — is not marked, and
|
||||
// is raised at once.
|
||||
type confirming struct {
|
||||
mu sync.Mutex
|
||||
// last is, by source, the keys of the Confirm findings its previous look saw.
|
||||
last map[string]map[string]bool
|
||||
}
|
||||
|
||||
// pass splits one source's findings into what is raised now and what is held for the next look, and
|
||||
// remembers what it saw. An open condition that cannot be read is kept: unknown is not a reason to
|
||||
// hold a finding back.
|
||||
func (c *confirming) pass(ctx context.Context, keeper *conditions.Keeper, source string,
|
||||
found []conditions.Observation) (raise, held []conditions.Observation) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.last == nil {
|
||||
c.last = map[string]map[string]bool{}
|
||||
}
|
||||
before, seen := c.last[source], map[string]bool{}
|
||||
for _, o := range found {
|
||||
if !o.Confirm {
|
||||
raise = append(raise, o)
|
||||
continue
|
||||
}
|
||||
key := o.Key()
|
||||
seen[key] = true
|
||||
if before[key] || isOpen(ctx, keeper, key) {
|
||||
raise = append(raise, o)
|
||||
continue
|
||||
}
|
||||
held = append(held, o)
|
||||
}
|
||||
c.last[source] = seen
|
||||
return raise, held
|
||||
}
|
||||
|
||||
// isOpen says whether a condition is open; one that cannot be read is taken as open.
|
||||
func isOpen(ctx context.Context, keeper *conditions.Keeper, key string) bool {
|
||||
if keeper == nil {
|
||||
return false
|
||||
}
|
||||
_, open, err := keeper.Get(ctx, key)
|
||||
return open || err != nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A fresh assignment is pushed before its credential exists (novox/hq issue 203): `assign` recorded
|
||||
// the module, `push` sealed a random own secret where the bus credential belongs, and the process
|
||||
// crash-looped until a person ran `module issue` and pushed again. Now assigning a module that speaks
|
||||
// on the bus issues its credential in the same act — or, when the bus cannot be reached from here,
|
||||
// says which verb to run — and a push never seals a placeholder in a credential's place.
|
||||
|
||||
func aTalker() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "talker", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/talker/broker"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/talker", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAssigningAModuleThatSpeaksOnTheBusNamesItsCredential(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
|
||||
// No bus is known to this process, so the credential cannot be issued here: the assignment
|
||||
// stands and says exactly what must happen before a push — never silently.
|
||||
said, err := assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "module issue talker --node laptop") {
|
||||
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
|
||||
}
|
||||
|
||||
// And the push refuses to send it, naming the same verb, rather than sealing a placeholder.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err == nil {
|
||||
t.Fatal("a push sealed a placeholder where talker's bus credential belongs")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "module issue talker --node laptop") || !strings.Contains(err.Error(), "issue 203") {
|
||||
t.Fatalf("the refusal does not say what to run: %v", err)
|
||||
}
|
||||
|
||||
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
|
||||
// does not mint again: a credential rotates on purpose, never by habit.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "laptop.talker", Kind: inventory.BusModule, Node: "laptop", Module: "talker"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err = assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "module issue") {
|
||||
t.Fatalf("a module with a minted credential was told to issue one:\n%s", said)
|
||||
}
|
||||
again, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again != hash {
|
||||
t.Fatal("re-assigning rotated the credential")
|
||||
}
|
||||
}
|
||||
|
||||
// A module that declares no broker secret is left alone: nothing to issue, nothing said.
|
||||
func TestAssigningAModuleThatDoesNotSpeakSaysNothingOfCredentials(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
register(t, open, helloWeb())
|
||||
said, err := assign(t.Context(), open, "laptop", "hello-web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "credential") {
|
||||
t.Fatalf("a module without a broker secret was told about credentials:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,931 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module declares the data it holds, and the mesh protects and watches it from that declaration
|
||||
// (novox/hq ADR 0233).
|
||||
//
|
||||
// The self-check's D13 composes what every machine declares, asks each machine's backup holder what
|
||||
// it measured of every item — size, newest write, newest good backup, the redundant storage it is on —
|
||||
// and keeps both. From that, and from what every provider says it holds for its consumers, it raises,
|
||||
// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking):
|
||||
//
|
||||
// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least
|
||||
// shrinkFloor less; `data-missing`: its path is gone;
|
||||
// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a
|
||||
// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on
|
||||
// empty databases while their real ones sat on another machine (issue 273);
|
||||
// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot
|
||||
// be compared;
|
||||
// - `data-quiet`: an item said to be written all the time has not been, within its bound;
|
||||
// - `backup-stale`: an item's newest good backup is older than its bound, or there is none;
|
||||
// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read;
|
||||
// `protection-missing`: an item said to be protected by redundancy is on storage that is not;
|
||||
// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person.
|
||||
//
|
||||
// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no
|
||||
// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by
|
||||
// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with
|
||||
// anything in it; this is the record of what it kept. An operator's path is never retired or deleted.
|
||||
|
||||
// The condition kinds of D13.
|
||||
const (
|
||||
kindDataShrank = "data-shrank"
|
||||
kindEmptyReplacement = "empty-replacement"
|
||||
kindDataHeldTwice = "data-held-twice"
|
||||
kindDataQuiet = "data-quiet"
|
||||
kindBackupStale = "backup-stale"
|
||||
kindDataUnmeasured = "data-unmeasured"
|
||||
// kindDataMissing is a watched item whose path is gone.
|
||||
kindDataMissing = "data-missing"
|
||||
// kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read.
|
||||
kindArrayDegraded = "array-degraded"
|
||||
// kindProtectionMissing is an item said to be protected by redundancy, on storage that is not.
|
||||
kindProtectionMissing = "protection-missing"
|
||||
)
|
||||
|
||||
// probeDataID is the self-check's id for this probe.
|
||||
const probeDataID = "D13"
|
||||
|
||||
// The bounds the findings are read against.
|
||||
var (
|
||||
// shrinkWindow is how far back the largest size is looked for.
|
||||
shrinkWindow = 7 * 24 * time.Hour
|
||||
// shrinkFloor is the least loss that is worth saying: two empty databases differ by a few
|
||||
// megabytes, and half of almost nothing is noise.
|
||||
shrinkFloor int64 = 16 << 20
|
||||
// dataAsk is how long one machine's holder, or one provider, is given to answer.
|
||||
dataAsk = 8 * time.Second
|
||||
)
|
||||
|
||||
// keyOfItem is one item's condition id: its machine, module and item.
|
||||
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
|
||||
|
||||
// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data).
|
||||
type holderAnswer struct {
|
||||
Module string `json:"module"`
|
||||
Data []holderItem `json:"data"`
|
||||
}
|
||||
|
||||
// holderItem is one item as the holder measured it.
|
||||
type holderItem struct {
|
||||
Item string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path"`
|
||||
SizeBytes *int64 `json:"size_bytes"`
|
||||
LastWrite *time.Time `json:"last_write"`
|
||||
MeasuredAt *time.Time `json:"measured_at"`
|
||||
LastBackup *time.Time `json:"last_backup"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233).
|
||||
Precision string `json:"precision,omitempty"`
|
||||
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
|
||||
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
|
||||
}
|
||||
|
||||
// readHolder reads a holder's answer into measurements by module and item.
|
||||
func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) {
|
||||
var modules []holderAnswer
|
||||
if err := json.Unmarshal(raw, &modules); err != nil {
|
||||
return nil, fmt.Errorf("its answer is not readable: %w", err)
|
||||
}
|
||||
out := map[string]map[string]inventory.Measurement{}
|
||||
for _, m := range modules {
|
||||
for _, it := range m.Data {
|
||||
if out[m.Module] == nil {
|
||||
out[m.Module] = map[string]inventory.Measurement{}
|
||||
}
|
||||
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
|
||||
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy,
|
||||
Precision: it.Precision}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaredOn is every data item a machine's composition declares, and the module there that holds
|
||||
// node-backup to measure them — empty for none.
|
||||
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) {
|
||||
var out []inventory.DeclaredData
|
||||
held := ""
|
||||
for _, m := range plan.Modules {
|
||||
for _, c := range m.Claims {
|
||||
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
|
||||
held = m.Module
|
||||
}
|
||||
}
|
||||
for _, it := range m.DataItems() {
|
||||
out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class,
|
||||
Owned: it.OwnedByModule(), Protection: it.Protection()})
|
||||
}
|
||||
}
|
||||
return out, held
|
||||
}
|
||||
|
||||
// consumerCopy is one provider's account of one consumer: where, how big, and whether still active.
|
||||
type consumerCopy struct {
|
||||
Node, Module, Consumer string
|
||||
Size *int64
|
||||
Retired bool
|
||||
// Class is how precious the consumer's data is: the stricter of what the provider keeps for its
|
||||
// consumers and what the consumer says it keeps there (`kept-by`).
|
||||
Class string
|
||||
}
|
||||
|
||||
// probeData is D13.
|
||||
func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("no bus to ask the machines over")
|
||||
}
|
||||
open := d.open
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
now := time.Now()
|
||||
delivered, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
type machine struct {
|
||||
name string
|
||||
declared []inventory.DeclaredData
|
||||
held bool
|
||||
measured map[string]map[string]inventory.Measurement
|
||||
askErr error
|
||||
}
|
||||
var machines []*machine
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// A machine that cannot be worked out declares nothing this run — which is not the same as
|
||||
// declaring nothing: retiring its data on that would be acting on an unreadable result.
|
||||
continue
|
||||
}
|
||||
declared, holder := declaredOn(plan)
|
||||
// **A holder is asked only once its machine has been sent it and had time to report** (novox/hq
|
||||
// issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it
|
||||
// measured" about it was a warning for a push nobody had made yet.
|
||||
held := holder != "" && delivered[n.Name].settled(holder, now)
|
||||
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
|
||||
}
|
||||
// Every holder asked at once, as D8 asks every ban list.
|
||||
var wg sync.WaitGroup
|
||||
for _, m := range machines {
|
||||
if !m.held || !heard[m.name] {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(m *machine) {
|
||||
defer wg.Done()
|
||||
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||
defer cancel()
|
||||
raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name)
|
||||
if err == nil {
|
||||
m.measured, err = readHolder(raw)
|
||||
}
|
||||
m.askErr = err
|
||||
}(m)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var out []conditions.Observation
|
||||
for _, m := range machines {
|
||||
if m.askErr != nil {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
|
||||
Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
|
||||
"nothing about that data is known", m.name),
|
||||
Said: firstLine(m.askErr.Error())})
|
||||
}
|
||||
why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
|
||||
change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err)
|
||||
}
|
||||
for _, r := range change.Retired {
|
||||
log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+
|
||||
"delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why)
|
||||
}
|
||||
for _, r := range change.Reenabled {
|
||||
log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine)
|
||||
}
|
||||
}
|
||||
|
||||
records, err := open.inventory.Data(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bindings, err := open.inventory.Bindings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||
copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func orUnknownPath(p string) string {
|
||||
if p == "" {
|
||||
return "a path its backup holder never named"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at
|
||||
// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding.
|
||||
func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var asked []providerInstance
|
||||
for _, p := range instances {
|
||||
m := shelf[p.Module]
|
||||
keeps := false
|
||||
for provision := range m.Grants {
|
||||
keeps = keeps || m.KeepsConsumerData(provision)
|
||||
}
|
||||
if keeps {
|
||||
asked = append(asked, p)
|
||||
}
|
||||
}
|
||||
states := make([]*link.RetirementState, len(asked))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range asked {
|
||||
wg.Add(1)
|
||||
go func(i int, p providerInstance) {
|
||||
defer wg.Done()
|
||||
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||
defer cancel()
|
||||
if s, err := askRetirement(asking, conn, p); err == nil {
|
||||
states[i] = &s
|
||||
}
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
var out []consumerCopy
|
||||
for i, p := range asked {
|
||||
s := states[i]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
class := consumersClass(shelf[p.Module])
|
||||
for _, c := range s.Held {
|
||||
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c,
|
||||
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])}
|
||||
if size, ok := s.HeldSizes[c]; ok && size >= 0 {
|
||||
size := size
|
||||
cp.Size = &size
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
for _, r := range s.Retired {
|
||||
if r.Kind != "" && r.Kind != "consumer" {
|
||||
continue
|
||||
}
|
||||
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true,
|
||||
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])}
|
||||
if r.SizeBytes != nil && *r.SizeBytes >= 0 {
|
||||
cp.Size = r.SizeBytes
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning
|
||||
// for anything else watched (the operator's ranking, ADR 0233).
|
||||
func severityOf(class string) conditions.Severity {
|
||||
if class == catalogue.ClassIrreplaceable {
|
||||
return conditions.Urgent
|
||||
}
|
||||
return conditions.Warning
|
||||
}
|
||||
|
||||
// consumersClass is the most precious class a provider keeps any of its consumers' data as.
|
||||
func consumersClass(m catalogue.Manifest) string {
|
||||
class := catalogue.ClassNone
|
||||
for provision := range m.Grants {
|
||||
if c, ok := m.ConsumerDataOf(provision); ok {
|
||||
class = catalogue.StricterClass(class, c.Class)
|
||||
} else if m.KeepsConsumerData(provision) {
|
||||
class = catalogue.StricterClass(class, catalogue.ClassValuable)
|
||||
}
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read
|
||||
// through where each is bound: by provider module and consumer identity, the class of that consumer's
|
||||
// data there; and by provider item key (machine/module/item), the class the item holding it is held to.
|
||||
func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) {
|
||||
upgraded, keptBy := map[string]string{}, map[string]string{}
|
||||
for _, b := range bindings {
|
||||
m, ok := shelf[b.Consumer]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
k, said := m.KeptByOf(b.Provision)
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module))
|
||||
key := b.Provider.Module + "/" + identity
|
||||
keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class)
|
||||
if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" {
|
||||
if _, own := shelf[b.Provider.Module].DataItem(pc.In); own {
|
||||
item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In
|
||||
upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class)
|
||||
}
|
||||
}
|
||||
}
|
||||
return upgraded, keptBy
|
||||
}
|
||||
|
||||
// dataFindings is every condition the data on record raises now. A function of what is known, so the
|
||||
// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer
|
||||
// of its module keeps data in it more precious than its own class says (`kept-by`), by its key.
|
||||
func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest,
|
||||
copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
byItem := map[string][]inventory.DataRecord{}
|
||||
arrays := map[string][]inventory.DataRecord{}
|
||||
type shrunk struct {
|
||||
machine, dataset, class string
|
||||
size, peak int64
|
||||
items []string
|
||||
}
|
||||
shrunkDatasets := map[string]shrunk{}
|
||||
for _, r := range records {
|
||||
if r.DeletedAt != nil {
|
||||
continue
|
||||
}
|
||||
class := catalogue.StricterClass(r.Class, upgraded[r.Key()])
|
||||
r.Class = class
|
||||
byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r)
|
||||
item, declared := shelf[r.Module].DataItem(r.Item)
|
||||
id := keyOfItem(r.Machine, r.Module, r.Item)
|
||||
if r.Retired() {
|
||||
if now.Sub(*r.RetiredAt) > cleanupAfter {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
|
||||
Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
|
||||
"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
|
||||
r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
|
||||
r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module),
|
||||
Said: "kept at " + orUnknownPath(r.Path)})
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !catalogue.Watched(class) {
|
||||
continue
|
||||
}
|
||||
severity := severityOf(class)
|
||||
if r.Redundancy != nil {
|
||||
where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
|
||||
arrays[where] = append(arrays[where], r)
|
||||
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
|
||||
Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
|
||||
"and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
|
||||
r.Item, r.Module, r.Machine, class),
|
||||
Said: orUnknownPath(r.Path) + " is on no redundant storage the backup holder can read"})
|
||||
}
|
||||
if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
|
||||
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.Item,
|
||||
r.Module, r.Machine, class),
|
||||
Said: orUnknownPath(r.Path) + " does not exist: " + firstLine(r.MeasureError)})
|
||||
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
|
||||
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
|
||||
// Several items on one dataset share its size: one condition for the dataset, as loud as the
|
||||
// most precious item on it.
|
||||
k := r.Machine + "/" + inventory.Dataset(r.Precision)
|
||||
ds := shrunkDatasets[k]
|
||||
ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak
|
||||
ds.class = catalogue.StricterClass(ds.class, class)
|
||||
ds.items = append(ds.items, r.Module+"/"+r.Item)
|
||||
shrunkDatasets[k] = ds
|
||||
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
|
||||
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
|
||||
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
|
||||
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
|
||||
"good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak),
|
||||
int(shrinkWindow.Hours()/24))})
|
||||
}
|
||||
if !declared {
|
||||
continue
|
||||
}
|
||||
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet,
|
||||
Kind: kindDataQuiet, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
|
||||
"whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339),
|
||||
within)})
|
||||
}
|
||||
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
|
||||
// plan, said only where the machine was measured — where a holder is there to take it.
|
||||
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
|
||||
within := item.BackupWithin()
|
||||
switch {
|
||||
case r.LastBackup == nil && now.Sub(r.FirstSeen) > within:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
|
||||
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
|
||||
r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))})
|
||||
case r.LastBackup != nil && now.Sub(*r.LastBackup) > within:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
|
||||
"of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, k := range keysSorted(shrunkDatasets) {
|
||||
ds := shrunkDatasets[k]
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
|
||||
Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than half of "+
|
||||
"what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak),
|
||||
int(shrinkWindow.Hours()/24)),
|
||||
Said: "the dataset " + ds.dataset})
|
||||
}
|
||||
// The redundant storage watched data is on: one condition per array, as loud as the most precious
|
||||
// item on it — the array, not each item, is what degrades.
|
||||
for _, where := range keysSorted(arrays) {
|
||||
rs := arrays[where]
|
||||
red := rs[0].Redundancy
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
continue
|
||||
}
|
||||
class, machine := catalogue.ClassValuable, rs[0].Machine
|
||||
var names []string
|
||||
for _, r := range rs {
|
||||
class = catalogue.StricterClass(class, r.Class)
|
||||
names = append(names, r.Module+"/"+r.Item)
|
||||
}
|
||||
state := "could not be read"
|
||||
if red.Healthy != nil {
|
||||
state = "is NOT healthy"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
|
||||
Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
|
||||
Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, strings.Join(names, ", "),
|
||||
state),
|
||||
Said: fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said))})
|
||||
}
|
||||
// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
|
||||
// an empty replacement. Only against a retired copy — a module running on two machines on purpose
|
||||
// keeps two different sets of data.
|
||||
for _, key := range keysSorted(byItem) {
|
||||
rs := byItem[key]
|
||||
for _, a := range rs {
|
||||
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) {
|
||||
continue
|
||||
}
|
||||
for _, o := range rs {
|
||||
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) ||
|
||||
!replacedByLess(*a.Size, *o.Size) {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||
Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
|
||||
"an empty replacement of its data. Move the data, or assign it back where its data is",
|
||||
a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, consumerFindings(copies)...)
|
||||
return out
|
||||
}
|
||||
|
||||
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than
|
||||
// half of it, and at least shrinkFloor less.
|
||||
func replacedByLess(inUse, kept int64) bool {
|
||||
return inUse*2 < kept && kept-inUse >= shrinkFloor
|
||||
}
|
||||
|
||||
// consumerFindings is the same question of consumers' data at providers: one consumer, the same
|
||||
// provider module on two machines.
|
||||
func consumerFindings(copies []consumerCopy) []conditions.Observation {
|
||||
by := map[string][]consumerCopy{}
|
||||
for _, c := range copies {
|
||||
k := c.Module + "/" + c.Consumer
|
||||
by[k] = append(by[k], c)
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, k := range keysSorted(by) {
|
||||
cs := by[k]
|
||||
if len(cs) < 2 {
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, a := range cs {
|
||||
if a.Retired || a.Size == nil {
|
||||
continue
|
||||
}
|
||||
for _, o := range cs {
|
||||
if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
|
||||
continue
|
||||
}
|
||||
state := "active"
|
||||
if o.Retired {
|
||||
state = "retired"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||
Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)),
|
||||
Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+
|
||||
"consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+
|
||||
"the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node,
|
||||
sizeWords(o.Size), o.Node)})
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if found {
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
continue
|
||||
}
|
||||
var active []consumerCopy
|
||||
for _, c := range cs {
|
||||
if !c.Retired {
|
||||
active = append(active, c)
|
||||
}
|
||||
}
|
||||
if len(active) >= 2 {
|
||||
var where []string
|
||||
var also []string
|
||||
for _, c := range active {
|
||||
where = append(where, c.Node+" ("+sizeWords(c.Size)+")")
|
||||
also = append(also, c.Node)
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice,
|
||||
Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses",
|
||||
active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keysSorted[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ---- the `data` verb ---------------------------------------------------------------------------
|
||||
|
||||
const dataUsage = "data [--json] [--machine <name>] [--retired]"
|
||||
|
||||
// dataRow is one item as `data` lists it.
|
||||
type dataRow struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
Item string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Protection string `json:"protection,omitempty"`
|
||||
// Array is the redundant storage it is on and its state, where its holder could tell.
|
||||
Array string `json:"array,omitempty"`
|
||||
Unmeasured string `json:"unmeasured,omitempty"`
|
||||
// Precision says what the size is: exact, a dataset's whole size, partial, or none.
|
||||
Precision string `json:"precision,omitempty"`
|
||||
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
||||
LastWrite string `json:"last-write,omitempty"`
|
||||
MeasuredAt string `json:"measured-at,omitempty"`
|
||||
LastBackup string `json:"last-backup,omitempty"`
|
||||
BackupDue string `json:"backup-within,omitempty"`
|
||||
Retired string `json:"retired,omitempty"`
|
||||
RetiredWhy string `json:"retired-why,omitempty"`
|
||||
Deleted string `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last
|
||||
// found it.
|
||||
func dataCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("data", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
only := set.String("machine", "", "one machine")
|
||||
retiredOnly := set.Bool("retired", false, "only what is retired")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(dataUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
records, err := open.inventory.Data(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows := dataRows(records, shelf, *only, *retiredOnly)
|
||||
if *asJSON {
|
||||
return printJSON(map[string]any{"data": rows})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run")
|
||||
return nil
|
||||
}
|
||||
for _, r := range rows {
|
||||
state := ""
|
||||
switch {
|
||||
case r.Deleted != "":
|
||||
state = " DELETED " + r.Deleted
|
||||
case r.Retired != "":
|
||||
state = " RETIRED " + r.Retired + " — " + r.RetiredWhy
|
||||
}
|
||||
fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class,
|
||||
sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state)
|
||||
if r.Array != "" {
|
||||
fmt.Printf(" on %s\n", r.Array)
|
||||
}
|
||||
if r.Precision != "" && r.Precision != "exact" {
|
||||
fmt.Printf(" size: %s\n", r.Precision)
|
||||
}
|
||||
if r.Unmeasured != "" {
|
||||
fmt.Printf(" not measured: %s\n", r.Unmeasured)
|
||||
}
|
||||
if r.Class == catalogue.ClassCache {
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt),
|
||||
orNever(r.LastBackup), within(r.BackupDue))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow {
|
||||
rows := []dataRow{}
|
||||
stamp := func(t *time.Time) string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
for _, r := range records {
|
||||
if only != "" && r.Machine != only {
|
||||
continue
|
||||
}
|
||||
if retiredOnly && !r.Retired() {
|
||||
continue
|
||||
}
|
||||
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
|
||||
Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
|
||||
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
|
||||
Deleted: stamp(r.DeletedAt)}
|
||||
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
|
||||
row.BackupDue = it.BackupWithin().String()
|
||||
}
|
||||
if red := r.Redundancy; red != nil {
|
||||
state := "state unread"
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
state = "healthy"
|
||||
} else if red.Healthy != nil {
|
||||
state = "NOT HEALTHY"
|
||||
}
|
||||
row.Array = red.Kind + " " + red.Where + ", " + state
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func orNothingWord(s string) string {
|
||||
if s == "" || s == "none" {
|
||||
return "nothing"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func orNever(s string) string {
|
||||
if s == "" {
|
||||
return "never"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func within(s string) string {
|
||||
if s == "" {
|
||||
return " (not backed up)"
|
||||
}
|
||||
return " (bound " + s + ")"
|
||||
}
|
||||
|
||||
// ---- cleanup of retired own data ---------------------------------------------------------------
|
||||
|
||||
// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first
|
||||
// takes a last restore point of it, tagged as retired, and only then removes it — in the background,
|
||||
// because a large item outlasts any call — and the second says how that went. Module tools, not seat
|
||||
// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete.
|
||||
const (
|
||||
ToolDeleteRetired = "backup_delete_retired"
|
||||
ToolDeletedOutcome = "backup_deleted"
|
||||
)
|
||||
|
||||
// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person.
|
||||
var deletionWait = 8 * time.Minute
|
||||
|
||||
// deletionPoll is how often it asks.
|
||||
var deletionPoll = 5 * time.Second
|
||||
|
||||
// deletion is a holder's account of one deletion.
|
||||
type deletion struct {
|
||||
Started bool `json:"started"`
|
||||
Running bool `json:"running"`
|
||||
Done bool `json:"done"`
|
||||
OK bool `json:"ok"`
|
||||
Snapshot string `json:"snapshot"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// retiredData is every retired item on record, as `cleanup list` shows them.
|
||||
func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow {
|
||||
var out []retiredRow
|
||||
for _, r := range records {
|
||||
if !r.Retired() {
|
||||
continue
|
||||
}
|
||||
out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind,
|
||||
RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24),
|
||||
SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer.
|
||||
const retiredDataKind = "own-data"
|
||||
|
||||
// holderOn is the module holding node-backup on a machine.
|
||||
func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) {
|
||||
held, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, h := range held {
|
||||
if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine {
|
||||
return h.Module, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+
|
||||
"(after a last restore point)", catalogue.BackupSeat, machine)
|
||||
}
|
||||
|
||||
// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and
|
||||
// never one not retired. The holder takes a last restore point of it first, so the deletion can be
|
||||
// undone until a person forgets that restore point; the record says deleted only once the holder says
|
||||
// it is.
|
||||
func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error {
|
||||
inv := open.inventory
|
||||
if !r.Retired() {
|
||||
return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done",
|
||||
r.Item, r.Module, r.Machine)
|
||||
}
|
||||
if r.Path == "" {
|
||||
return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted",
|
||||
r.Item, r.Module, r.Machine)
|
||||
}
|
||||
if plan, _, err := planFor(ctx, open, r.Machine); err == nil {
|
||||
for _, m := range plan.Modules {
|
||||
if _, still := m.DataItem(r.Item); still && m.Module == r.Module {
|
||||
return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done",
|
||||
r.Module, r.Machine, r.Item)
|
||||
}
|
||||
}
|
||||
}
|
||||
holder, err := holderOn(ctx, inv, r.Machine)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item})
|
||||
args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item,
|
||||
"why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController}
|
||||
ask := func(tool string) (deletion, error) {
|
||||
var d deletion
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error)
|
||||
}
|
||||
return d, unmarshalAnswer(answer, &d)
|
||||
}
|
||||
d, err := ask(ToolDeleteRetired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(deletionPoll):
|
||||
}
|
||||
if d, err = ask(ToolDeletedOutcome); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !d.Done:
|
||||
fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+
|
||||
"showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n",
|
||||
holder, r.Machine, r.Path)
|
||||
return nil
|
||||
case !d.OK:
|
||||
return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error)
|
||||
}
|
||||
if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil {
|
||||
return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err)
|
||||
}
|
||||
fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n",
|
||||
holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot))
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its
|
||||
// own directories on the machine:
|
||||
// kept, and retired at the self-check's next run.
|
||||
func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string {
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()}
|
||||
}
|
||||
var out []string
|
||||
for _, r := range records {
|
||||
if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned {
|
||||
continue
|
||||
}
|
||||
for _, m := range modules {
|
||||
if r.Module == m {
|
||||
out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+
|
||||
"never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)",
|
||||
r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,465 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func bytesOf(n int64) *int64 { return &n }
|
||||
|
||||
func when(t time.Time) *time.Time { return &t }
|
||||
|
||||
func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
|
||||
t.Helper()
|
||||
out := map[string]catalogue.Manifest{}
|
||||
for _, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[m.Module] = m
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const houseManifest = `{"module":"house","version":"1",
|
||||
"data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]},
|
||||
"resources":[{"id":"config","type":"directory","mode":"0700"}]}`
|
||||
|
||||
func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation {
|
||||
out := map[string]conditions.Observation{}
|
||||
for _, o := range linted(obs) {
|
||||
out[o.Kind] = o
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound,
|
||||
// by one changed rule, to the store on the control node, which made each an empty database — while
|
||||
// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired
|
||||
// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines
|
||||
// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says
|
||||
// its data there is irreplaceable (`kept-by`).
|
||||
func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) {
|
||||
var copies []consumerCopy
|
||||
for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} {
|
||||
copies = append(copies,
|
||||
consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"},
|
||||
consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"})
|
||||
}
|
||||
copies[1].Class = "irreplaceable" // the photo site's own database says so
|
||||
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
|
||||
if len(got) != 5 {
|
||||
t.Fatalf("%d findings for five empty replacements: %+v", len(got), got)
|
||||
}
|
||||
for i, o := range got {
|
||||
want := conditions.Warning
|
||||
if strings.Contains(o.ID, "mesh_home_board") {
|
||||
want = conditions.Urgent
|
||||
}
|
||||
_ = i
|
||||
if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" ||
|
||||
len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") {
|
||||
t.Errorf("%+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// While the old copy is still active (the first ten minutes), it is the same finding.
|
||||
copies[0].Retired = false
|
||||
copies[1].Class = "valuable"
|
||||
if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement {
|
||||
t.Fatalf("with the old copy still active: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and
|
||||
// a retired one at the old: nothing to say here; `cleanup` covers the old one.
|
||||
func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) {
|
||||
copies := []consumerCopy{
|
||||
{Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true},
|
||||
{Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)},
|
||||
}
|
||||
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||
t.Fatalf("a deliberate move raised %+v", got)
|
||||
}
|
||||
// Two small databases differing by less than the floor are not a finding either.
|
||||
copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20)
|
||||
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||
t.Fatalf("noise between two empty databases raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning,
|
||||
// since which one is empty cannot be told.
|
||||
func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) {
|
||||
copies := []consumerCopy{
|
||||
{Node: "home", Module: "minio", Consumer: "mesh_home_photos"},
|
||||
{Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"},
|
||||
}
|
||||
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
|
||||
if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The same incident for a module's own data: a module unassigned from one machine and assigned on
|
||||
// another starts over in an empty directory while its full one is kept, retired, where it was.
|
||||
func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) {
|
||||
now := time.Now()
|
||||
retired := now.Add(-time.Hour)
|
||||
records := []inventory.DataRecord{
|
||||
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||
Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)},
|
||||
{Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||
Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)},
|
||||
}
|
||||
got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
|
||||
o, ok := got[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
// The same of a valuable item is a warning.
|
||||
records[0].Class, records[1].Class = "valuable", "valuable"
|
||||
if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning {
|
||||
t.Fatalf("a valuable empty replacement: %+v", o)
|
||||
}
|
||||
records[0].Class, records[1].Class = "irreplaceable", "irreplaceable"
|
||||
// Two machines running a module on purpose, both active, keep two sets of data: nothing to say.
|
||||
records[0].RetiredAt = nil
|
||||
if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" {
|
||||
t.Fatalf("two active copies were read as a replacement: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss,
|
||||
// or a loss under the floor, is not a finding.
|
||||
func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||
now := time.Now()
|
||||
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)}
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank]
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
for _, peak := range []int64{500 << 20, 20 << 20} {
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||
t.Errorf("a peak of %d raised a shrink", peak)
|
||||
}
|
||||
}
|
||||
small := r
|
||||
small.Size = bytesOf(1 << 20)
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||
t.Error("a loss under the floor raised a shrink")
|
||||
}
|
||||
}
|
||||
|
||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||
now := time.Now()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)),
|
||||
LastBackup: when(now.Add(-72 * time.Hour))}
|
||||
got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))
|
||||
if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent {
|
||||
t.Fatalf("irreplaceable: %+v", got)
|
||||
}
|
||||
valuable := r
|
||||
valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning ||
|
||||
got[kindBackupStale].Severity != conditions.Warning {
|
||||
t.Fatalf("valuable: %+v", got)
|
||||
}
|
||||
never := r
|
||||
never.LastBackup = nil
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") {
|
||||
t.Fatalf("never backed up: %+v", o)
|
||||
}
|
||||
fresh := never
|
||||
fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now)
|
||||
if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An item retired more than thirty days waits for a person; less, it is only listed.
|
||||
func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) {
|
||||
now := time.Now()
|
||||
old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour)
|
||||
records := []inventory.DataRecord{
|
||||
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old},
|
||||
{Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent},
|
||||
}
|
||||
got := linted(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The holder's answer reads into measurements, by module and item.
|
||||
func TestTheHoldersAnswerIsRead(t *testing.T) {
|
||||
raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3,
|
||||
"data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps",
|
||||
"size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`)
|
||||
got, err := readHolder(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got["postgres"]["store"]
|
||||
if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil {
|
||||
t.Fatalf("%+v", m)
|
||||
}
|
||||
// An older holder, which says no data, reads as nothing measured rather than a failure.
|
||||
if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 {
|
||||
t.Fatalf("%v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it
|
||||
// measured.
|
||||
type fakeHolder struct {
|
||||
mu sync.Mutex
|
||||
modules []map[string]any
|
||||
}
|
||||
|
||||
func (f *fakeHolder) set(modules ...map[string]any) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.modules = modules
|
||||
}
|
||||
|
||||
func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) {
|
||||
t.Helper()
|
||||
sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func measuredHouse(path string, size int64, at time.Time) map[string]any {
|
||||
return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{
|
||||
"item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size,
|
||||
"last_write": at, "measured_at": at, "last_backup": at}}}
|
||||
}
|
||||
|
||||
// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty
|
||||
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
|
||||
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
|
||||
// up on another machine with an empty directory while the full one waits retired, that is urgent.
|
||||
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
now := time.Now()
|
||||
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||
var d13 probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeDataID {
|
||||
d13 = p
|
||||
}
|
||||
}
|
||||
run := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||
obs, err := probeData(probing, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||
if obs := run(); len(obs) != 0 {
|
||||
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "house")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||
}
|
||||
laptop.set()
|
||||
run()
|
||||
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||
}
|
||||
records, _ := inv.Data(ctx)
|
||||
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
|
||||
// Assigned on the anchor, onto an empty directory.
|
||||
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||
obs := findingsByKind(run())
|
||||
o, ok := obs[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
|
||||
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
|
||||
// whose path is gone is said.
|
||||
func TestTheArrayUnderDataIsWatched(t *testing.T) {
|
||||
now := time.Now()
|
||||
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"},
|
||||
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}`
|
||||
shelf := shelfFor(t, media)
|
||||
sick := false
|
||||
on := func(item string, healthy *bool) inventory.DataRecord {
|
||||
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false,
|
||||
Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}}
|
||||
}
|
||||
got := linted(dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
well := true
|
||||
if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("a healthy array raised %+v", got)
|
||||
}
|
||||
plain := on("films", nil)
|
||||
plain.Redundancy = nil
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("redundancy said and not found: %+v", o)
|
||||
}
|
||||
gone := on("films", &well)
|
||||
gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0)
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("a vanished library: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class:
|
||||
// the photo site's objects make the object store's data an urgent matter.
|
||||
func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
|
||||
objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"},
|
||||
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
||||
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`
|
||||
photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`
|
||||
shelf := shelfFor(t, objects, photos)
|
||||
bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket",
|
||||
Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}}
|
||||
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||
if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" {
|
||||
t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy)
|
||||
}
|
||||
now := time.Now()
|
||||
r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true,
|
||||
Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))}
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("the photos' store without a backup: %+v", o)
|
||||
}
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning {
|
||||
t.Fatalf("a valuable store without a backup: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition
|
||||
// naming every item on it, not one per item; and a partial walk's lower bound is never compared.
|
||||
func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) {
|
||||
now := time.Now()
|
||||
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"},
|
||||
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}`
|
||||
shelf := shelfFor(t, media, houseManifest)
|
||||
well := true
|
||||
on := func(item string, size int64) inventory.DataRecord {
|
||||
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable",
|
||||
Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||
Precision: "dataset tank/media: its whole size",
|
||||
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}}
|
||||
}
|
||||
films, shows := on("films", 30<<40), on("shows", 30<<40)
|
||||
peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40}
|
||||
got := linted(dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now),
|
||||
LastBackup: when(now), Precision: "partial: measured partially"}
|
||||
if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("a partial size was compared: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,643 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The controller's part in a delivery (novox/hq ADR 0239, to-be 47).
|
||||
//
|
||||
// **A delivery is mesh-delivery's; the walk is the controller's.** A delivery is one commit in one
|
||||
// repository from its pull request's head to every machine, and a delivery group a few of them sharing a
|
||||
// branch name; their states, order, holds and record belong to the module holding the `mesh-delivery`
|
||||
// seat. The controller keeps what it already owned — the planner, the gate, registration, sending, the
|
||||
// rollback — and the **walk**: one trunk commit's plan, tier by tier across the machines, one machine
|
||||
// first and judged at the gate (ADR 0236). What changes here is when a walk starts.
|
||||
//
|
||||
// - A walk that moves the controller, the node-engine, the node tools, the bus or mesh-delivery itself is
|
||||
// **on the controller's own path**: started by the merge, as every plan was. mesh-delivery cannot gate
|
||||
// or deliver itself, and nothing the core needs to be repaired may wait for it.
|
||||
// - Any other walk, **while the seat has a holder on record**, is opened by the merge and waits — nothing
|
||||
// asked, nothing registered, nothing sent — until mesh-delivery says `deliver`, or a person says
|
||||
// `plans go`. Nothing of it is registered before then, so no other send can carry it to a machine
|
||||
// before its turn (ADR 0236 §4a carries everything registered).
|
||||
// - With no holder on record, every walk starts at the merge, exactly as before this existed.
|
||||
//
|
||||
// The verbs mesh-delivery asks with are here: `delivery plan` (the planner's one answer for a diffset),
|
||||
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
|
||||
// go`, `delivery stop` and `delivery walks`.
|
||||
|
||||
// onTheControllersPath are the modules whose walk never waits for the delivery's owner, and what each is.
|
||||
var onTheControllersPath = map[string]string{
|
||||
"mesh-controller": "the controller",
|
||||
"mesh-host": "the node-engine",
|
||||
"node-tools": "the node tools",
|
||||
"nats": "the bus",
|
||||
catalogue.DeliverySeat: "the delivery's owner",
|
||||
}
|
||||
|
||||
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
|
||||
// holder on record. Read from the catalogue the merge handler already holds; never from whether the
|
||||
// holder answers, so a walk does not start by itself because mesh-delivery is down — that wait is said.
|
||||
func deliverySeatHeld(entries []inventory.Entry) bool {
|
||||
for _, e := range entries {
|
||||
if len(e.On) > 0 && e.Manifest.ClaimsSeat(catalogue.DeliverySeat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// awaitsFor is what a new walk waits for: nil when it starts at once — no holder on record, or a module
|
||||
// on the controller's own path among those it moves.
|
||||
func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDelivery {
|
||||
if !deliverySeatHeld(entries) {
|
||||
return nil
|
||||
}
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}
|
||||
}
|
||||
|
||||
// waitingNote is what a walk waiting for its word says, wherever it is read.
|
||||
func waitingNote(p inventory.Plan) string {
|
||||
return fmt.Sprintf("published; its walk waits for %s's word — `plans go %s --why …` starts it by hand",
|
||||
p.Delivery.Awaits, p.ID)
|
||||
}
|
||||
|
||||
// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its
|
||||
// first tier. Refused for a walk that does not wait.
|
||||
func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
switch {
|
||||
case !p.Open():
|
||||
return p, fmt.Errorf("%s is %s: there is no walk to start", p.ID, p.State)
|
||||
case p.Delivery == nil || p.Delivery.Awaits == "":
|
||||
return p, fmt.Errorf("%s waits for nobody: it started at its merge", p.ID)
|
||||
case p.Delivery.Go != nil:
|
||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||
p.Delivery.Go.Local().Format("15:04:05"))
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
||||
// builds and registers; nothing further is asked or sent.
|
||||
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
if !p.Open() {
|
||||
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
if p.Delivery == nil {
|
||||
p.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// orderMember is one member of a group, as mesh-delivery says it.
|
||||
type orderMember struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
PathsTruncated bool `json:"paths_truncated,omitempty"`
|
||||
Removed []string `json:"removed,omitempty"`
|
||||
ModuleDirs []string `json:"module_dirs,omitempty"`
|
||||
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
|
||||
CloneURL string `json:"clone_url,omitempty"`
|
||||
// After are the repositories its pull request says it goes after (`after: <repository>`).
|
||||
After []string `json:"after,omitempty"`
|
||||
}
|
||||
|
||||
// pull is the member as the forge announced it.
|
||||
func (m orderMember) pull() link.PullUpdated {
|
||||
owner, repo, _ := strings.Cut(m.Repository, "/")
|
||||
base := m.Base
|
||||
if base == "" {
|
||||
base = "main"
|
||||
}
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head,
|
||||
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
||||
}
|
||||
|
||||
// orderPair is one "before" among a group's members, and why.
|
||||
type orderPair struct {
|
||||
Before string `json:"before"`
|
||||
After string `json:"after"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// orderReach is what a member moves, as the planner says.
|
||||
type orderReach struct {
|
||||
Moved []string `json:"moved,omitempty"`
|
||||
Dependents []string `json:"dependents,omitempty"`
|
||||
New []string `json:"new,omitempty"`
|
||||
Manifests []string `json:"manifests,omitempty"`
|
||||
}
|
||||
|
||||
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one.
|
||||
type groupOrder struct {
|
||||
Order []string `json:"order"`
|
||||
Pairs []orderPair `json:"pairs,omitempty"`
|
||||
Cycle []string `json:"cycle,omitempty"`
|
||||
Reach map[string]orderReach `json:"reach,omitempty"`
|
||||
}
|
||||
|
||||
// The reasons a pair is ordered, in the words the delivery plan shows.
|
||||
const (
|
||||
orderDeclared = "declared"
|
||||
orderBuiltBy = "built by"
|
||||
orderVersionSkew = "version skew"
|
||||
orderEngineFirst = "engine before controller"
|
||||
)
|
||||
|
||||
// orderOf is a group's order (novox/hq ADR 0239 decision 4), pure. A member goes before another when:
|
||||
//
|
||||
// - its pull request is named in the other's `after:` lines (declared);
|
||||
// - it moves a module the other's moved modules are built by, stand on, package or declare (built by);
|
||||
// - it moves the controller and the other changes any module's manifest (version skew: the newer
|
||||
// controller parses what the newer manifest says) — the node-engine's excepted, which goes first;
|
||||
// - it moves the node-engine and the other moves the controller (the witness reads nothing the controller
|
||||
// does not yet grant, and a controller sends nothing an older engine would refuse — ADR 0236's rollout
|
||||
// order).
|
||||
//
|
||||
// Otherwise, by repository then id, so every reading gives one order. A pair both ways is a cycle: the
|
||||
// members in it are named, and none of them is ordered.
|
||||
func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge) groupOrder {
|
||||
out := groupOrder{Reach: reach}
|
||||
byID := map[string]orderMember{}
|
||||
for _, m := range members {
|
||||
byID[m.ID] = m
|
||||
}
|
||||
add := func(before, after, why string) {
|
||||
if before == after {
|
||||
return
|
||||
}
|
||||
for _, p := range out.Pairs {
|
||||
if p.Before == before && p.After == after {
|
||||
return
|
||||
}
|
||||
}
|
||||
out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why})
|
||||
}
|
||||
named := func(said, repository string) bool {
|
||||
said = strings.TrimSuffix(strings.TrimSpace(said), ".git")
|
||||
if strings.EqualFold(said, repository) {
|
||||
return true
|
||||
}
|
||||
_, repo, _ := strings.Cut(repository, "/")
|
||||
return strings.EqualFold(said, repo)
|
||||
}
|
||||
for _, a := range members {
|
||||
for _, b := range members {
|
||||
if a.ID == b.ID {
|
||||
continue
|
||||
}
|
||||
ra, rb := reach[a.ID], reach[b.ID]
|
||||
for _, said := range b.After {
|
||||
if named(said, a.Repository) {
|
||||
add(a.ID, b.ID, orderDeclared)
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
if slices.Contains(ra.Moved, e.To) && slices.Contains(rb.Moved, e.From) && e.From != e.To {
|
||||
add(a.ID, b.ID, orderBuiltBy)
|
||||
break
|
||||
}
|
||||
}
|
||||
if slices.Contains(ra.Moved, "mesh-controller") && len(rb.Manifests) > 0 &&
|
||||
!slices.Contains(rb.Moved, "mesh-controller") && !slices.Contains(rb.Moved, "mesh-host") {
|
||||
add(a.ID, b.ID, orderVersionSkew)
|
||||
}
|
||||
if slices.Contains(ra.Moved, "mesh-host") && slices.Contains(rb.Moved, "mesh-controller") &&
|
||||
!slices.Contains(ra.Moved, "mesh-controller") {
|
||||
add(a.ID, b.ID, orderEngineFirst)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out.Pairs, func(i, j int) bool {
|
||||
if out.Pairs[i].Before != out.Pairs[j].Before {
|
||||
return out.Pairs[i].Before < out.Pairs[j].Before
|
||||
}
|
||||
return out.Pairs[i].After < out.Pairs[j].After
|
||||
})
|
||||
// Kahn's walk, the next always the first by repository and id among those with nothing before them.
|
||||
waiting := map[string]int{}
|
||||
for _, m := range members {
|
||||
waiting[m.ID] = 0
|
||||
}
|
||||
for _, p := range out.Pairs {
|
||||
waiting[p.After]++
|
||||
}
|
||||
done := map[string]bool{}
|
||||
for len(done) < len(members) {
|
||||
var ready []orderMember
|
||||
for _, m := range members {
|
||||
if !done[m.ID] && waiting[m.ID] == 0 {
|
||||
ready = append(ready, m)
|
||||
}
|
||||
}
|
||||
if len(ready) == 0 {
|
||||
for _, m := range members {
|
||||
if !done[m.ID] {
|
||||
out.Cycle = append(out.Cycle, m.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out.Cycle)
|
||||
return out
|
||||
}
|
||||
sort.Slice(ready, func(i, j int) bool {
|
||||
if ready[i].Repository != ready[j].Repository {
|
||||
return ready[i].Repository < ready[j].Repository
|
||||
}
|
||||
return ready[i].ID < ready[j].ID
|
||||
})
|
||||
next := ready[0]
|
||||
done[next.ID] = true
|
||||
out.Order = append(out.Order, next.ID)
|
||||
for _, p := range out.Pairs {
|
||||
if p.Before == next.ID {
|
||||
waiting[p.After]--
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reachOfMembers is each member's reach, as the planner says it.
|
||||
func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) map[string]orderReach {
|
||||
out := map[string]orderReach{}
|
||||
for _, m := range members {
|
||||
s := pullScope(m.pull(), entries, read, edges)
|
||||
out[m.ID] = orderReach{Moved: s.Modules, Dependents: s.Dependents, New: s.New, Manifests: s.Manifests}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner:
|
||||
//
|
||||
// delivery plan --repository owner/repo --head <commit> [--base main] --paths a,b [--module-dirs …] [--removed …]
|
||||
// delivery order --members <json>
|
||||
// delivery check --group <id> --members <json>
|
||||
// delivery go <plan> [--by <who>] [--why <text>]
|
||||
// delivery stop <plan> --why <text> [--by <who>]
|
||||
// delivery walks [-n 50] [--plan <id>]
|
||||
func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("delivery plan|order|check|go|stop|walks")
|
||||
}
|
||||
sub, rest := args[0], args[1:]
|
||||
set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError)
|
||||
repository := set.String("repository", "", "owner/repository")
|
||||
base := set.String("base", "main", "the branch it merges into")
|
||||
head := set.String("head", "", "the commit at hand")
|
||||
paths := set.String("paths", "", "the files it changes, comma-separated")
|
||||
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
|
||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||
why := set.String("why", "", "why")
|
||||
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
||||
planID := set.String("plan", "", "one walk")
|
||||
positionals, err := parseAround(set, rest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var members []orderMember
|
||||
if *membersJSON != "" {
|
||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||
return fmt.Errorf("the members are not readable: %w", err)
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
answer := func(v any) error {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(v)
|
||||
}
|
||||
switch sub {
|
||||
case "plan":
|
||||
if *repository == "" || *head == "" && *paths == "" {
|
||||
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
|
||||
}
|
||||
m := orderMember{ID: *repository + "@" + *head, Repository: *repository, Base: *base, Head: *head,
|
||||
Paths: splitList(*paths), Removed: splitList(*removed)}
|
||||
if d := moduleDirsOf(*moduleDirs); d != nil {
|
||||
m.ModuleDirs, m.ModuleDirsSaid = *d, true
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s := pullScope(m.pull(), entries, read, edges)
|
||||
plan := changePlanOf(*repository, *base, *head, s.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
return answer(map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
|
||||
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()})
|
||||
case "order":
|
||||
if len(members) == 0 {
|
||||
return errors.New("delivery order --members <json>: a group has members")
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges))
|
||||
case "check":
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
// own path for a pull request, run because the delivery's owner asked.
|
||||
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"rechecked": members[0].ID})
|
||||
}
|
||||
if *group == "" || len(members) < 2 {
|
||||
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
|
||||
}
|
||||
id, err := askGroupCheck(ctx, open, *group, members)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"asked": id, "group": *group})
|
||||
case "go":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
|
||||
short(p.Commit), *by)
|
||||
return nil
|
||||
case "stop":
|
||||
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
||||
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
||||
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
||||
return nil
|
||||
case "walks":
|
||||
var walks []inventory.Plan
|
||||
if *planID != "" {
|
||||
p, err := inv.PlanByID(ctx, *planID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
walks = []inventory.Plan{p}
|
||||
} else {
|
||||
if walks, err = inv.OpenPlans(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, *limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if !slices.ContainsFunc(walks, func(w inventory.Plan) bool { return w.ID == p.ID }) {
|
||||
walks = append(walks, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||
"own-path": sortedKeysOf(ownPathWords())})
|
||||
}
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||
}
|
||||
|
||||
// ownPathWords is the controller's own path as words, for an answer.
|
||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||
|
||||
// theGraph is what the planner reads.
|
||||
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
|
||||
[]inventory.Edge, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return entries, read, edges, nil
|
||||
}
|
||||
|
||||
// groupPrimary is the head a group's composed check is run from: the one moving the controller, which then
|
||||
// judges the group by itself; else the one moving the node-engine, whose validator judges; else the first.
|
||||
func groupPrimary(members []orderMember, reach map[string]orderReach) int {
|
||||
for _, want := range []string{"mesh-controller", "mesh-host"} {
|
||||
for i, m := range members {
|
||||
if slices.Contains(reach[m.ID].Moved, want) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// askGroupCheck asks the build seat for a group's composed check: every head laid over the mesh in turn,
|
||||
// judged as one future state (novox/hq ADR 0239). The verdict comes back as `checked` with the group's id,
|
||||
// for mesh-delivery; the forge's holder sets no status from it.
|
||||
func askGroupCheck(ctx context.Context, open *stores, group string, members []orderMember) (string, error) {
|
||||
entries, read, edges, err := theGraph(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
reach := reachOfMembers(members, entries, read, edges)
|
||||
primary := groupPrimary(members, reach)
|
||||
p := members[primary].pull()
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// The gate runs over what any member moves; a judge from the primary alone.
|
||||
for _, m := range members {
|
||||
r := reach[m.ID]
|
||||
scope.Modules = appendNew(scope.Modules, r.Moved...)
|
||||
scope.Dependents = appendNew(scope.Dependents, r.Dependents...)
|
||||
}
|
||||
request, err := checkRequestFor(ctx, open, p, scope, entries)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Check.Group = group
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for i, m := range members {
|
||||
if i == primary {
|
||||
continue
|
||||
}
|
||||
mp := m.pull()
|
||||
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
|
||||
source := inventory.Source{Seat: gitSeat, Repository: mp.Owner + "/" + mp.Repo}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && sameRepository(e.Source.Repository, link.SourceMoved{Owner: mp.Owner, Repo: mp.Repo}) {
|
||||
source = e.Source
|
||||
break
|
||||
}
|
||||
}
|
||||
url := source.Repository
|
||||
if source.Seat != "" {
|
||||
url, err = clonedFromSeat(world, source.Seat, source.Repository)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
|
||||
}
|
||||
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths})
|
||||
}
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "group %s: asked %s to check %d head(s) composed together, as %s\n", group, seat,
|
||||
len(members), request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// appendNew appends what is not there yet.
|
||||
func appendNew(to []string, items ...string) []string {
|
||||
for _, i := range items {
|
||||
if !slices.Contains(to, i) {
|
||||
to = append(to, i)
|
||||
}
|
||||
}
|
||||
return to
|
||||
}
|
||||
|
||||
// shelfOf is the catalogue's manifests by name.
|
||||
func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
return shelf
|
||||
}
|
||||
|
||||
// sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the
|
||||
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
||||
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
||||
// comparison. Records arriving out of order are told apart by the plan's revision.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, p)
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, body); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// sayingOnTheBus runs a command that keeps walks or says verdicts with the bus to say them on: the serving
|
||||
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
|
||||
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
|
||||
// still runs; what it did is found by comparison.
|
||||
func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
if checkEvents != nil {
|
||||
return f()
|
||||
}
|
||||
ran := false
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bus := link.OverNATS{Conn: conn, JS: js}
|
||||
checkEvents = bus
|
||||
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
|
||||
ran = true
|
||||
return f()
|
||||
})
|
||||
if !ran {
|
||||
fmt.Fprintf(os.Stderr, "the bus cannot be reached (%v): what this does is not said, and is found by comparison\n", err)
|
||||
return f()
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table
|
||||
// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the
|
||||
// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery.<id>.stalled`; healer
|
||||
// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only
|
||||
// the next observation says whether it worked (ADR 0231).
|
||||
|
||||
// probeDeliveriesID is the probe that reads the delivery's owner.
|
||||
const probeDeliveriesID = "D14"
|
||||
|
||||
// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope.
|
||||
const kindDeliveryStalled = "stalled"
|
||||
|
||||
// deliveryOwnerWithin is how long the owner is given to answer.
|
||||
var deliveryOwnerWithin = 10 * time.Second
|
||||
|
||||
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
|
||||
type stalledLine struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||
|
||||
// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted;
|
||||
// a seam a test replaces. The holder's own refusal is an error naming it.
|
||||
var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
||||
granted = granted || v.Verb == verb
|
||||
}
|
||||
if !granted {
|
||||
return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
if conn == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error)
|
||||
}
|
||||
return unwrapToolResult(answer.Result), nil
|
||||
}
|
||||
|
||||
// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the
|
||||
// protocol's content list holding it as text.
|
||||
func unwrapToolResult(raw json.RawMessage) json.RawMessage {
|
||||
var wrapped struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) {
|
||||
return json.RawMessage(wrapped.Content[0].Text)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record,
|
||||
// or none answers (D3 says that one).
|
||||
func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) {
|
||||
if !held {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{})
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return nil, nil // the holder not running is D3's holder-silent, said once there
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lines []stalledLine
|
||||
if err := json.Unmarshal(raw, &lines); err != nil {
|
||||
return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// stalledObservations are the conditions of what is stalled.
|
||||
func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
|
||||
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
|
||||
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
|
||||
if l.operatorsOnly() {
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeDeliveries is D14: every delivery held past its state's bound is said.
|
||||
func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var conn *nats.Conn
|
||||
if d.js != nil {
|
||||
conn = d.js.Conn()
|
||||
}
|
||||
lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stalledObservations(lines), nil
|
||||
}
|
||||
|
||||
// --- H2, for a delivery ---------------------------------------------------------------------------------
|
||||
|
||||
// connOf is the bus a healer asks over.
|
||||
func (h *healing) connOf() *nats.Conn {
|
||||
if h.js == nil {
|
||||
return nil
|
||||
}
|
||||
return h.js.Conn()
|
||||
}
|
||||
|
||||
// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table
|
||||
// lets H2 take; never one whose state is the operator's.
|
||||
func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
lines, err := deliveriesStalled(ctx, h.connOf(), true)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
for _, l := range lines {
|
||||
if l.ID != c.Subject.ID {
|
||||
continue
|
||||
}
|
||||
if l.operatorsOnly() {
|
||||
return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil
|
||||
}
|
||||
|
||||
// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the
|
||||
// transition its table names. A refusal is no repair, said; the next observation says whether it worked.
|
||||
func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key})
|
||||
if err != nil {
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return "", "", err
|
||||
}
|
||||
return "closed nothing", oneLine(err.Error()), nil
|
||||
}
|
||||
var said string
|
||||
if json.Unmarshal(raw, &said) != nil {
|
||||
said = string(raw)
|
||||
}
|
||||
return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's
|
||||
// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`.
|
||||
|
||||
// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records
|
||||
// what close was asked.
|
||||
func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string {
|
||||
t.Helper()
|
||||
var closed []string
|
||||
was := askDeliveryOwner
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
switch verb {
|
||||
case "stalled":
|
||||
raw, _ := json.Marshal(lines)
|
||||
return raw, nil
|
||||
case "close":
|
||||
closed = append(closed, args["id"].(string))
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil
|
||||
}
|
||||
t.Fatalf("asked the owner %s", verb)
|
||||
return nil, nil
|
||||
}
|
||||
t.Cleanup(func() { askDeliveryOwner = was })
|
||||
return &closed
|
||||
}
|
||||
|
||||
func holdTheDeliverySeat(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
m := catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var twoStalled = []stalledLine{
|
||||
{ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s",
|
||||
H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"},
|
||||
{ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s",
|
||||
H2: "none: the state is the operator's", Says: "it waits for the operator"},
|
||||
}
|
||||
|
||||
func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
ownerAnswers(t, twoStalled, nil)
|
||||
d := &doctor{open: open}
|
||||
got, err := probeDeliveries(ctx, d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("with no holder on record D14 said %+v %v", got, err)
|
||||
}
|
||||
holdTheDeliverySeat(t, open)
|
||||
got, err = probeDeliveries(ctx, d)
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("D14 said %+v %v", got, err)
|
||||
}
|
||||
if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning ||
|
||||
got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") {
|
||||
t.Fatalf("the first is %+v", got[0])
|
||||
}
|
||||
if got[1].Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("a held delivery is not the operator's: %+v", got[1])
|
||||
}
|
||||
// The holder not running is D3's to say, not D14's.
|
||||
was := askDeliveryOwner
|
||||
askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) {
|
||||
return nil, link.ErrNothingServes
|
||||
}
|
||||
defer func() { askDeliveryOwner = was }()
|
||||
if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 {
|
||||
t.Fatalf("an owner that is down was said by D14: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a
|
||||
// refusal is no repair.
|
||||
func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
closed := ownerAnswers(t, twoStalled, nil)
|
||||
for _, o := range stalledObservations(twoStalled) {
|
||||
o.Source = probeDeliveriesID
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) {
|
||||
t.Fatalf("close was asked for %v", *closed)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// A refusal: closed nothing, said so.
|
||||
c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled")
|
||||
ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering"))
|
||||
act, said, err := repairDelivery(ctx, h, c)
|
||||
if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") {
|
||||
t.Fatalf("a refused close reads %q %q %v", act, said, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else
|
||||
// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read.
|
||||
func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
|
||||
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, verb := range []string{"stalled", "close"} {
|
||||
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
|
||||
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
}
|
||||
if _, err := askDeliveryOwner(t.Context(), nil, "stop", nil); err == nil || !strings.Contains(err.Error(), "grant") {
|
||||
t.Fatalf("a verb the grant does not name was asked: %v", err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := deliveriesStalled(t.Context(), conn, true); err != nil {
|
||||
t.Fatalf("an owner not running is D3's, not an error: %v", err)
|
||||
}
|
||||
lines, _ := json.Marshal(twoStalled)
|
||||
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}})
|
||||
sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) {
|
||||
body, _ := json.Marshal(link.Answer{Result: wrapped})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
got, err := deliveriesStalled(ctx, conn, true)
|
||||
if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" {
|
||||
t.Fatalf("over the bus: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
|
||||
type recordingDelivery struct {
|
||||
did []string
|
||||
grantErr error
|
||||
declareErr error
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
|
||||
for _, s := range sending {
|
||||
r.did = append(r.did, "grant "+s.node)
|
||||
}
|
||||
return r.grantErr
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
|
||||
if r.declareErr != nil {
|
||||
return "", r.declareErr
|
||||
}
|
||||
r.did = append(r.did, "declare "+s.node)
|
||||
return "digest-" + s.node, nil
|
||||
}
|
||||
|
||||
func ready(names ...string) []readyNode {
|
||||
var out []readyNode
|
||||
for _, n := range names {
|
||||
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
|
||||
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
|
||||
// carries the controller's own right to issue them, and goes first.
|
||||
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
|
||||
d := &recordingDelivery{}
|
||||
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
|
||||
if !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
|
||||
}
|
||||
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
|
||||
t.Fatalf("the digests sent were not answered: %v", digests)
|
||||
}
|
||||
|
||||
held := &recordingDelivery{}
|
||||
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
|
||||
if !reflect.DeepEqual(held.did, want) {
|
||||
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
|
||||
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
|
||||
// grant that would let the controller issue memberships is never held behind them.
|
||||
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
|
||||
// A failure naming no machine (the buckets): everything but the bus's machine.
|
||||
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
|
||||
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
|
||||
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
|
||||
!strings.Contains(err.Error(), "anchor, laptop not sent") {
|
||||
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
|
||||
}
|
||||
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
|
||||
}
|
||||
|
||||
// A membership that failed for one machine: that machine alone.
|
||||
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
|
||||
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
|
||||
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
|
||||
t.Fatalf("one machine's refused membership was not said: %v", err)
|
||||
}
|
||||
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
|
||||
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
|
||||
}
|
||||
|
||||
// The announced upgrade that hit it is asked again.
|
||||
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
|
||||
t.Fatal("an announcement whose send stopped at its grants is not asked again")
|
||||
}
|
||||
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
|
||||
t.Fatal("any failure is asked again, not only a grant's")
|
||||
}
|
||||
|
||||
// Nothing to send is nothing granted either.
|
||||
none := &recordingDelivery{grantErr: errors.New("never asked")}
|
||||
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
|
||||
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Whether the bus's machine goes first is read from the user list alone, by its digest.
|
||||
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
list := "users: [a, b]"
|
||||
if userListBehind(list, digestOf([]byte(list))) {
|
||||
t.Fatal("the list it was sent reads as behind")
|
||||
}
|
||||
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
|
||||
t.Fatal("a changed or never-sent list reads as current")
|
||||
}
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
// checked against. Among the machines it is moved to the front; not among them it is added only
|
||||
// when it is behind.
|
||||
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
names []string
|
||||
holder string
|
||||
behind bool
|
||||
want []string
|
||||
}{
|
||||
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
|
||||
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
|
||||
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
|
||||
} {
|
||||
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every
|
||||
// reading — the build agent before what it builds, the controller before a manifest that needs it, the
|
||||
// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered.
|
||||
func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "cat", Repository: "novox/mesh-catalog"},
|
||||
{ID: "ctl", Repository: "novox/mesh-controller"},
|
||||
{ID: "host", Repository: "novox/mesh-host"},
|
||||
{ID: "agent", Repository: "novox/build-agent"},
|
||||
{ID: "app", Repository: "novox/app", After: []string{"lab"}},
|
||||
{ID: "lab", Repository: "novox/lab"},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
|
||||
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
|
||||
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
|
||||
"agent": {Moved: []string{"build-agent"}},
|
||||
"app": {Moved: []string{"app"}},
|
||||
"lab": {Moved: []string{"lab"}},
|
||||
}
|
||||
edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
|
||||
got := orderOf(members, reach, edges)
|
||||
if len(got.Cycle) > 0 {
|
||||
t.Fatalf("a cycle where there is none: %v", got.Cycle)
|
||||
}
|
||||
// By repository among those with nothing before them: the build agent, the lab, then what waited on both.
|
||||
want := []string{"agent", "lab", "app", "host", "ctl", "cat"}
|
||||
if !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
|
||||
}
|
||||
why := map[string]string{}
|
||||
for _, p := range got.Pairs {
|
||||
why[p.Before+">"+p.After] = p.Why
|
||||
}
|
||||
for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew,
|
||||
"ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} {
|
||||
if why[pair] != reason {
|
||||
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
|
||||
}
|
||||
}
|
||||
// The same members in another order read the same.
|
||||
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
|
||||
if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) {
|
||||
t.Fatalf("another reading ordered %v", again.Order)
|
||||
}
|
||||
|
||||
// A declared order against an inferred one is a cycle: named, and nothing ordered.
|
||||
members[1].After = []string{"mesh-catalog"}
|
||||
got = orderOf(members, reach, edges)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) {
|
||||
t.Fatalf("the cycle is %v, ordered %v", got.Cycle, got.Order)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
|
||||
// record, and never one that moves a module on the controller's own path.
|
||||
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
|
||||
holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
|
||||
unassigned := holder
|
||||
unassigned.On = nil
|
||||
for _, c := range []struct {
|
||||
entries []inventory.Entry
|
||||
moved []string
|
||||
waits bool
|
||||
}{
|
||||
{nil, []string{"gitea"}, false},
|
||||
{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
|
||||
{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"node-tools"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"nats"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
|
||||
} {
|
||||
d := awaitsFor(c.entries, c.moved)
|
||||
if (d != nil) != c.waits {
|
||||
t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
|
||||
c.moved, d != nil, c.waits)
|
||||
}
|
||||
if d != nil && d.Awaits != catalogue.DeliverySeat {
|
||||
t.Errorf("waits for %q", d.Awaits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
|
||||
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
|
||||
// hand when the owner is down, and the owner's stop ends one as stopped.
|
||||
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
withConditionsInMemory(t)
|
||||
for _, name := range []string{"app", "mesh-delivery"} {
|
||||
m := catalogue.Manifest{Module: name, Version: "1"}
|
||||
if name == "mesh-delivery" {
|
||||
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
merge := func(commit string, paths ...string) inventory.Plan {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
|
||||
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 1)
|
||||
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
|
||||
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
|
||||
}
|
||||
return recent[0]
|
||||
}
|
||||
|
||||
// finish ends a walk as done, so the next merge has nothing of it to take over.
|
||||
finish := func(id string) {
|
||||
t.Helper()
|
||||
w, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.State = inventory.PlanDone
|
||||
if err := inv.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No holder on record: the merge starts its walk, as before.
|
||||
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
||||
if p.Waiting() || len(*asked) != 1 {
|
||||
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either.
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p = merge("c2bbbbbbbb", "modules/app/index.ts")
|
||||
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
|
||||
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
|
||||
}
|
||||
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
|
||||
t.Errorf("a waiting walk reads %q", line)
|
||||
}
|
||||
|
||||
// The delivery's word starts it.
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
||||
t.Fatal("a walk was let go twice")
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the word did not start the walk: %v", *asked)
|
||||
}
|
||||
got, err := inv.PlanByID(ctx, p.ID)
|
||||
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
|
||||
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
||||
}
|
||||
|
||||
// The delivery's owner's own merge never waits for it — and takes over what the older walk had not
|
||||
// built (app, folded in: ADR 0218), which goes with it on the controller's own path.
|
||||
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
||||
if p.Waiting() || len(*asked) != 4 {
|
||||
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The owner down: a person starts a waiting walk, with why.
|
||||
finish(p.ID)
|
||||
p = merge("c4dddddddd", "modules/app/index.ts")
|
||||
if !p.Waiting() {
|
||||
t.Fatal("the walk did not wait")
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a walk was started by hand without why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 5 {
|
||||
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
||||
}
|
||||
|
||||
// The owner's stop: failed, said as stopped by it.
|
||||
finish(p.ID)
|
||||
p = merge("c5eeeeeeee", "modules/app/index.ts")
|
||||
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
|
||||
"mesh-delivery for jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
|
||||
!strings.Contains(got.Note, "the operator stopped it") {
|
||||
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
|
||||
}
|
||||
|
||||
// Unassigned: the mesh is back on the controller's own path.
|
||||
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asks := len(*asked)
|
||||
if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
|
||||
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
|
||||
}
|
||||
}
|
||||
|
||||
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
|
||||
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
|
||||
[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
|
||||
{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
|
||||
[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
|
||||
{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
|
||||
{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
|
||||
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
|
||||
{"delivery-check", map[string]any{"group": "feat/x", "members": "[]"},
|
||||
[]string{"delivery", "check", "--group", "feat/x", "--members", "[]"}},
|
||||
{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"},
|
||||
[]string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}},
|
||||
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
|
||||
} {
|
||||
got, err := argvFor(c.verb, c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("deliver", map[string]any{}); err == nil {
|
||||
t.Error("deliver without a walk was composed")
|
||||
}
|
||||
if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
|
||||
t.Error("a stop without why was composed")
|
||||
}
|
||||
}
|
||||
|
||||
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
|
||||
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
|
||||
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
|
||||
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
var said inventory.Plan
|
||||
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
|
||||
said.Delivery.Go == nil {
|
||||
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a walk let go by a verb was not said")
|
||||
}
|
||||
if checkEvents != nil || inventory.PlanSaved != nil {
|
||||
t.Fatal("the command's own bus was left in place")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
|
||||
//
|
||||
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
|
||||
// design — every machine's declaration composes and validates, every resolver answers, every seat's
|
||||
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
|
||||
// condition it raises when the invariant does not hold. The serving controller runs the registry every
|
||||
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
|
||||
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
|
||||
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
|
||||
// machine: a controller that stops checking is itself said, through a channel that does not pass
|
||||
// through it.
|
||||
//
|
||||
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
|
||||
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
|
||||
|
||||
// The self-check's clocks.
|
||||
var (
|
||||
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
|
||||
// starts the first run waits, so what the controller hears at its start has arrived.
|
||||
doctorEvery = 5 * time.Minute
|
||||
doctorFirstAfter = time.Minute
|
||||
// probeWithin is each probe's bound.
|
||||
probeWithin = 30 * time.Second
|
||||
)
|
||||
|
||||
// The verdicts a probe can have.
|
||||
const (
|
||||
verdictPass = "pass"
|
||||
verdictFail = "fail"
|
||||
verdictFailedToRun = "failed-to-run"
|
||||
verdictDeferred = "deferred"
|
||||
)
|
||||
|
||||
// probe is one live invariant.
|
||||
type probe struct {
|
||||
ID string
|
||||
Asserts string
|
||||
From string
|
||||
// Kind is the condition kind raised when the invariant does not hold.
|
||||
Kind string
|
||||
// Raises are the other kinds its findings carry, each its own (a consumer missing, one far behind):
|
||||
// what a healer may be registered against (healers.go).
|
||||
Raises []string
|
||||
Phase int
|
||||
// Deferred says why it is not run yet; empty for one that is.
|
||||
Deferred string
|
||||
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
|
||||
// controller's grant names every one (a test over this registry): a probe whose question the bus
|
||||
// refuses checks nothing (D8, 2026-10-06).
|
||||
Asks []broker.SeatVerb
|
||||
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||
}
|
||||
|
||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
|
||||
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
|
||||
Kind: "archives-unheld", Phase: 1, run: probeArchives},
|
||||
{ID: "D5", Asserts: "exactly one lease holder — the key names this controller at its epoch, and the record " +
|
||||
"holds no other epoch open; no message from a stale epoch refused in the last interval",
|
||||
From: "issue 204", Kind: "lease-split", Phase: 2, run: probeLease},
|
||||
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Raises: []string{"consumer-lost", kindConsumerBehind},
|
||||
Phase: 1, run: probeConsumers},
|
||||
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
|
||||
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
|
||||
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
|
||||
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
|
||||
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
|
||||
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
|
||||
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
|
||||
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding " +
|
||||
"its cleanup", From: "ADR 0230", Kind: kindCleanupWaiting, Phase: 2, run: probeRetired},
|
||||
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
|
||||
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
|
||||
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
|
||||
{ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " +
|
||||
"written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " +
|
||||
"empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " +
|
||||
"valuable is retired, not forgotten", From: "issue 273, ADR 0233",
|
||||
Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale,
|
||||
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
|
||||
Phase: 2, run: probeData,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
|
||||
// The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by
|
||||
// the controller, and H2 works it through the owner's `close`.
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
// judged by on its first machine, run against every machine between upgrades too.
|
||||
{ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " +
|
||||
"ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth},
|
||||
{ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " +
|
||||
"build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth},
|
||||
{ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus",
|
||||
From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth},
|
||||
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " +
|
||||
"it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4,
|
||||
run: probeBusHealth},
|
||||
// The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its
|
||||
// condition until a newer build passes; each rollback a witness stands by is said.
|
||||
{ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " +
|
||||
"a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack,
|
||||
Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates},
|
||||
// The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus.
|
||||
{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
|
||||
"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
type probeVerdict struct {
|
||||
ID string `json:"id"`
|
||||
Verdict string `json:"verdict"`
|
||||
Found []string `json:"found,omitempty"`
|
||||
// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before:
|
||||
// raised if the next run sees them too (confirm.go). Not a pass, and not yet a condition.
|
||||
Unconfirmed []string `json:"unconfirmed,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
}
|
||||
|
||||
// doctorCounts are a run's verdicts, counted.
|
||||
type doctorCounts struct {
|
||||
Passed int `json:"passed"`
|
||||
Failed int `json:"failed"`
|
||||
FailedToRun int `json:"failed-to-run"`
|
||||
Deferred int `json:"deferred"`
|
||||
}
|
||||
|
||||
// doctorRun is one run of the registry, and the body of its heartbeat.
|
||||
type doctorRun struct {
|
||||
Run string `json:"run"`
|
||||
At time.Time `json:"at"`
|
||||
Started time.Time `json:"started"`
|
||||
Took string `json:"took"`
|
||||
IntervalSeconds int `json:"interval-seconds"`
|
||||
Counts doctorCounts `json:"counts"`
|
||||
Probes []probeVerdict `json:"probes"`
|
||||
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
|
||||
Controller string `json:"controller"`
|
||||
Why string `json:"why"`
|
||||
// Unsaid is how many condition transitions this controller could not say, since it started.
|
||||
Unsaid int `json:"unsaid,omitempty"`
|
||||
}
|
||||
|
||||
// doctor is the registry and what its probes need.
|
||||
type doctor struct {
|
||||
open *stores
|
||||
js *broker.JetStream
|
||||
keeper *conditions.Keeper
|
||||
teller conditions.Teller
|
||||
watchdogs *watchdogs
|
||||
host string
|
||||
// confirm holds back what one run alone saw of a finding a single look can be wrong about.
|
||||
confirm confirming
|
||||
|
||||
running sync.Mutex
|
||||
mu sync.Mutex
|
||||
last *doctorRun
|
||||
ended time.Time
|
||||
}
|
||||
|
||||
// lastRunEnded is when the last run ended; zero before the first.
|
||||
func (d *doctor) lastRunEnded() time.Time {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.ended
|
||||
}
|
||||
|
||||
// lastRun is the last run's verdict; nil before the first.
|
||||
func (d *doctor) lastRun() *doctorRun {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.last
|
||||
}
|
||||
|
||||
// keep runs the registry on its schedule until ctx ends.
|
||||
func (d *doctor) keep(ctx context.Context) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(doctorFirstAfter):
|
||||
}
|
||||
tick := time.NewTicker(doctorEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
// Only the controller acting checks the mesh on a schedule: one standing by would say a
|
||||
// heartbeat for a self-check that is not the mesh's.
|
||||
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
|
||||
d.runOnce(ctx, "the schedule")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var doctorRuns struct {
|
||||
sync.Mutex
|
||||
n uint64
|
||||
}
|
||||
|
||||
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
|
||||
// at a time: a person's `doctor run` during a scheduled one waits for it.
|
||||
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||
d.running.Lock()
|
||||
defer d.running.Unlock()
|
||||
doctorRuns.Lock()
|
||||
doctorRuns.n++
|
||||
n := doctorRuns.n
|
||||
doctorRuns.Unlock()
|
||||
started := time.Now()
|
||||
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
|
||||
|
||||
type result struct {
|
||||
obs []conditions.Observation
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
results := make([]result, len(probeRegistry))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range probeRegistry {
|
||||
if p.run == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int, p probe) {
|
||||
defer wg.Done()
|
||||
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
|
||||
defer cancel()
|
||||
began := time.Now()
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||
}
|
||||
}()
|
||||
obs, err := p.run(probing, d)
|
||||
done <- result{obs: obs, err: err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
r.took = time.Since(began)
|
||||
results[i] = r
|
||||
case <-probing.Done():
|
||||
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
|
||||
}
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var blind []conditions.Observation
|
||||
for i, p := range probeRegistry {
|
||||
v := probeVerdict{ID: p.ID}
|
||||
r := results[i]
|
||||
switch {
|
||||
case p.run == nil:
|
||||
v.Verdict = verdictDeferred
|
||||
run.Counts.Deferred++
|
||||
case r.err != nil:
|
||||
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
|
||||
run.Counts.FailedToRun++
|
||||
// A probe that could not run once — a question timed out on a loaded machine — is said in
|
||||
// the verdict at once, and raised as a condition when the next run cannot run it either.
|
||||
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
|
||||
Token: "failed", Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||
Said: firstLine(r.err.Error())})
|
||||
default:
|
||||
raise, held := d.confirm.pass(ctx, d.keeper, p.ID, kindedAs(r.obs, p.Kind))
|
||||
if err := d.keeper.Reconcile(ctx, p.ID, raise); err != nil {
|
||||
v.Error = "what it found could not be kept: " + err.Error()
|
||||
}
|
||||
for _, o := range held {
|
||||
v.Unconfirmed = append(v.Unconfirmed, o.Summary)
|
||||
}
|
||||
if len(raise) == 0 {
|
||||
v.Verdict = verdictPass
|
||||
run.Counts.Passed++
|
||||
} else {
|
||||
v.Verdict = verdictFail
|
||||
run.Counts.Failed++
|
||||
for _, o := range raise {
|
||||
v.Found = append(v.Found, o.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.run != nil {
|
||||
v.Took = r.took.Round(time.Millisecond).String()
|
||||
}
|
||||
run.Probes = append(run.Probes, v)
|
||||
}
|
||||
blind, _ = d.confirm.pass(ctx, d.keeper, sourceDoctor, blind)
|
||||
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
|
||||
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
|
||||
}
|
||||
ended := time.Now()
|
||||
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
|
||||
d.mu.Lock()
|
||||
d.last, d.ended = &run, ended
|
||||
d.mu.Unlock()
|
||||
d.sayHeartbeat(ctx, run)
|
||||
return run
|
||||
}
|
||||
|
||||
// sourceDoctor is what raises a probe's own failure to run.
|
||||
const sourceDoctor = "doctor"
|
||||
|
||||
// kindedAs gives each observation of a probe the probe's kind where it named none.
|
||||
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(obs))
|
||||
for _, o := range obs {
|
||||
if o.Kind == "" {
|
||||
o.Kind = kind
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
|
||||
// says it outward: the watcher hears nothing.
|
||||
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
|
||||
if d.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
|
||||
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
|
||||
"will say the self-check is silent: %v\n", run.Run, err)
|
||||
}
|
||||
}
|
||||
|
||||
// doctorFrom is the serving controller's self-check; nil in any other process.
|
||||
var doctorFrom *doctor
|
||||
|
||||
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
|
||||
func doctorCommand(ctx context.Context, args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("doctor [run|probes|signals] [--json]")
|
||||
}
|
||||
answer, err := doctorAnswer(ctx, sub)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(answer)
|
||||
}
|
||||
fmt.Print(doctorText(answer))
|
||||
return nil
|
||||
}
|
||||
|
||||
// doctorAnswer is what the verb answers, as data.
|
||||
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
switch sub {
|
||||
case "":
|
||||
if doctorFrom != nil {
|
||||
if run := doctorFrom.lastRun(); run != nil {
|
||||
return verdictAnswer(*run, time.Now()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||
}
|
||||
run, err := lastHeartbeat(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return verdictAnswer(run, time.Now()), nil
|
||||
case "run":
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
local, closeIt, err := localDoctor(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeIt()
|
||||
d = local
|
||||
}
|
||||
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||
case "probes":
|
||||
return probesAnswer(), nil
|
||||
case "signals":
|
||||
if doctorFrom == nil || doctorFrom.watchdogs == nil {
|
||||
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
|
||||
"hears them: ask it through the mesh-controller seat's doctor verb")
|
||||
}
|
||||
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
|
||||
}
|
||||
|
||||
// verdictAnswer is a run as the verb answers it, with its age.
|
||||
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it, " +
|
||||
"and one a single look can be wrong about — an unanswered question, a slow answer — is raised when two " +
|
||||
"runs in a row see it"}
|
||||
}
|
||||
|
||||
// probesAnswer is the registry.
|
||||
func probesAnswer() map[string]any {
|
||||
var out []map[string]any
|
||||
for _, p := range probeRegistry {
|
||||
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
|
||||
if len(p.Raises) > 0 {
|
||||
row["raises"] = p.Raises
|
||||
}
|
||||
if p.Deferred != "" {
|
||||
row["deferred"] = p.Deferred
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
|
||||
}
|
||||
|
||||
// signalsAnswer is every row of the signals table with the age of its newest signal.
|
||||
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
|
||||
var rows []map[string]any
|
||||
for _, r := range signalsTable {
|
||||
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
|
||||
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
|
||||
switch {
|
||||
case r.Deferred != "":
|
||||
row["deferred"] = r.Deferred
|
||||
case f == nil:
|
||||
row["newest"] = "not yet looked at"
|
||||
default:
|
||||
if err := r.needs(f); err != nil {
|
||||
row["blind"] = err.Error()
|
||||
} else if newest := r.newest(f); newest.IsZero() {
|
||||
row["newest"] = "none heard"
|
||||
} else {
|
||||
row["newest"] = newest.UTC().Format(time.RFC3339)
|
||||
row["age"] = f.now.Sub(newest).Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
out := map[string]any{"signals": rows, "every": watchEvery.String()}
|
||||
if !ticked.IsZero() {
|
||||
out["looked"] = ticked.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// doctorText is an answer as a person reads it.
|
||||
func doctorText(answer any) string {
|
||||
body, _ := json.Marshal(answer)
|
||||
var b strings.Builder
|
||||
var verdict struct {
|
||||
Run doctorRun `json:"run"`
|
||||
Age string `json:"age"`
|
||||
}
|
||||
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
|
||||
r := verdict.Run
|
||||
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
|
||||
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
|
||||
for _, p := range r.Probes {
|
||||
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
|
||||
for _, f := range p.Found {
|
||||
fmt.Fprintf(&b, " %s\n", f)
|
||||
}
|
||||
for _, f := range p.Unconfirmed {
|
||||
fmt.Fprintf(&b, " unconfirmed, raised if the next run sees it too: %s\n", f)
|
||||
}
|
||||
if p.Error != "" {
|
||||
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
pretty, _ := json.MarshalIndent(answer, "", " ")
|
||||
return string(pretty) + "\n"
|
||||
}
|
||||
|
||||
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
|
||||
// the serving controller answers `doctor` from.
|
||||
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
|
||||
var run doctorRun
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
|
||||
}
|
||||
return json.Unmarshal(msg.Data, &run)
|
||||
})
|
||||
return run, err
|
||||
}
|
||||
|
||||
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
|
||||
// its own connection, and its own keeper, closed after.
|
||||
func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
k, err := keeperOn(ctx, js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
jsCtx := js.Context()
|
||||
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
|
||||
host: controlHost(ctx, open.inventory)}
|
||||
return d, func() {
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
k.Close(flushing)
|
||||
js.Close()
|
||||
open.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
|
||||
func sortedFound(obs []conditions.Observation) []conditions.Observation {
|
||||
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
|
||||
return obs
|
||||
}
|
||||
|
||||
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
|
||||
type probeAsksKey struct{}
|
||||
|
||||
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
|
||||
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
|
||||
p, ok := ctx.Value(probeAsksKey{}).(probe)
|
||||
if !ok {
|
||||
return "a caller outside the self-check", false
|
||||
}
|
||||
for _, v := range p.Asks {
|
||||
if v.Seat == seat && v.Verb == verb {
|
||||
return p.ID, true
|
||||
}
|
||||
}
|
||||
return p.ID, false
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/net/dns/dnsmessage"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
|
||||
|
||||
// withProbes runs the test with a registry of its own.
|
||||
func withProbes(t *testing.T, probes ...probe) {
|
||||
t.Helper()
|
||||
before := probeRegistry
|
||||
probeRegistry = probes
|
||||
t.Cleanup(func() { probeRegistry = before })
|
||||
}
|
||||
|
||||
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||
var broken atomic.Bool
|
||||
broken.Store(true)
|
||||
withProbes(t,
|
||||
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return []conditions.Observation{failing}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return nil, errors.New("the store is away")
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
<-ctx.Done()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||
)
|
||||
before := probeWithin
|
||||
probeWithin = 200 * time.Millisecond
|
||||
t.Cleanup(func() { probeWithin = before })
|
||||
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||
// A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot
|
||||
// run it either (novox/hq issue 277): one timed-out question is not a probe gone blind.
|
||||
first := d.runOnce(t.Context(), "a test")
|
||||
if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", first.Counts)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" {
|
||||
t.Fatalf("after one run, open: %+v", open)
|
||||
}
|
||||
run := d.runOnce(t.Context(), "a test")
|
||||
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", run.Counts)
|
||||
}
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key+"="+c.Kind)
|
||||
}
|
||||
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||
"probe.P4.failed=probe-failed"} {
|
||||
if !slices.Contains(keys, want) {
|
||||
t.Errorf("%s is not open: %v", want, keys)
|
||||
}
|
||||
}
|
||||
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||
if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent {
|
||||
t.Fatalf("said %v", told.Names)
|
||||
}
|
||||
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||
t.Fatal("the run is not the last verdict")
|
||||
}
|
||||
body, _ := json.Marshal(run)
|
||||
var shape map[string]any
|
||||
_ = json.Unmarshal(body, &shape)
|
||||
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
|
||||
// Mended: the next run clears every one of them.
|
||||
broken.Store(false)
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("a passing run left open %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **The registry says what each probe asserts**, and a probe not built says why and when.
|
||||
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, p := range probeRegistry {
|
||||
if seen[p.ID] {
|
||||
t.Errorf("%s twice", p.ID)
|
||||
}
|
||||
seen[p.ID] = true
|
||||
if p.Asserts == "" || p.From == "" || p.Kind == "" {
|
||||
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
|
||||
}
|
||||
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
|
||||
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
|
||||
if !seen[id] {
|
||||
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
|
||||
// stopped is S10 (signals_test.go).
|
||||
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
|
||||
w := &watchdogs{started: time.Now().Add(-time.Hour)}
|
||||
d := &doctor{watchdogs: w, host: "anchor"}
|
||||
got, err := probeWatchdogs(t.Context(), d)
|
||||
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
w.ticked = time.Now()
|
||||
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
|
||||
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
|
||||
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{"rival-one", "rival-two"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||
linted(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
|
||||
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||
answerAs := func(rcode dnsmessage.RCode) string {
|
||||
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
go func() {
|
||||
buf := make([]byte, 1500)
|
||||
for {
|
||||
n, from, err := conn.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var q dnsmessage.Message
|
||||
if q.Unpack(buf[:n]) != nil {
|
||||
continue
|
||||
}
|
||||
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
|
||||
if q.Questions[0].Type == dnsmessage.TypeA {
|
||||
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
|
||||
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
|
||||
} else {
|
||||
reply.RCode = rcode
|
||||
}
|
||||
packed, _ := reply.Pack()
|
||||
_, _ = conn.WriteTo(packed, from)
|
||||
}
|
||||
}()
|
||||
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||
return port
|
||||
}
|
||||
before := resolverPort
|
||||
t.Cleanup(func() { resolverPort = before })
|
||||
|
||||
resolverPort = answerAs(dnsmessage.RCodeSuccess)
|
||||
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
|
||||
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
|
||||
t.Fatalf("%v %v %v", v4, rcode, err)
|
||||
}
|
||||
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
|
||||
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
|
||||
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
|
||||
}
|
||||
resolverPort = answerAs(dnsmessage.RCodeNameError)
|
||||
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
|
||||
t.Fatalf("NXDOMAIN read as %v", rcode)
|
||||
}
|
||||
}
|
||||
|
||||
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||
got, err := p(t.Context(), d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := info.Config
|
||||
cfg.MaxMsgsPerSubject = 3
|
||||
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumers, err := probeConsumers(t.Context(), d)
|
||||
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||
}
|
||||
streams, err := probeStreams(t.Context(), d)
|
||||
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
heard := make(chan *nats.Msg, 16)
|
||||
for _, subject := range broker.BusAdvisories {
|
||||
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
api, _ := jetstream.New(conn)
|
||||
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kinds := map[string]string{}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for len(kinds) < 2 {
|
||||
select {
|
||||
case m := <-heard:
|
||||
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||
kinds[a.Kind] = a.Said
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the bus said only %v", kinds)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||
t.Fatalf("%v", kinds)
|
||||
}
|
||||
}
|
||||
|
||||
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
|
||||
// machine read as behind right after a push sent it the current build — it says the digest-named
|
||||
// directory it runs from, and the mesh holds a commit).
|
||||
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
|
||||
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
|
||||
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
|
||||
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
|
||||
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
|
||||
}}
|
||||
delivered := deliveredVersions(m)
|
||||
if !slices.Equal(delivered, []string{"31045596c83a"}) {
|
||||
t.Fatalf("%v", delivered)
|
||||
}
|
||||
commit := "1545b00a9f0c"
|
||||
for _, c := range []struct {
|
||||
reported string
|
||||
behind bool
|
||||
}{
|
||||
{"31045596c83a", false}, // the live case: current, and was called behind
|
||||
{"0123456789ab", true}, // another delivery
|
||||
{"1545b00a", false}, // placed by hand, stamped with the commit
|
||||
{"", false}, // not said
|
||||
} {
|
||||
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
|
||||
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
|
||||
}
|
||||
}
|
||||
if engineBehind("31045596c83a", nil, commit) {
|
||||
t.Error("behind a mesh that holds no delivered build")
|
||||
}
|
||||
}
|
||||
|
||||
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
|
||||
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
|
||||
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
|
||||
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked := 0
|
||||
for _, p := range probeRegistry {
|
||||
for _, v := range p.Asks {
|
||||
asked++
|
||||
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
|
||||
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
|
||||
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
|
||||
}
|
||||
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
|
||||
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
if asked == 0 {
|
||||
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
|
||||
}
|
||||
// And a probe asking what it did not declare is refused before anything is sent.
|
||||
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
|
||||
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
|
||||
!strings.Contains(err.Error(), "does not declare") {
|
||||
t.Fatalf("an undeclared question was asked: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// subjectMatches is the bus's matching of a permission pattern against a subject.
|
||||
func subjectMatches(pattern, subject string) bool {
|
||||
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
|
||||
for i, tok := range p {
|
||||
if tok == ">" {
|
||||
return len(s) > i
|
||||
}
|
||||
if i >= len(s) || (tok != "*" && tok != s[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(s)
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
|
||||
// store at all, so anything that tried to record or register would fail rather than pass quietly.
|
||||
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
|
||||
err := builds{}.Built(t.Context(), link.BuildResult{
|
||||
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
|
||||
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The mark survives the wire both ways: asked as a dry run, answered as one.
|
||||
func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
|
||||
raw, _ := json.Marshal(link.BuildRequest{ID: "build-1", Repository: "r", DryRun: true})
|
||||
var asked link.BuildRequest
|
||||
if err := json.Unmarshal(raw, &asked); err != nil || !asked.DryRun {
|
||||
t.Fatalf("the request lost its dry-run mark: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(link.BuildResult{ID: "build-1", DryRun: true})
|
||||
var answered link.BuildResult
|
||||
if err := json.Unmarshal(raw, &answered); err != nil || !answered.DryRun {
|
||||
t.Fatalf("the outcome lost its dry-run mark: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(link.BuildResult{ID: "build-2"})
|
||||
if string(raw) != `{"id":"build-2","repository":"","on":""}` {
|
||||
t.Fatalf("an ordinary outcome carries a dry-run mark: %s", raw)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||
if asked.IsZero() || result.ID == "" {
|
||||
return
|
||||
}
|
||||
subject := result.Module
|
||||
if subject == "" {
|
||||
subject = result.Repository
|
||||
}
|
||||
detail := "built"
|
||||
if result.Failed != "" {
|
||||
detail = "failed: " + firstLine(result.Failed)
|
||||
}
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
case inventory.DurationHeartbeatGap:
|
||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said
|
||||
// it reads one, and to no other: an older node-engine refuses a key it does not know, whole.
|
||||
|
||||
// bodiesDelivery records each send as the mesh does, and keeps the bodies.
|
||||
type bodiesDelivery struct {
|
||||
recordedDelivery
|
||||
bodies map[string][]byte
|
||||
}
|
||||
|
||||
func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
b.bodies[s.node] = body
|
||||
return b.recordedDelivery.declare(ctx, s, body)
|
||||
}
|
||||
|
||||
func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
epoch := uint64(57)
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return epoch, nil }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
|
||||
anchor, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carried := func(node string) (epoch float64, has bool) {
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal(d.bodies[node], &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epoch, has = envelope["epoch"].(float64)
|
||||
return epoch, has
|
||||
}
|
||||
if e, has := carried("anchor"); !has || e != 57 {
|
||||
t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"])
|
||||
}
|
||||
if _, has := carried("laptop"); has {
|
||||
t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"])
|
||||
}
|
||||
|
||||
// A new holder of the lease is not a change of the machine: neither reads as behind.
|
||||
epoch = 58
|
||||
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
sent, err := inv.Outstanding(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if would[node] != sent {
|
||||
t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A process that may not act composes nothing and sends nothing: its number is not taken.
|
||||
func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") {
|
||||
t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused)
|
||||
}
|
||||
anchor, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||
if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 {
|
||||
t.Fatalf("a controller without the lease took sequence %d", seq)
|
||||
}
|
||||
|
||||
// And at the send itself: the gate every declaration passes.
|
||||
gate := link.ActingGate
|
||||
link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { link.ActingGate = gate })
|
||||
if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil ||
|
||||
!strings.Contains(err.Error(), "lost the lease") {
|
||||
t.Fatalf("a declaration was let through the gate: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,645 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/validate"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The facts snapshot (novox/hq to-be 45 §9, ADR 0227 rule 9): what a merge check needs to judge a change
|
||||
// against the mesh that runs, written by the controller to the artifact store, where the build seat reads
|
||||
// it. internal/facts says what it holds and what it never holds; this composes it from the store and
|
||||
// keeps it current.
|
||||
|
||||
// factsEvery is how often the snapshot is composed. It is kept when it moved — a machine, an
|
||||
// assignment, a seat, a setting, a build — or once a day when nothing did, so its age says the
|
||||
// controller is still writing it (S14).
|
||||
var factsEvery = 10 * time.Minute
|
||||
|
||||
// factsDaily is how old a snapshot of an unchanged mesh may grow before it is written again.
|
||||
const factsDaily = 24 * time.Hour
|
||||
|
||||
// factsStaleAfter is S14's bound: a snapshot older than this is one no check should be fed.
|
||||
const factsStaleAfter = 48 * time.Hour
|
||||
|
||||
// factsExport is what this controller knows of the snapshot it keeps: when the newest was taken, its
|
||||
// content, and the last attempt's error.
|
||||
type factsExport struct {
|
||||
mu sync.Mutex
|
||||
taken time.Time
|
||||
content string
|
||||
digest string
|
||||
err error
|
||||
began time.Time
|
||||
}
|
||||
|
||||
// exportedFacts is this process's export, read by S14.
|
||||
var exportedFacts = &factsExport{}
|
||||
|
||||
func (e *factsExport) last() (taken time.Time, digest string, began time.Time, err error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return e.taken, e.digest, e.began, e.err
|
||||
}
|
||||
|
||||
func (e *factsExport) kept(f snapshot.Facts, content, digest string) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.taken, e.content, e.digest, e.err = f.Taken, content, digest, nil
|
||||
}
|
||||
|
||||
func (e *factsExport) failed(err error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.err = err
|
||||
}
|
||||
|
||||
// exportingFacts keeps the snapshot current for as long as this controller holds the lease: ctx ends
|
||||
// when it stops acting.
|
||||
func exportingFacts(ctx context.Context, open *stores, busVersion func() string) {
|
||||
exportedFacts.mu.Lock()
|
||||
exportedFacts.began = time.Now()
|
||||
exportedFacts.mu.Unlock()
|
||||
// What the store holds already, so a restarted controller neither writes an unchanged snapshot again
|
||||
// nor reads its age as zero.
|
||||
if address, err := factsStore(ctx, open); err == nil {
|
||||
if body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag); err == nil {
|
||||
if f, err := snapshot.Decode(body); err == nil {
|
||||
content, _ := f.Content()
|
||||
exportedFacts.kept(f, content, digest)
|
||||
}
|
||||
}
|
||||
}
|
||||
failing := ""
|
||||
first := time.NewTimer(time.Minute)
|
||||
defer first.Stop()
|
||||
tick := time.NewTicker(factsEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-first.C:
|
||||
case <-tick.C:
|
||||
}
|
||||
wrote, err := exportFacts(ctx, open, busVersion(), false)
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
exportedFacts.failed(err)
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("the facts snapshot cannot be kept: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("the facts snapshot is kept again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
if wrote != "" {
|
||||
fmt.Printf("the facts snapshot moved and is kept as %s\n", short(strings.TrimPrefix(wrote, "sha256:")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// exportFacts composes the snapshot and keeps it when it moved, or when the one kept is a day old, or
|
||||
// when told to. Answers the digest it kept, empty when it kept nothing.
|
||||
func exportFacts(ctx context.Context, open *stores, busVersion string, force bool) (string, error) {
|
||||
f, err := gatherFacts(ctx, open, busVersion)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
content, err := f.Content()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
exportedFacts.mu.Lock()
|
||||
unchanged := content == exportedFacts.content && time.Since(exportedFacts.taken) < factsDaily
|
||||
exportedFacts.mu.Unlock()
|
||||
if unchanged && !force {
|
||||
return "", nil
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
address, err := factsStore(ctx, open)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest, err := (artifacts.Store{Address: address}).PutTagged(ctx, snapshot.Repository, snapshot.Tag, snapshot.MediaType, body)
|
||||
if err != nil && digest == "" {
|
||||
return "", err
|
||||
}
|
||||
exportedFacts.kept(f, content, digest)
|
||||
return digest, err
|
||||
}
|
||||
|
||||
// factsStore is the artifact store as this controller reaches it.
|
||||
func factsStore(ctx context.Context, open *stores) (string, error) {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if address == "" {
|
||||
return "", errors.New("the artifact store is not on the private network, so there is nowhere to keep the facts")
|
||||
}
|
||||
return address, nil
|
||||
}
|
||||
|
||||
// gatherFacts composes one snapshot from the store: read only, nothing made, nothing sent.
|
||||
func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot.Facts, error) {
|
||||
inv := open.inventory
|
||||
f := snapshot.Facts{Format: snapshot.Format, Taken: time.Now().UTC(), Controller: snapshot.Build{Version: version}}
|
||||
f.Versions.Bus = busVersion
|
||||
storeVersion, err := inv.ServerVersion(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, fmt.Errorf("the store will not say its version: %w", err)
|
||||
}
|
||||
f.Versions.Store = storeVersion
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
place := map[string]inventory.Overlay{}
|
||||
for _, o := range overlays {
|
||||
place[o.Name] = o
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
|
||||
// **Every name first**, so text read afterwards — a setting naming a machine, a problem naming a
|
||||
// site — has it replaced wherever it appears.
|
||||
scrub := snapshot.NewScrubber()
|
||||
domains := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
scrub.Machine(n.Name)
|
||||
if n.Account != "" && n.Account != "root" {
|
||||
scrub.Account(n.Account)
|
||||
}
|
||||
d, err := inv.PublicDomainOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
domains[n.Name] = scrub.Domain(d)
|
||||
}
|
||||
for _, o := range overlays {
|
||||
scrub.Site(o.Site)
|
||||
}
|
||||
|
||||
// What a merge check runs in and reads beside it, as the build seat would be asked for it.
|
||||
if held, err := inv.Held(ctx); err == nil {
|
||||
for language, reference := range builder.ToolchainsOf(held) {
|
||||
if f.Versions.Toolchains == nil {
|
||||
f.Versions.Toolchains = map[string]string{}
|
||||
}
|
||||
f.Versions.Toolchains[language] = catalogue.Recorded(reference)
|
||||
}
|
||||
} else {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if dir, core := coreModules[e.Manifest.Module]; core && !e.Provided && e.Source.Repository != "" {
|
||||
for d, ref := range besideRefs(dir, current[e.Manifest.Module].Commit) {
|
||||
if f.Beside == nil {
|
||||
f.Beside = map[string]string{}
|
||||
}
|
||||
f.Beside[d] = ref
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if c, ok := current["mesh-controller"]; ok {
|
||||
f.Controller.Commit = c.Commit
|
||||
}
|
||||
|
||||
gens, gensErr := generators(ctx, open)
|
||||
hostShelf := shelf[hostModule]
|
||||
meshWide := map[string]bool{}
|
||||
engines := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
m := snapshot.Machine{Name: scrub.Machine(n.Name), Length: len(n.Name), Adopted: n.Adopted,
|
||||
AccountHome: scrub.Text(n.AccountHome), NodeEngine: n.HostVersion, PublicDomain: domains[n.Name]}
|
||||
switch n.Account {
|
||||
case "", "root":
|
||||
m.Account = n.Account
|
||||
default:
|
||||
m.Account = scrub.Account(n.Account)
|
||||
}
|
||||
if n.HostVersion != "" {
|
||||
engines[n.HostVersion] = true
|
||||
}
|
||||
m.System = systemOf(hostShelf, n.HostVersion)
|
||||
m.Libc = libcOf(m.System)
|
||||
reported, err := inv.DescribedOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
|
||||
outward, err := inv.OutwardLinksOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, l := range outward {
|
||||
m.OutwardLinks = append(m.OutwardLinks, scrub.Text(l))
|
||||
}
|
||||
capabilities, err := inv.Profile(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, c := range capabilities {
|
||||
kept := snapshot.Capability{Name: c.Name, Present: c.Present}
|
||||
// The detail only where it is a version: everything else a detector says — a ruleset, a
|
||||
// device, a path — is the machine's own business and no check reads it.
|
||||
if c.Present && (c.Name == "container-runtime" || c.Name == "package-manager") {
|
||||
kept.Detail = scrub.Text(c.Detail)
|
||||
}
|
||||
m.Capabilities = append(m.Capabilities, kept)
|
||||
}
|
||||
if o, ok := place[n.Name]; ok {
|
||||
m.Site, m.Hub, m.Public, m.OnNetwork = scrub.Site(o.Site), o.Hub, o.Endpoint != "", o.Address != ""
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Assigned = assigned
|
||||
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
if known && slices.Contains(assigned, broker.RuntimeModule) {
|
||||
m.NodeTools = sent[broker.RuntimeModule]
|
||||
}
|
||||
pins, err := inv.PinsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for provision, c := range pins {
|
||||
m.Pins = append(m.Pins, snapshot.Pin{Provision: provision, Machine: scrub.Machine(c.Node), Module: c.Module})
|
||||
}
|
||||
for _, module := range assigned {
|
||||
held, err := inv.SecretsOf(ctx, n.Name, module)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, h := range held {
|
||||
if h.Origin == inventory.OriginAccepted {
|
||||
m.Accepted = append(m.Accepted, snapshot.Accepted{Module: module, Name: h.Name,
|
||||
Provider: scrub.Machine(h.Provider), Local: h.Local})
|
||||
}
|
||||
}
|
||||
layers, err := inv.SettingsFor(ctx, n.Name, module)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, layer := range layers {
|
||||
if layer.From == catalogue.MeshWideLayer {
|
||||
if !meshWide[module] {
|
||||
meshWide[module] = true
|
||||
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
}
|
||||
continue
|
||||
}
|
||||
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
}
|
||||
}
|
||||
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
|
||||
if ctx.Err() != nil {
|
||||
return snapshot.Facts{}, ctx.Err()
|
||||
}
|
||||
f.Machines = append(f.Machines, m)
|
||||
}
|
||||
for e := range engines {
|
||||
f.Versions.NodeEngines = append(f.Versions.NodeEngines, e)
|
||||
}
|
||||
|
||||
// Seats and their holders, and from them the roles a machine is named by.
|
||||
roles := map[string][]string{}
|
||||
seats := map[string]*snapshot.Seat{}
|
||||
for _, h := range holdings {
|
||||
key := h.Claim + "\x00" + h.Scope
|
||||
s, ok := seats[key]
|
||||
if !ok {
|
||||
s = &snapshot.Seat{Name: h.Claim, Scope: h.Scope}
|
||||
seats[key] = s
|
||||
}
|
||||
s.Holders = append(s.Holders, snapshot.Holder{Machine: scrub.Machine(h.Node), Module: h.Module})
|
||||
if h.Scope == catalogue.ScopeMesh {
|
||||
role := "holds " + h.Claim
|
||||
if h.Claim == catalogue.ControllerSeatName {
|
||||
role = "the control node"
|
||||
}
|
||||
roles[h.Node] = append(roles[h.Node], role)
|
||||
}
|
||||
}
|
||||
for _, s := range seats {
|
||||
f.Seats = append(f.Seats, *s)
|
||||
}
|
||||
for i := range f.Machines {
|
||||
for _, n := range nodes {
|
||||
if scrub.Machine(n.Name) != f.Machines[i].Name {
|
||||
continue
|
||||
}
|
||||
f.Machines[i].Roles = roles[n.Name]
|
||||
if f.Machines[i].Hub {
|
||||
f.Machines[i].Roles = append(f.Machines[i].Roles, "the hub")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every module, as the mesh holds it, and where it is built from.
|
||||
newest := map[string]inventory.Source{}
|
||||
count := map[string]int{}
|
||||
for _, e := range entries {
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
mod := snapshot.Module{Name: e.Manifest.Module, Repository: e.Source.Repository, Path: e.Source.Path,
|
||||
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
||||
Manifest: raw}
|
||||
for _, r := range read[e.Manifest.Module] {
|
||||
mod.Reads = append(mod.Reads, r.Repository)
|
||||
}
|
||||
f.Modules = append(f.Modules, mod)
|
||||
if e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
count[e.Source.Repository]++
|
||||
if was, ok := newest[e.Source.Repository]; !ok || e.Source.Seen.After(was.Seen) {
|
||||
newest[e.Source.Repository] = e.Source
|
||||
}
|
||||
}
|
||||
for repository, s := range newest {
|
||||
commit := s.Head
|
||||
if commit == "" {
|
||||
commit = s.BuiltFrom
|
||||
}
|
||||
f.Sources = append(f.Sources, snapshot.Source{Repository: repository, Commit: commit, Modules: count[repository]})
|
||||
}
|
||||
for _, e := range edges {
|
||||
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
|
||||
}
|
||||
f.Sorted()
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// declarationFacts is how one machine's declaration composes now, as the next push would compose it and
|
||||
// without making anything (D1's composition), and whether the node-engine's validator takes it.
|
||||
func declarationFacts(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
|
||||
gensErr error, scrub *snapshot.Scrubber) snapshot.Declaration {
|
||||
var d snapshot.Declaration
|
||||
if gensErr != nil {
|
||||
d.Problems = []string{scrub.Text("the private network cannot be computed: " + oneLine(gensErr.Error()))}
|
||||
return d
|
||||
}
|
||||
declared, problems, err := composedAndValidated(ctx, open, node, gens, Foreseeing)
|
||||
if err != nil {
|
||||
d.Problems = []string{scrub.Text(oneLine(err.Error()))}
|
||||
return d
|
||||
}
|
||||
for _, p := range problems {
|
||||
d.Problems = append(d.Problems, scrub.Text(p))
|
||||
}
|
||||
d.Composes = len(problems) == 0
|
||||
if body, err := declared.Body(); err == nil {
|
||||
d.Digest = fmt.Sprintf("sha256:%x", sha256.Sum256(body))
|
||||
}
|
||||
// Scrubbed like every other word: a resource is named after the machine a grant is for.
|
||||
for _, r := range resourceNames(declared.Resources) {
|
||||
d.Resources = append(d.Resources, scrub.Text(r))
|
||||
}
|
||||
for module, why := range declared.leftOutWhy {
|
||||
if d.LeftOut == nil {
|
||||
d.LeftOut = map[string]string{}
|
||||
}
|
||||
d.LeftOut[module] = scrub.Text(why)
|
||||
}
|
||||
for _, o := range declared.withheld {
|
||||
d.Withheld = append(d.Withheld, scrub.Text(o.String()))
|
||||
}
|
||||
for _, u := range declared.unbound {
|
||||
d.Unbound = append(d.Unbound, scrub.Text(u.String()))
|
||||
}
|
||||
sort.Strings(d.Withheld)
|
||||
sort.Strings(d.Unbound)
|
||||
return d
|
||||
}
|
||||
|
||||
// composedAndValidated composes one machine's declaration — as a push would (Allocating) or as the next
|
||||
// push will without making anything (Foreseeing) — with the order it was last sent, and runs the
|
||||
// node-engine's own validator over the body. An error is that it did not compose; problems are what the
|
||||
// validator refuses.
|
||||
func composedAndValidated(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
|
||||
choosing Choosing) (sendable, []string, error) {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
declared, err := declarationWith(ctx, open, node, plan, settings, gens, choosing)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
if declared.Sequence, err = open.inventory.Sequence(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
return declared, validate.Declaration(body), nil
|
||||
}
|
||||
|
||||
// resourceNames are a declaration's resources as `type:id`, sorted.
|
||||
func resourceNames(resources []map[string]any) []string {
|
||||
out := make([]string, 0, len(resources))
|
||||
for _, r := range resources {
|
||||
kind, _ := r["type"].(string)
|
||||
id, _ := r["id"].(string)
|
||||
out = append(out, kind+":"+id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// systemOf is the system a node-engine of that version was built for, read from the build the mesh
|
||||
// holds: the artifact a resource delivered into `versions/<version>` came from is named for its system
|
||||
// (`host-arch`). Empty when the version is not a delivered one — an engine placed by hand.
|
||||
func systemOf(host catalogue.Manifest, version string) string {
|
||||
if version == "" {
|
||||
return ""
|
||||
}
|
||||
for _, r := range host.Resources {
|
||||
path, _ := r["path"].(string)
|
||||
if !strings.HasSuffix(path, "/versions/"+version) {
|
||||
continue
|
||||
}
|
||||
source, _ := r["source"].(string)
|
||||
for _, part := range strings.Split(source, "/") {
|
||||
if system, ok := strings.CutPrefix(part, "host-"); ok && system != "" {
|
||||
return system
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// libcOf is the C library of a system the node-engine is built for.
|
||||
func libcOf(system string) string {
|
||||
switch system {
|
||||
case "arch":
|
||||
return "glibc"
|
||||
case "alpine":
|
||||
return "musl"
|
||||
case "android":
|
||||
return "bionic"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// factsCommand is `facts`: what the controller keeps, and keeping it now.
|
||||
//
|
||||
// facts the snapshot the artifact store holds: when, which, how many machines
|
||||
// facts show the same, whole, as JSON
|
||||
// facts export compose and keep one now
|
||||
// facts compose compose one and print it, keeping nothing
|
||||
func factsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("facts", flag.ContinueOnError)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := ""
|
||||
if len(rest) > 0 {
|
||||
what = rest[0]
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
switch what {
|
||||
case "", "show":
|
||||
address, err := factsStore(ctx, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if what == "show" {
|
||||
_, err := os.Stdout.Write(body)
|
||||
return err
|
||||
}
|
||||
f, err := snapshot.Decode(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the facts snapshot kept as %s:%s is %s, taken %s (%s ago) by controller %s\n",
|
||||
snapshot.Repository, snapshot.Tag, short(strings.TrimPrefix(digest, "sha256:")),
|
||||
f.Taken.Format(time.RFC3339), ago(time.Since(f.Taken)), orNone(f.Controller.Commit))
|
||||
fmt.Printf(" %d machine(s), %d module(s), %d seat(s); the longest machine name is %d characters\n",
|
||||
len(f.Machines), len(f.Modules), len(f.Seats), f.Longest())
|
||||
fmt.Printf(" the bus runs %s, the store %s\n", orNone(f.Versions.Bus), orNone(f.Versions.Store))
|
||||
for _, m := range f.Machines {
|
||||
state := "composes"
|
||||
if !m.Declaration.Composes {
|
||||
state = "does NOT compose: " + strings.Join(m.Declaration.Problems, "; ")
|
||||
}
|
||||
fmt.Printf(" %-12s %s; %d module(s); %s\n", m.Name, m.Described(), len(m.Assigned), state)
|
||||
}
|
||||
return nil
|
||||
case "export", "compose":
|
||||
busVersion := ""
|
||||
if server, err := connectLink(ctx, nil, nil, nil); err == nil {
|
||||
busVersion = busVersionOf(server)
|
||||
server.Close()
|
||||
}
|
||||
if what == "compose" {
|
||||
f, err := gatherFacts(ctx, open, busVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = os.Stdout.Write(append(body, '\n'))
|
||||
return err
|
||||
}
|
||||
digest, err := exportFacts(ctx, open, busVersion, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the facts snapshot is kept as %s:%s, %s\n", snapshot.Repository, snapshot.Tag, digest)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("facts [show|export|compose], not %q", what)
|
||||
}
|
||||
|
||||
// busVersionOf is the bus server's release, as it told this connection.
|
||||
func busVersionOf(server *link.Server) string {
|
||||
if bus, ok := server.Bus().(link.OverNATS); ok && bus.Conn != nil {
|
||||
return bus.Conn.ConnectedServerVersion()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
)
|
||||
|
||||
// The facts snapshot (novox/hq to-be 45 §9): composed from the store, every machine under a pseudonym
|
||||
// of its name's length, and nothing of the installation in it — no secret, no address, no name.
|
||||
|
||||
// aMeshWithSecrets is aMesh with a provider and its consumers, a value given by hand, settings carrying
|
||||
// a password, an address and a machine's name, and a push's worth of credentials made.
|
||||
func aMeshWithSecrets(t *testing.T) (*stores, []string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"},
|
||||
Resources: []map[string]any{{"id": "config", "type": "file", "path": "/etc/files/config.json",
|
||||
"mode": "0600", "content": "{}", "merge": "json"}}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
secrets := []string{"Hunter2-Is-Not-A-Password-9f8e7d", "0123456789abcdefABCDEF0123456789zz"}
|
||||
if err := open.inventory.SetSettings(ctx, "", "files", map[string]any{
|
||||
"admin_password": secrets[0], "upstream": "10.77.0.9", "hub": "anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "laptop", "files", map[string]any{
|
||||
"note": "reach me at 192.168.1.135, token " + secrets[1]}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A push's worth of composition, which makes the pair credential the consumer is sent.
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, _, err := composedAndValidated(ctx, open, node, gens, Allocating); err != nil {
|
||||
t.Fatalf("%s does not compose: %v", node, err)
|
||||
}
|
||||
}
|
||||
issued, err := open.inventory.SecretsFrom(ctx, "anchor")
|
||||
if err != nil || len(issued) == 0 {
|
||||
t.Fatalf("no credential was made for the consumer: %v", err)
|
||||
}
|
||||
for _, s := range issued {
|
||||
// Sealed, never kept plain (ADR 0004): the sealed blobs are what the store holds, and none may leave.
|
||||
secrets = append(secrets, s.ForConsumer, s.ForProvider)
|
||||
}
|
||||
return open, secrets
|
||||
}
|
||||
|
||||
func TestTheFactsCarryNoSecretNoAddressAndNoName(t *testing.T) {
|
||||
open, secrets := aMeshWithSecrets(t)
|
||||
f, err := gatherFacts(t.Context(), open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(body)
|
||||
for _, s := range secrets {
|
||||
if s != "" && strings.Contains(text, s) {
|
||||
t.Errorf("a secret is in the snapshot: %q", s)
|
||||
}
|
||||
}
|
||||
for _, leaked := range []string{"anchor", "laptop", "10.77.0.", "192.168.1.135", ".example:51820"} {
|
||||
if strings.Contains(text, leaked) {
|
||||
t.Errorf("%q is in the snapshot", leaked)
|
||||
}
|
||||
}
|
||||
// Every address it carries is a documentation address.
|
||||
for _, a := range regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b`).FindAllString(text, -1) {
|
||||
if !strings.HasPrefix(a, "192.0.2.") && !strings.HasPrefix(a, "198.51.100.") && !strings.HasPrefix(a, "203.0.113.") {
|
||||
t.Errorf("%s is an address outside the documentation ranges", a)
|
||||
}
|
||||
}
|
||||
|
||||
// What a check needs is there: every machine, its length, its modules, its declaration composing.
|
||||
if len(f.Machines) != 2 || f.Longest() != len("laptop") {
|
||||
t.Fatalf("machines %+v, longest %d", f.Machines, f.Longest())
|
||||
}
|
||||
anchor := snapshot.Pseudonym("machine", "anchor")
|
||||
m, ok := f.Machine(anchor)
|
||||
if !ok || !m.Hub || !strings.Contains(m.Described(), "the hub") || len(m.Name) != len("anchor") {
|
||||
t.Fatalf("the anchor reads as %+v", m)
|
||||
}
|
||||
if !m.Declaration.Composes || m.Declaration.Digest == "" || len(m.Declaration.Resources) == 0 {
|
||||
t.Errorf("the anchor's declaration reads as %+v", m.Declaration)
|
||||
}
|
||||
laptop, _ := f.Machine(snapshot.Pseudonym("machine", "laptop"))
|
||||
if strings.Join(laptop.Assigned, ",") != "files,mesh-wireguard" && !strings.Contains(strings.Join(laptop.Assigned, ","), "files") {
|
||||
t.Errorf("the laptop's assignments read as %v", laptop.Assigned)
|
||||
}
|
||||
var meshWide map[string]any
|
||||
for _, s := range f.Settings {
|
||||
if s.Module == "files" {
|
||||
meshWide = s.Values
|
||||
}
|
||||
}
|
||||
if meshWide["admin_password"] != snapshot.Withheld || meshWide["hub"] != anchor {
|
||||
t.Errorf("the mesh-wide settings read as %v", meshWide)
|
||||
}
|
||||
if f.Versions.Bus != "2.11.17" || f.Versions.Store == "" {
|
||||
t.Errorf("versions read as %+v", f.Versions)
|
||||
}
|
||||
var objects bool
|
||||
for _, mod := range f.Modules {
|
||||
objects = objects || mod.Name == "objects" && len(mod.Manifest) > 0
|
||||
}
|
||||
if !objects {
|
||||
t.Error("the modules the mesh holds are not in the snapshot")
|
||||
}
|
||||
|
||||
// And an unchanged mesh is the same content a moment later.
|
||||
again, err := gatherFacts(t.Context(), open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := f.Content()
|
||||
b, _ := again.Content()
|
||||
if a != b {
|
||||
t.Error("two snapshots of an unchanged mesh differ, so it would be written again every ten minutes")
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package main
|
||||
|
||||
// The broker opening belongs only on the node that listens on it (novox/hq: it leaked onto
|
||||
// every enrolled node's declaration, opening a from-anywhere hole for a port nothing there
|
||||
// serves). foundationPortsFor is the scope.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
func TestTheBrokerHostGetsTheFoundationOpening(t *testing.T) {
|
||||
broker := catalogue.Manifest{Module: "lavinmq", Listens: []catalogue.Listening{
|
||||
{Port: 5671, Protocol: "tcp", From: "mesh"},
|
||||
{Port: 5672, Protocol: "tcp", From: "mesh"},
|
||||
}}
|
||||
got := foundationPortsFor(5671, []catalogue.Manifest{broker})
|
||||
if len(got) != 1 || got[0] != 5671 {
|
||||
t.Fatalf("the node that listens on the broker port keeps it; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatOnlyDialsTheBrokerGetsNoOpening(t *testing.T) {
|
||||
// ace's set: things that reach the broker as a client, none listening on 5671.
|
||||
ace := []catalogue.Manifest{
|
||||
{Module: "plex", Listens: []catalogue.Listening{{Port: 32400, Protocol: "tcp", From: "anywhere"}}},
|
||||
{Module: "postgres", Listens: []catalogue.Listening{{Port: 5432, Protocol: "tcp", From: "mesh"}}},
|
||||
}
|
||||
if got := foundationPortsFor(5671, ace); got != nil {
|
||||
t.Fatalf("a node that only dials out opens nothing for the broker; got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,942 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0236, to-be 45
|
||||
// §8, ADR 0227 rule 8).
|
||||
//
|
||||
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||
// the component's health — the core's health definitions, or a module's own — passing three times over
|
||||
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
|
||||
//
|
||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||
// build goes back to the one the first machine ran before, and that machine is sent it: the ordinary
|
||||
// path, again. Once per build: the verdict is written before the send, and a verdict once written is
|
||||
// not written over. Said as a condition, urgent for the core and when it could not be put back, and as
|
||||
// the event `rolled-back`.
|
||||
|
||||
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
|
||||
var (
|
||||
// gateSettle is how long the first machine must stay healthy, at the least.
|
||||
gateSettle = 2 * time.Minute
|
||||
// gateBound is how long after the apply the build may take to become healthy.
|
||||
gateBound = 10 * time.Minute
|
||||
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
|
||||
gatePasses = 3
|
||||
gateEvery = 40 * time.Second
|
||||
)
|
||||
|
||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
||||
const gateProbe = "DG"
|
||||
|
||||
// The kinds a gate raises.
|
||||
const (
|
||||
kindRolledBack = "rolled-back"
|
||||
kindRollbackFailed = "rollback-failed"
|
||||
)
|
||||
|
||||
// coreComponent is the core component a module is, as a witness names it — controller, node-engine,
|
||||
// node-tools — or empty: the core is judged by its health definitions, anything else by its own health.
|
||||
func coreComponent(module string) string {
|
||||
switch module {
|
||||
case catalogue.ControllerSeatName:
|
||||
return lease.ComponentController
|
||||
case hostModule:
|
||||
return lease.ComponentEngine
|
||||
case broker.RuntimeModule:
|
||||
return lease.ComponentNodeTools
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// health is a judging's word on one machine.
|
||||
type health int
|
||||
|
||||
const (
|
||||
healthGood health = iota
|
||||
healthNotYet
|
||||
healthBroken
|
||||
)
|
||||
|
||||
// served is what one machine's node tools answered the bus's discovery with.
|
||||
type served struct {
|
||||
runtime bool
|
||||
tools map[string]bool
|
||||
}
|
||||
|
||||
// gateFacts is what one judging reads, gathered once for every machine it judges.
|
||||
type gateFacts struct {
|
||||
now time.Time
|
||||
reports map[string]inventory.Reported
|
||||
// engines is each machine's node-engine build as it last reported it.
|
||||
engines map[string]string
|
||||
// served is what the bus's discovery answered; servedErr why it could not be asked.
|
||||
served map[string]served
|
||||
servedErr error
|
||||
// open are the open conditions; openErr why they could not be read (nil keeper: not judged).
|
||||
open []conditions.Condition
|
||||
openErr error
|
||||
judged bool
|
||||
// rolledBack is what each machine's witnesses say they decided.
|
||||
rolledBack map[string][]lease.Rollback
|
||||
// holder is who holds the controller lease, for judging the controller.
|
||||
holder *lease.Holder
|
||||
holderErr error
|
||||
}
|
||||
|
||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||
// variable so a test can hand a judging its facts.
|
||||
var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
inv := open.inventory
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: witnessed.all()}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, n := range nodes {
|
||||
f.engines[n.Name] = n.HostVersion
|
||||
}
|
||||
if d := doctorFrom; d != nil {
|
||||
if d.keeper != nil {
|
||||
f.judged = true
|
||||
f.open, f.openErr = d.keeper.Open(ctx)
|
||||
}
|
||||
if d.js != nil {
|
||||
f.served, f.servedErr = servedOnTheBus(ctx, d.js.Conn())
|
||||
} else {
|
||||
f.servedErr = errors.New("this controller has no bus to ask")
|
||||
}
|
||||
} else {
|
||||
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
|
||||
}
|
||||
if theLease != nil {
|
||||
h, found, err := theLease.holder(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
f.holderErr = err
|
||||
case found:
|
||||
f.holder = &h
|
||||
}
|
||||
if component != lease.ComponentController && f.holderErr != nil {
|
||||
f.holderErr = nil // read only for the controller's own judging
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// judgeHealth is one machine's health for a module's new build, from what one judging read: healthy,
|
||||
// not yet (with what is wanting), or healthBroken — a witness put it back, or the machine refused or failed
|
||||
// what it was sent. Pure.
|
||||
func judgeHealth(module, component string, m catalogue.Manifest, machine string, since time.Time, f gateFacts) (health, string) {
|
||||
// A witness's verdict made since the build was sent: the build failed its health there. One that
|
||||
// could not judge at all (unwitnessed) is not a verdict on the build.
|
||||
for _, r := range f.rolledBack[machine] {
|
||||
if component == "" || r.Component != component || r.Outcome == lease.OutcomeUnwitnessed || r.At.Before(since) {
|
||||
continue
|
||||
}
|
||||
return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component,
|
||||
short(r.From), r.Outcome, r.Why)
|
||||
}
|
||||
r, said := f.reports[machine]
|
||||
switch {
|
||||
case !said || r.At == nil || !r.Current:
|
||||
return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine)
|
||||
case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused:
|
||||
return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome)
|
||||
case r.Outcome != inventory.OutcomeApplied:
|
||||
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
|
||||
}
|
||||
// **No new condition about it**: about the machine itself, or naming the module on that machine,
|
||||
// raised since the judging began. The gate's own are not evidence about the build.
|
||||
if f.judged {
|
||||
if f.openErr != nil {
|
||||
return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " +
|
||||
firstLine(f.openErr.Error())
|
||||
}
|
||||
for _, c := range f.open {
|
||||
if c.Source == gateProbe || c.Raised.Before(since) {
|
||||
continue
|
||||
}
|
||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||
(c.Subject.Scope == conditions.ScopeMachine && c.Subject.ID == machine)
|
||||
if !onIt {
|
||||
continue
|
||||
}
|
||||
if c.Subject.Scope == conditions.ScopeMachine || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
|
||||
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch component {
|
||||
case lease.ComponentEngine:
|
||||
// The node-engine has reported its current declaration under its own build.
|
||||
want := deliveredVersions(m)
|
||||
if len(want) > 0 && !slices.Contains(want, f.engines[machine]) {
|
||||
return healthNotYet, fmt.Sprintf("%s's node-engine reports build %s, not the new %s", machine,
|
||||
orNotKnown(f.engines[machine]), strings.Join(want, " or "))
|
||||
}
|
||||
case lease.ComponentNodeTools:
|
||||
// The node tools are announced and answer.
|
||||
if f.servedErr != nil {
|
||||
return healthNotYet, "whether the node tools answer cannot be asked: " + firstLine(f.servedErr.Error())
|
||||
}
|
||||
if !f.served[machine].runtime {
|
||||
return healthNotYet, fmt.Sprintf("the node tools on %s do not answer the bus", machine)
|
||||
}
|
||||
case lease.ComponentController:
|
||||
// The new controller holds the lease and says it is ready.
|
||||
switch {
|
||||
case f.holderErr != nil:
|
||||
return healthNotYet, "who holds the controller lease cannot be read: " + firstLine(f.holderErr.Error())
|
||||
case f.holder == nil:
|
||||
return healthNotYet, "no controller holds the lease"
|
||||
case f.holder.Taken.Before(since.Add(-time.Minute)):
|
||||
return healthNotYet, fmt.Sprintf("the lease is held since %s, by a controller older than the new build",
|
||||
f.holder.Taken.UTC().Format(time.RFC3339))
|
||||
case f.holder.Health == nil || !f.holder.Health.Ready:
|
||||
why := "the controller holding the lease does not say it is ready"
|
||||
if f.holder.Health != nil && f.holder.Health.Why != "" {
|
||||
why += ": " + f.holder.Health.Why
|
||||
}
|
||||
return healthNotYet, why
|
||||
}
|
||||
default:
|
||||
// A module's tools answer, where it has any and the machine runs the node tools that serve them.
|
||||
if len(m.Tools) > 0 {
|
||||
if f.servedErr != nil {
|
||||
return healthNotYet, "whether its tools are served cannot be asked: " + firstLine(f.servedErr.Error())
|
||||
}
|
||||
if s := f.served[machine]; s.runtime && !s.tools[module] {
|
||||
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is
|
||||
// the judging's facts without those conditions, on what each names among the modules the send moved,
|
||||
// and whole what holds the machine back as a whole.
|
||||
type machineWord struct {
|
||||
facts gateFacts
|
||||
on map[string]string
|
||||
whole string
|
||||
}
|
||||
|
||||
// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
|
||||
// yet reported applying the node tools it was last sent.
|
||||
const kindCoreBehind = "core-behind"
|
||||
|
||||
// aboutTheMachine sorts the conditions raised about a machine itself since a send was made there
|
||||
// (novox/hq issue 281). The gate read every one of them as the module's it was kept on: a machine-level
|
||||
// condition caused by anything else in the send — or by a tier sent one module at a time — failed that
|
||||
// module, at the bound, with a reason that was never about it.
|
||||
//
|
||||
// - one naming a module the send moved is that module's;
|
||||
// - the core being behind (D10) is the core's: of the node-engine or the node tools when the send
|
||||
// moved them — inside their settle window, which is the gate's bound — and otherwise no evidence about
|
||||
// what was sent: a send not yet applied the machine's reports already say, and a newer core build
|
||||
// that no plan sends is not this send's;
|
||||
// - anything else holds the machine back as a whole: everything the send moved there waits on it, and
|
||||
// fails with it, together, at the bound, with that reason.
|
||||
//
|
||||
// Pure.
|
||||
func aboutTheMachine(machine string, moved []string, since time.Time, f gateFacts) machineWord {
|
||||
w := machineWord{facts: f, on: map[string]string{}}
|
||||
if !f.judged || f.openErr != nil {
|
||||
return w
|
||||
}
|
||||
var core []string
|
||||
for _, m := range moved {
|
||||
if c := coreComponent(m); c == lease.ComponentEngine || c == lease.ComponentNodeTools {
|
||||
core = append(core, m)
|
||||
}
|
||||
}
|
||||
kept := make([]conditions.Condition, 0, len(f.open))
|
||||
for _, c := range f.open {
|
||||
aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine ||
|
||||
slices.Contains(c.Subject.Also, machine))
|
||||
if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) {
|
||||
kept = append(kept, c)
|
||||
continue
|
||||
}
|
||||
said := fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
||||
parts := strings.Split(c.Subject.ID, ".")
|
||||
var named []string
|
||||
for _, m := range moved {
|
||||
if slices.Contains(parts, m) {
|
||||
named = append(named, m)
|
||||
}
|
||||
}
|
||||
coreBehind := c.Kind == kindCoreBehind || strings.HasSuffix(c.Key, "."+kindCoreBehind)
|
||||
switch {
|
||||
case len(named) > 0:
|
||||
for _, m := range named {
|
||||
if _, already := w.on[m]; !already {
|
||||
w.on[m] = said
|
||||
}
|
||||
}
|
||||
case coreBehind && len(core) > 0:
|
||||
for _, m := range core {
|
||||
if _, already := w.on[m]; !already {
|
||||
w.on[m] = said + " (its settle window runs to the gate's bound)"
|
||||
}
|
||||
}
|
||||
case coreBehind:
|
||||
// Not what the send moved: said nowhere against it.
|
||||
default:
|
||||
if w.whole == "" {
|
||||
w.whole = machine + " as a whole: " + said
|
||||
}
|
||||
}
|
||||
}
|
||||
w.facts.open = kept
|
||||
return w
|
||||
}
|
||||
|
||||
// judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass
|
||||
// counted, a pass missed (and why), or the verdict. Answers the verdict once there is one.
|
||||
func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
|
||||
running []string, now time.Time) (string, error) {
|
||||
g := state.Gate
|
||||
if g == nil {
|
||||
// Judged from the send: a witness's verdict, a condition, the bound — all counted from when the
|
||||
// first machine was sent the build.
|
||||
start := now
|
||||
if state.FirstAt != nil {
|
||||
start = *state.FirstAt
|
||||
}
|
||||
var machines []string
|
||||
for _, n := range state.First {
|
||||
if slices.Contains(running, n) {
|
||||
machines = append(machines, n)
|
||||
}
|
||||
}
|
||||
g = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, From: state.Previous,
|
||||
To: state.Commit, Since: &start}
|
||||
state.Gate = g
|
||||
}
|
||||
pairs := []judged{}
|
||||
for _, n := range g.Machines {
|
||||
pairs = append(pairs, judged{module: module, node: n})
|
||||
}
|
||||
return judgeMoves(ctx, open, g, pairs, now)
|
||||
}
|
||||
|
||||
// judged is one module on one machine, as a gate judges it.
|
||||
type judged struct{ module, node string }
|
||||
|
||||
// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and
|
||||
// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is
|
||||
// wanting, and which modules), or the verdict. Answers the verdict once there is one.
|
||||
func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) {
|
||||
if g.Verdict != "" {
|
||||
return g.Verdict, nil
|
||||
}
|
||||
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
|
||||
return "", nil
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) {
|
||||
pairs = append(pairs, judged{module: c.Module, node: c.Node})
|
||||
}
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
facts, err := gatherGateFacts(ctx, open, g.Component)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
byMachine := map[string][]string{}
|
||||
for _, j := range pairs {
|
||||
byMachine[j.node] = append(byMachine[j.node], j.module)
|
||||
}
|
||||
words := map[string]machineWord{}
|
||||
for node, moved := range byMachine {
|
||||
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
|
||||
}
|
||||
worst, why := healthGood, ""
|
||||
var failing []string
|
||||
broken := map[string]bool{}
|
||||
for _, j := range pairs {
|
||||
w := words[j.node]
|
||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||
if h == healthGood {
|
||||
if on, named := w.on[j.module]; named {
|
||||
h, said = healthNotYet, on
|
||||
} else if w.whole != "" {
|
||||
h, said = healthNotYet, w.whole
|
||||
}
|
||||
}
|
||||
if h != healthGood && !slices.Contains(failing, j.module) {
|
||||
failing = append(failing, j.module)
|
||||
}
|
||||
if h == healthBroken {
|
||||
broken[j.module] = true
|
||||
}
|
||||
if h > worst {
|
||||
worst, why = h, said
|
||||
} else if h == worst && h != healthGood && why == "" {
|
||||
why = said
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
|
||||
g.Failing = failing
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
case worst == healthNotYet:
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
if now.Sub(*g.Since) > gateBound {
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
|
||||
}
|
||||
default:
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second)), now)
|
||||
}
|
||||
}
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// decide sets a gate's verdict.
|
||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||
g.Took = now.Sub(*g.Since).Round(time.Second).String()
|
||||
}
|
||||
|
||||
// gatePassed keeps a passing build's verdict, so `plans` and the gate's probe can read it.
|
||||
func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule) {
|
||||
g := state.Gate
|
||||
err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
|
||||
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
|
||||
Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
|
||||
if err != nil && state.Build != "" {
|
||||
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
passCarried(ctx, open, p, g, module)
|
||||
fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module,
|
||||
strings.Join(g.Machines, ", "), g.Why)
|
||||
if module == catalogue.ControllerSeatName {
|
||||
carryUserList(ctx, open, p)
|
||||
}
|
||||
}
|
||||
|
||||
// carryUserList sends the machine holding the bus the user list a new controller composes, once that
|
||||
// controller passed its gate (ADR 0236). The controller's own grants travel in that list, and the old
|
||||
// controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new
|
||||
// controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR
|
||||
// 0221): then it is said, as a push would say it.
|
||||
func carryUserList(ctx context.Context, open *stores, p *inventory.Plan) {
|
||||
holder, behind, err := brokerBehind(ctx, open, nil)
|
||||
if err != nil || holder == "" || !behind {
|
||||
if err != nil {
|
||||
fmt.Printf("%s: whether the bus's user list is behind the new controller cannot be read: %v\n", p.ID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
held, err := heldMachines(ctx, open, []string{holder})
|
||||
if err != nil {
|
||||
fmt.Printf("%s: whether %s may be sent the new user list cannot be read: %v\n", p.ID, holder, err)
|
||||
return
|
||||
}
|
||||
if why, isHeld := held[holder]; isHeld {
|
||||
fmt.Printf("%s: the new controller's user list is not carried to %s, which holds the bus: %s — `push %s` "+
|
||||
"carries it\n", p.ID, holder, strings.Join(why, "; "), holder)
|
||||
return
|
||||
}
|
||||
if _, err := sendRollout(ctx, open, []string{holder}); err != nil {
|
||||
fmt.Printf("%s: the new controller's user list could not be carried to %s: %v\n", p.ID, holder, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%s: carried the new controller's user list to %s, which holds the bus\n", p.ID, holder)
|
||||
}
|
||||
|
||||
// gateFailed stops the plan at a build that failed its gate and puts the previous build back on the
|
||||
// machines it was judged on — once per build, said as a condition and an event. The plan is saved
|
||||
// before the send: a controller that is itself the build being put back does not outlive it.
|
||||
func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
|
||||
machines []string, why string) {
|
||||
inv := open.inventory
|
||||
now := time.Now().UTC()
|
||||
if state.Gate == nil {
|
||||
state.Gate = &inventory.PlanGate{Component: coreComponent(module), Machines: machines,
|
||||
From: state.Previous, To: state.Commit, Since: state.FirstAt}
|
||||
}
|
||||
g := state.Gate
|
||||
if g.Verdict == "" {
|
||||
if g.Since == nil {
|
||||
g.Since = &now
|
||||
}
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
}
|
||||
if len(g.Machines) == 0 {
|
||||
g.Machines = machines
|
||||
}
|
||||
state.Why = "failed its gate: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed its gate on %s in tier %d: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why)
|
||||
|
||||
verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous,
|
||||
Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why,
|
||||
Component: g.Component, JudgingFrom: g.Since}
|
||||
// **A send that changed nothing of the module there is no verdict on its build** (novox/hq issue
|
||||
// 280). The machine already ran this build, or one that made the same artifacts from the same
|
||||
// manifest: whatever the gate found wanting, this build did not bring it, and there is nothing to
|
||||
// put back. On 2026-10-06 such a module was marked failed, and the rollback looked for an earlier
|
||||
// build of the very commit it had failed — "no build kept" — while the build it had run before was
|
||||
// kept all along. Said, left as it is, never marked.
|
||||
if unchangedBy(ctx, inv, module, state.Previous, state.Commit) {
|
||||
g.Rollback = gateUnchanged
|
||||
state.Why = "stopped with its send; the send changed nothing of it: " + g.Why
|
||||
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was left as it was — %s already ran "+
|
||||
"%s %s, or a build identical to it, before the send, so nothing of it moved and nothing is put back",
|
||||
strings.Join(g.Machines, ", "), p.Tier, g.Why, module, strings.Join(g.Machines, ", "), module, short(state.Commit))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return
|
||||
}
|
||||
if state.Build == "" {
|
||||
// A plan from before builds were asked by id: nothing to mark, so nothing is put back by the
|
||||
// mesh — said, for a person.
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; not put back: the plan does not know which build it sent"
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
return
|
||||
}
|
||||
if err := inv.RecordGate(ctx, verdict); err != nil {
|
||||
if errors.Is(err, inventory.ErrGateKept) {
|
||||
// Already judged and acted on, by this controller before a restart or by another: never twice.
|
||||
if kept, found, _ := inv.GateOf(ctx, state.Build); found {
|
||||
g.Rollback = kept.Rollback
|
||||
}
|
||||
p.Note += "; its rollback was already made once and is not made again"
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; not put back: its verdict could not be kept, and a rollback that cannot be counted is not made — " + err.Error()
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
return
|
||||
}
|
||||
// The previous build: the one the first machine ran, from the build records.
|
||||
notBack := func(why string) {
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; NOT put back: " + why
|
||||
if err := inv.SetRollback(ctx, state.Build, inventory.NotRolledBack, g.Why+"; not put back: "+why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
sayRollback(ctx, open, module, g, why)
|
||||
}
|
||||
if state.Previous == "" {
|
||||
// A first build there: nothing ran before it, so nothing can be put back, and the machine is left
|
||||
// with it — said as that, not as a build the mesh lost.
|
||||
notBack(fmt.Sprintf("this is the first build of %s that %s was sent, or what it was sent before is not known: "+
|
||||
"there is no earlier build there to put back, so it is left with this one — `unassign` takes it off, "+
|
||||
"a newer merge replaces it", module, strings.Join(g.Machines, ", ")))
|
||||
return
|
||||
}
|
||||
failed, _, err := inv.BuildByID(ctx, state.Build)
|
||||
if err != nil {
|
||||
notBack("the failed build's record cannot be read: " + err.Error())
|
||||
return
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, module, state.Previous, failed)
|
||||
if err != nil {
|
||||
notBack("the build records cannot be read: " + err.Error())
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
notBack(fmt.Sprintf("no build of %s from %s, asked before the failed one, is among the %d newest kept to put "+
|
||||
"back", module, short(state.Previous), inventory.KeptBuilds))
|
||||
return
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, previous); err != nil {
|
||||
notBack(err.Error())
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.RollingBack
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err)
|
||||
}
|
||||
// Put back with the rest of its send, in one send per machine (issue 281).
|
||||
if b, batched := ctx.Value(rollbacksKey{}).(*rollbacks); batched {
|
||||
b.pending = append(b.pending, pendingRollback{module: module, state: state, g: g, previous: previous})
|
||||
b.modules[module] = true
|
||||
for _, n := range g.Machines {
|
||||
if !slices.Contains(b.machines, n) {
|
||||
b.machines = append(b.machines, n)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
|
||||
if err != nil {
|
||||
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.RolledBack
|
||||
p.Note += fmt.Sprintf("; put back to %s on %s", short(previous.Commit), strings.Join(sent, ", "))
|
||||
if err := inv.SetRollback(ctx, state.Build, inventory.RolledBack, g.Why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
}
|
||||
|
||||
// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one
|
||||
// send judges what it moved as one, and what it found wanting goes back in one send per machine — not
|
||||
// in a send for each module, which is the churn that failed the gate in the first place.
|
||||
type rollbacks struct {
|
||||
modules map[string]bool
|
||||
machines []string
|
||||
pending []pendingRollback
|
||||
}
|
||||
|
||||
type pendingRollback struct {
|
||||
module string
|
||||
state *inventory.PlanModule
|
||||
g *inventory.PlanGate
|
||||
previous inventory.Build
|
||||
}
|
||||
|
||||
type rollbacksKey struct{}
|
||||
|
||||
// batchingRollbacks is a context under which gateFailed registers what it puts back and leaves the send
|
||||
// to sendRollbacks.
|
||||
func batchingRollbacks(ctx context.Context) (context.Context, *rollbacks) {
|
||||
b := &rollbacks{modules: map[string]bool{}}
|
||||
return context.WithValue(ctx, rollbacksKey{}, b), b
|
||||
}
|
||||
|
||||
// sendRollbacks sends what a failed send put back, once to each machine, and says each module's rollback.
|
||||
func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *rollbacks) {
|
||||
if len(b.pending) == 0 {
|
||||
return
|
||||
}
|
||||
inv := open.inventory
|
||||
sort.Strings(b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
|
||||
var back []string
|
||||
for _, r := range b.pending {
|
||||
if err != nil {
|
||||
why := fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(r.previous.Commit), strings.Join(r.g.Machines, ", "), err, firstOf(r.g.Machines))
|
||||
r.g.Rollback = inventory.NotRolledBack
|
||||
if err := inv.SetRollback(ctx, r.state.Build, inventory.NotRolledBack, r.g.Why+"; not put back: "+why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
|
||||
}
|
||||
sayRollback(ctx, open, r.module, r.g, why)
|
||||
continue
|
||||
}
|
||||
r.g.Rollback = inventory.RolledBack
|
||||
back = append(back, r.module+" to "+short(r.previous.Commit))
|
||||
if err := inv.SetRollback(ctx, r.state.Build, inventory.RolledBack, r.g.Why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
|
||||
}
|
||||
sayRollback(ctx, open, r.module, r.g, "")
|
||||
}
|
||||
if err != nil {
|
||||
p.Note += fmt.Sprintf("; NOT put back: sending %s was refused: %v", strings.Join(b.machines, ", "), err)
|
||||
return
|
||||
}
|
||||
p.Note += fmt.Sprintf("; put back %s on %s, in one send", strings.Join(back, ", "), strings.Join(sent, ", "))
|
||||
}
|
||||
|
||||
// gateUnchanged is a failed gate's word on a module its send changed nothing of (novox/hq issue 281):
|
||||
// left as it was, never marked failed, and so no verdict on its build — `plans retry` asks it again.
|
||||
const gateUnchanged = "unchanged"
|
||||
|
||||
// unchangedBy says whether a send moving a module from one build to another changed nothing of it: the
|
||||
// same commit, builds made from the same source, or builds that made the same artifacts from the same
|
||||
// manifest. Not known is changed.
|
||||
func unchangedBy(ctx context.Context, inv *inventory.Inventory, module, from, to string) bool {
|
||||
if from == "" || to == "" {
|
||||
return false
|
||||
}
|
||||
if sameCommit(from, to) {
|
||||
return true
|
||||
}
|
||||
// Made from the same source (issue 280), or the same artifacts from the same manifest.
|
||||
f := moveFacts{}
|
||||
var err error
|
||||
if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
|
||||
return false
|
||||
}
|
||||
if f.fps, err = inv.Fingerprints(ctx); err != nil {
|
||||
return false
|
||||
}
|
||||
return f.identical(module, from, to)
|
||||
}
|
||||
|
||||
// rolledBackEvent is the body of `rolled-back` (ADR 0236): a contract, like a condition's events.
|
||||
type rolledBackEvent struct {
|
||||
Event string `json:"event"`
|
||||
At time.Time `json:"at"`
|
||||
Module string `json:"module"`
|
||||
Component string `json:"component,omitempty"`
|
||||
Machines []string `json:"machines"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to,omitempty"`
|
||||
Why string `json:"why"`
|
||||
// Rollback is rolled-back, or not-rolled-back with NotWhy.
|
||||
Rollback string `json:"rollback"`
|
||||
NotWhy string `json:"not_why,omitempty"`
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// sayRollback raises the gate's condition at once — the probe keeps it from then — and says the event.
|
||||
func sayRollback(ctx context.Context, open *stores, module string, g *inventory.PlanGate, notWhy string) {
|
||||
o := gateObservation(module, g.Component, g.Machines, g.Rollback, g.Why, notWhy, g.To, g.From)
|
||||
fmt.Println(o.Summary)
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
if d.keeper != nil {
|
||||
o.Source = gateProbe
|
||||
if _, err := d.keeper.Observe(ctx, o); err != nil {
|
||||
fmt.Printf("the gate's condition %s could not be kept: %v\n", o.Key(), err)
|
||||
}
|
||||
}
|
||||
if d.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(rolledBackEvent{Event: link.KeyRolledBack, At: time.Now().UTC(), Module: module,
|
||||
Component: g.Component, Machines: g.Machines, From: g.To, To: g.From, Why: g.Why, Rollback: g.Rollback,
|
||||
NotWhy: notWhy, Show: conditions.Condition{Key: o.Key()}.Show()})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyRolledBack, body); err != nil {
|
||||
fmt.Printf("%s's rollback could NOT be said on the bus: %v\n", module, err)
|
||||
}
|
||||
}
|
||||
|
||||
// gateObservation is a failed gate as a condition: `core.<component>.<machine>.rolled-back` for the
|
||||
// core (to-be 45 §2), `build.<module>.<machine>.rolled-back` for any other module; `rollback-failed`
|
||||
// when it could not be put back. Urgent for the core and for a build left in place; a warning for a
|
||||
// module put back, which runs what it ran before.
|
||||
func gateObservation(module, component string, machines []string, rollback, why, notWhy, failed, previous string) conditions.Observation {
|
||||
machine := strings.Join(machines, ",")
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: module + "." + machine, Kind: kindRolledBack,
|
||||
Token: kindRolledBack, Machine: firstOf(machines), Severity: conditions.Warning, Source: gateProbe,
|
||||
Summary: fmt.Sprintf("%s's build %s failed its gate on %s and was put back to %s: %s", module, short(failed),
|
||||
machine, short(previous), why)}
|
||||
if component != "" {
|
||||
o.Scope, o.ID, o.Severity = conditions.ScopeCore, component+"."+machine, conditions.Urgent
|
||||
}
|
||||
if len(machines) > 1 {
|
||||
o.Also = machines[1:]
|
||||
}
|
||||
if rollback != inventory.RolledBack && rollback != inventory.RollingBack {
|
||||
o.Kind, o.Token, o.Severity = kindRollbackFailed, kindRollbackFailed, conditions.Urgent
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
o.Summary = fmt.Sprintf("%s's build %s failed its gate on %s and was NOT put back: %s — %s", module, short(failed),
|
||||
machine, why, orNone(notWhy))
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// probeGates is DG: no build that failed its gate is left without its condition, and no witness's
|
||||
// rollback goes unsaid. Each module whose newest verdict is a failure keeps its condition; a newer build
|
||||
// that passes clears it. Each core component a machine's witness says it put back is `core.<component>.
|
||||
// <machine>.rolled-back`, urgent, until the machine's reports stop saying it.
|
||||
func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
latest, err := d.open.inventory.LatestGates(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, v := range latest {
|
||||
if v.Verdict != inventory.GateFailed {
|
||||
continue
|
||||
}
|
||||
notWhy := ""
|
||||
if v.Rollback == inventory.NotRolledBack {
|
||||
notWhy = v.Why
|
||||
}
|
||||
out = append(out, gateObservation(v.Module, v.Component, v.Machines, v.Rollback, v.Why, notWhy, v.Commit, v.Previous))
|
||||
}
|
||||
for node, list := range witnessed.all() {
|
||||
for _, r := range list {
|
||||
out = append(out, witnessObservation(node, r))
|
||||
}
|
||||
}
|
||||
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
||||
out = append(out, backlogObservation()...)
|
||||
return sortedFound(dedupeObservations(out)), nil
|
||||
}
|
||||
|
||||
// dedupeObservations keeps one observation per key, the first.
|
||||
func dedupeObservations(list []conditions.Observation) []conditions.Observation {
|
||||
seen := map[string]bool{}
|
||||
var out []conditions.Observation
|
||||
for _, o := range list {
|
||||
if seen[o.Key()] {
|
||||
continue
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules'
|
||||
// tools are served where. A variable so a test needs no runtime.
|
||||
var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) {
|
||||
inbox := conn.NewRespInbox()
|
||||
sub, err := conn.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||
}
|
||||
out := map[string]served{}
|
||||
add := func(node string) served {
|
||||
s, ok := out[node]
|
||||
if !ok {
|
||||
s = served{tools: map[string]bool{}}
|
||||
}
|
||||
return s
|
||||
}
|
||||
deadline := time.Now().Add(discoveryPatience)
|
||||
for time.Now().Before(deadline) {
|
||||
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
break
|
||||
}
|
||||
var info micro.Info
|
||||
if json.Unmarshal(msg.Data, &info) != nil {
|
||||
continue
|
||||
}
|
||||
if info.Name == broker.RuntimeModule {
|
||||
node := info.Metadata["node"]
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
s := add(node)
|
||||
s.runtime = true
|
||||
out[node] = s
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" {
|
||||
continue
|
||||
}
|
||||
node := e.Metadata["node"]
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
s := add(node)
|
||||
s.tools[e.Metadata["module"]] = true
|
||||
out[node] = s
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// gateLines is a plan's gates as `plans <id>` says them: the rollout's record.
|
||||
func gateLine(g *inventory.PlanGate) string {
|
||||
if g == nil {
|
||||
return ""
|
||||
}
|
||||
what := "judging"
|
||||
switch g.Verdict {
|
||||
case inventory.GatePassed:
|
||||
what = "passed"
|
||||
case inventory.GateFailed:
|
||||
what = "FAILED"
|
||||
}
|
||||
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
||||
if g.Component != "" {
|
||||
line += " (core: " + g.Component + ")"
|
||||
}
|
||||
if g.From != "" || g.To != "" {
|
||||
line += fmt.Sprintf(", %s → %s", short(orNone(g.From)), short(g.To))
|
||||
}
|
||||
if g.Took != "" {
|
||||
line += ", after " + g.Took
|
||||
}
|
||||
if g.Why != "" {
|
||||
line += ": " + g.Why
|
||||
} else if g.Last != "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
|
||||
} else if g.Verdict == "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||
}
|
||||
if g.Rollback != "" {
|
||||
line += "; " + g.Rollback
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// witnessObservation is a witness's verdict as a condition (ADR 0236, the host's contract):
|
||||
// `core.<component>.<node>.<outcome>`, urgent for rolled-back, not-reversible, restore-failed and
|
||||
// halted, a warning for nothing-to-restore and unwitnessed.
|
||||
func witnessObservation(node string, r lease.Rollback) conditions.Observation {
|
||||
severity := conditions.Warning
|
||||
if lease.Urgent(r.Outcome) {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
outcome := r.Outcome
|
||||
if outcome == "" {
|
||||
outcome = lease.OutcomeRolledBack
|
||||
}
|
||||
what := map[string]string{
|
||||
lease.OutcomeRolledBack: "and put back " + short(orNone(r.To)),
|
||||
lease.OutcomeNotReversible: "and left it: it is not reversible",
|
||||
lease.OutcomeNothingToRestore: "and had nothing to put back",
|
||||
lease.OutcomeRestoreFailed: "and could not put the previous build back",
|
||||
lease.OutcomeUnwitnessed: "and could not judge it at all",
|
||||
lease.OutcomeHalted: "and gave up: the build before it fails too",
|
||||
}[outcome]
|
||||
return conditions.Observation{Scope: conditions.ScopeCore, ID: r.Component + "." + node, Kind: kindRolledBack,
|
||||
Token: outcome, Machine: node, Severity: severity,
|
||||
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
||||
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
||||
}
|
||||
@@ -0,0 +1,551 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).
|
||||
|
||||
// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
|
||||
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
|
||||
// sent and reports it — with the gate's bounds shortened to judge in three steps.
|
||||
type gateMesh struct {
|
||||
open *stores
|
||||
sent [][]string
|
||||
health map[string]health // per machine, what a judging finds; healthy when unsaid
|
||||
keeper *conditions.Keeper
|
||||
told *conditions.Told
|
||||
}
|
||||
|
||||
func aGateMesh(t *testing.T) *gateMesh {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register := func(commit string, asked time.Time) {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
||||
Head: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c1", time.Now().Add(-2*time.Hour))
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// Every send: recorded with the build the module is at, applied and reported by the machine.
|
||||
n := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
g.sent = append(g.sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
// What a judging reads: the store's reports, and a health the test says per machine.
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if g.health[r.Node] == healthBroken {
|
||||
r.Outcome = inventory.OutcomeFailed
|
||||
}
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
for node, h := range g.health {
|
||||
if h == healthNotYet {
|
||||
f.rolledBack[node] = nil
|
||||
r := f.reports[node]
|
||||
r.Current = false
|
||||
f.reports[node] = r
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||
gateSettle, gateEvery = 0, time.Hour
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// A module's build that fails its gate on the first machine is put back there — the previous build
|
||||
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
||||
// condition and an event, never registered or rolled back again.
|
||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v, not the first machine alone", g.sent)
|
||||
}
|
||||
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
|
||||
}
|
||||
|
||||
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
|
||||
gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
|
||||
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
|
||||
t.Fatalf("anchor was last sent %v, not the previous build", sent)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
|
||||
t.Fatalf("laptop was sent the failed build: %v", sent)
|
||||
}
|
||||
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
|
||||
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
|
||||
}
|
||||
|
||||
// Said: a condition for the operator, and the event.
|
||||
open, err := g.keeper.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var key string
|
||||
for _, c := range open {
|
||||
if c.Kind == kindRolledBack {
|
||||
key = c.Key
|
||||
}
|
||||
}
|
||||
if key != "build.app.anchor.rolled-back" {
|
||||
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
|
||||
}
|
||||
saidIt := false
|
||||
for _, e := range g.told.Said() {
|
||||
saidIt = saidIt || e.Event == link.KeyRolledBack
|
||||
}
|
||||
if !saidIt {
|
||||
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
|
||||
}
|
||||
|
||||
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
|
||||
// heard again is not registered.
|
||||
advancePlans(ctx, g.open)
|
||||
state := p.Modules["app"]
|
||||
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
|
||||
if len(g.sent) != 2 {
|
||||
t.Fatalf("sent again after the rollback: %v", g.sent)
|
||||
}
|
||||
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"})})
|
||||
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("the failed build was registered again: %v", err)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module moved to %s", current["app"].Commit)
|
||||
}
|
||||
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
|
||||
}
|
||||
|
||||
// The probe keeps the condition while the newest verdict is the failure.
|
||||
obs, err := probeGates(ctx, doctorFrom)
|
||||
if err != nil || len(obs) != 1 || obs[0].Key() != key {
|
||||
t.Fatalf("the gate's probe found %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
|
||||
// the rest are sent, the verdict kept, and the plan done.
|
||||
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v", g.sent)
|
||||
}
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, g.open)
|
||||
g.health["anchor"] = healthNotYet
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
|
||||
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
|
||||
}
|
||||
gateBound = -time.Second
|
||||
advancePlans(ctx, g.open)
|
||||
p := g.plan(t)
|
||||
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
|
||||
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
|
||||
}
|
||||
}
|
||||
|
||||
// The health a judging finds, per core component and for a module.
|
||||
func TestTheHealthDefinitions(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
applied := func(node string) gateFacts {
|
||||
at := now
|
||||
return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
||||
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
|
||||
}
|
||||
m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}
|
||||
|
||||
f := applied("anchor")
|
||||
f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
|
||||
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
|
||||
t.Errorf("a module whose tools are not served is %v (%s)", h, why)
|
||||
}
|
||||
f.served["anchor"].tools["app"] = true
|
||||
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("a module applied with its tools served is %v (%s)", h, why)
|
||||
}
|
||||
// A condition raised about it on that machine since it was sent: not yet healthy.
|
||||
f.judged = true
|
||||
f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
|
||||
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("a module with a new condition about it is %v", h)
|
||||
}
|
||||
f.open[0].Raised = since.Add(-time.Hour)
|
||||
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("a condition older than the send counted against it: %v", h)
|
||||
}
|
||||
// A witness that put it back: broken.
|
||||
f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
|
||||
Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
||||
t.Errorf("a component a witness put back is %v", h)
|
||||
}
|
||||
// The node tools answer, or not.
|
||||
f = applied("anchor")
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("node tools not answering are %v", h)
|
||||
}
|
||||
f.served["anchor"] = served{runtime: true}
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("node tools answering are %v", h)
|
||||
}
|
||||
// The controller: the lease held since the send, by a controller that says it is ready.
|
||||
f = applied("control")
|
||||
f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
|
||||
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
|
||||
t.Errorf("a lease held by a controller older than the build is %v", h)
|
||||
}
|
||||
f.holder.Taken = now
|
||||
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
|
||||
t.Errorf("a new controller holding the lease and ready is %v", h)
|
||||
}
|
||||
f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
|
||||
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
|
||||
!strings.Contains(why, "first run") {
|
||||
t.Errorf("a controller not ready is %v (%s)", h, why)
|
||||
}
|
||||
// A machine that refused what it was sent: broken.
|
||||
f = applied("anchor")
|
||||
r := f.reports["anchor"]
|
||||
r.Outcome = inventory.OutcomeRefused
|
||||
f.reports["anchor"] = r
|
||||
if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
||||
t.Errorf("a refusal is %v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
|
||||
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
|
||||
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
|
||||
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
|
||||
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := inv.UpgradeOf(ctx, "nats")
|
||||
if err != nil || u.RollOut || u.From != catalogue.FromBus {
|
||||
t.Fatalf("the bus's policy is %+v %v", u, err)
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
|
||||
t.Fatalf("a person rolled the bus out: %v", err)
|
||||
}
|
||||
var sent [][]string
|
||||
was := sendRollout
|
||||
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
now := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
|
||||
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
|
||||
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
|
||||
}
|
||||
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
|
||||
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
|
||||
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
|
||||
}
|
||||
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
|
||||
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
|
||||
t.Fatalf("a send to the bus's machine was not refused: %v", err)
|
||||
}
|
||||
// A rebuild that made the same artifacts is no move.
|
||||
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
|
||||
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
|
||||
b.Asked, b.At = time.Now(), time.Now()
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
|
||||
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
|
||||
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: "registry/nats@sha256:new"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
|
||||
// first by the bus machine's backup holder.
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
|
||||
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
|
||||
}
|
||||
wasSnapshot := takeBusSnapshot
|
||||
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
|
||||
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
|
||||
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
|
||||
}
|
||||
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
|
||||
return node + "'s restore point of " + module, nil
|
||||
}
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("the step sent %v, not the bus's machine", sent)
|
||||
}
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
|
||||
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
|
||||
t.Fatalf("the step is %+v %v %v", s, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
|
||||
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
|
||||
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
for _, name := range []string{"gone", "kept"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
|
||||
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
|
||||
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a deleted module was asked to build: %v", *asked)
|
||||
}
|
||||
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
|
||||
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := shelf["gone"]; still {
|
||||
t.Fatal("a deleted module nothing holds was not forgotten")
|
||||
}
|
||||
|
||||
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
|
||||
asked0 := time.Now().UTC().Add(-time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
|
||||
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
|
||||
"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
|
||||
p, _ := inv.PlanByID(ctx, "plan-deleted")
|
||||
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
|
||||
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func nodeID(t *testing.T, open *stores, name string) string {
|
||||
t.Helper()
|
||||
n, err := open.inventory.NodeByName(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n.ID
|
||||
}
|
||||
|
||||
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
|
||||
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
|
||||
// gone with the first report that no longer carries it.
|
||||
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
d := &doctor{open: open}
|
||||
at := time.Now().UTC()
|
||||
witnessed.heard("control", []lease.Rollback{
|
||||
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
|
||||
Why: "the new controller did not take the lease within 60s", At: at},
|
||||
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
|
||||
}, at)
|
||||
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
|
||||
obs, err := probeGates(t.Context(), d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]conditions.Severity{}
|
||||
for _, o := range obs {
|
||||
got[o.Key()] = o.Severity
|
||||
}
|
||||
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
|
||||
"core.node-tools.control.unwitnessed": conditions.Warning}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("the witness's verdicts are %v", got)
|
||||
}
|
||||
witnessed.heard("control", nil, time.Now())
|
||||
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
|
||||
//
|
||||
// The serving controller hears every report; a clean one about the declaration a machine was last
|
||||
// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is
|
||||
// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
|
||||
// so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log
|
||||
// is read as the count of repairs a healer is wanted for.
|
||||
|
||||
// SecretReplaced is the controller seat's fact that a value given by hand was replaced
|
||||
// (link.KeySecretReplaced). Never the value: neither the old one, which the mesh cannot read,
|
||||
// nor the new one, sealed to the machine as it was made.
|
||||
type SecretReplaced struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
Given time.Time `json:"given"`
|
||||
// Sent are the machines sent so the module starts again on the new value; Unsent says why
|
||||
// they could not be, in which case the next push carries it.
|
||||
Sent []string `json:"sent"`
|
||||
Unsent string `json:"unsent,omitempty"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// givenEvents is where the serving controller states it; nil in a command.
|
||||
var givenEvents link.Bus
|
||||
|
||||
// replacing keeps one replacement per machine at a time: two reports arriving together find the
|
||||
// same rows, and the store's claim makes one of them the replacer, but the sends need not race.
|
||||
var replacing sync.Map
|
||||
|
||||
// startedWell says a report is a machine's clean account of a declaration: everything applied,
|
||||
// nothing failed or refused. Whether it is the declaration last sent is the store's to answer.
|
||||
func startedWell(report link.Report) bool {
|
||||
return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil
|
||||
}
|
||||
|
||||
const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
|
||||
|
||||
// replaceGiven replaces what the report makes due, sends the machines, and says so.
|
||||
func replaceGiven(ctx context.Context, open *stores, report link.Report) {
|
||||
if _, busy := replacing.LoadOrStore(report.Node, true); busy {
|
||||
return
|
||||
}
|
||||
defer replacing.Delete(report.Node)
|
||||
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared)
|
||||
if err != nil {
|
||||
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err)
|
||||
}
|
||||
for _, r := range replaced {
|
||||
said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
|
||||
Sent: r.Machines, Why: givenWhy}
|
||||
log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; sending %s",
|
||||
r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
|
||||
if err := sendTo(ctx, open, r.Machines); err != nil {
|
||||
said.Sent, said.Unsent = nil, err.Error()
|
||||
log.Printf("the new %q of %s is sealed and not yet delivered to %s — the next push carries it: %v",
|
||||
r.Name, r.Module, strings.Join(r.Machines, ", "), err)
|
||||
}
|
||||
stateReplaced(ctx, said)
|
||||
}
|
||||
}
|
||||
|
||||
func stateReplaced(ctx context.Context, said SecretReplaced) {
|
||||
if givenEvents == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(said)
|
||||
if err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
return
|
||||
}
|
||||
stating, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := givenEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySecretReplaced, body); err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A rotation asked through the seat carries why to the command, which records it in the hand-act
|
||||
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
|
||||
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
|
||||
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password --why leaked into logs --cause leaked" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
|
||||
t.Fatalf("without why: %v %v", argv, err)
|
||||
}
|
||||
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
|
||||
t.Fatalf("why beside a provision was passed over: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that
|
||||
// the machine is there is not (novox/hq ADR 0228).
|
||||
func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
report link.Report
|
||||
good bool
|
||||
}{
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true},
|
||||
{link.Report{Node: "anchor"}, false},
|
||||
{link.Report{Node: "anchor", Applied: []string{"x"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false},
|
||||
} {
|
||||
if got := startedWell(c.report); got != c.good {
|
||||
t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 274: a provider is granted exactly the consumers whose own resolution binds them to
|
||||
// it — not every consumer a pair credential from it was ever made for.
|
||||
|
||||
// grantOf is the grant of one provision to one consuming module, and whether there is one at all.
|
||||
func grantOf(grants []catalogue.Grant, provision, consumer, module string) (catalogue.Grant, bool) {
|
||||
for _, g := range grants {
|
||||
if g.Provision == provision && g.Consumer == consumer && (g.From == module || g.From == "") {
|
||||
return g, true
|
||||
}
|
||||
}
|
||||
return catalogue.Grant{}, false
|
||||
}
|
||||
|
||||
// The morning after issue 273, through the stores: a consumer was bound to the store on another
|
||||
// machine, a credential from there was made, and a person pinned it back to the store beside it. Both
|
||||
// credentials are on record. The store it left is no longer granted it — so it retires it and keeps
|
||||
// its data (ADR 0230) — and says so; the store it is bound to keeps its grant; and the credential from
|
||||
// the store it left stays on record.
|
||||
func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// Pinned back beside its data, as the operator did.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" {
|
||||
t.Fatalf("pinned back to the laptop and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(holders) != 2 {
|
||||
t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders)
|
||||
}
|
||||
|
||||
// The store it left: no longer granted, and said.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" {
|
||||
t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g)
|
||||
}
|
||||
if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" ||
|
||||
strings.Join(unbound[0].BoundTo, ",") != "laptop" {
|
||||
t.Fatalf("the consumer that moved is not the one said: %+v", unbound)
|
||||
}
|
||||
planned, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", planned, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := declared.Received["store"]["postgres-database"]; len(got) != 0 {
|
||||
t.Fatalf("the anchor's store is still told about %+v", got)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") ||
|
||||
!strings.Contains(said, "cleanup delete") {
|
||||
t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said)
|
||||
}
|
||||
|
||||
// The store it is bound to: granted.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// And the credential from the store it left is kept: the key to the login and data held there.
|
||||
if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 {
|
||||
t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer whose resolution cannot be read is an error, never a consumer bound nowhere — withdrawing
|
||||
// a grant on that reading would take its access away (issue 152).
|
||||
func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and
|
||||
// that is not its set failing to compose.
|
||||
laptop, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) {
|
||||
t.Fatalf("the seam this test relies on moved: %v", err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err == nil {
|
||||
t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") {
|
||||
t.Fatalf("the error does not say whose resolution could not be read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The rule itself, on a resolution: bound is the provider a need for exactly that credential is
|
||||
// answered by, never one answered by a record, and a different local name is a different credential.
|
||||
func TestBindsFromIsTheProviderOfThatCredential(t *testing.T) {
|
||||
r := catalogue.Resolution{Needs: []catalogue.Needed{
|
||||
{Name: "postgres-database", For: "board", From: "laptop"},
|
||||
{Name: "postgres-database", For: "board", From: "laptop", Local: "reports"},
|
||||
{Name: "postgres-database", For: "wiki", From: "anchor"},
|
||||
{Name: "a-licence", For: "board", From: "the-licence", ByRecord: true},
|
||||
}}
|
||||
s := inventory.Secret{Name: "postgres-database", ConsumerModule: "board"}
|
||||
if got := r.BindsFrom(s.Name, s.ConsumerModule, s.Local); strings.Join(got, ",") != "laptop" {
|
||||
t.Fatalf("board's credential is bound to %v", got)
|
||||
}
|
||||
if got := r.BindsFrom("postgres-database", "board", "archive"); len(got) != 0 {
|
||||
t.Fatalf("a local name nothing asks for is bound to %v", got)
|
||||
}
|
||||
if got := r.BindsFrom("a-licence", "board", ""); len(got) != 0 {
|
||||
t.Fatalf("a need answered by a record is bound to %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
//
|
||||
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||
// (Phase 1).
|
||||
|
||||
// handActVerb is one verb that writes the hand-act log, and whether what it records is a repair.
|
||||
type handActVerb struct {
|
||||
Verb string
|
||||
// Decision says why an act of this verb is a person's decision by design rather than a repair a
|
||||
// healer could take over; empty for a repair.
|
||||
Decision string
|
||||
// DecidedFor limits Decision to these causes; empty, it holds for every act of the verb.
|
||||
DecidedFor []string
|
||||
}
|
||||
|
||||
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
|
||||
const causeLeakedInLogs = "leaked-in-logs"
|
||||
|
||||
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
|
||||
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
|
||||
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
|
||||
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
|
||||
var handActVerbs = []handActVerb{
|
||||
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
|
||||
// default (ADR 0236) exists to end.
|
||||
{Verb: "push"},
|
||||
{Verb: "plans stop"},
|
||||
{Verb: "plans close"},
|
||||
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
|
||||
// not trusted with it — either is a repair the owner should have made.
|
||||
{Verb: "plans go"},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending.
|
||||
{Verb: "conditions silence"},
|
||||
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts.
|
||||
{Verb: "hand-act record"},
|
||||
// A person's decisions by design.
|
||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
||||
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
|
||||
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
|
||||
"person starts (ADR 0236)"},
|
||||
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
|
||||
"person releases it (ADR 0236)"},
|
||||
// A leak is judged by a person — which value was exposed, to whom — and its rotation is the answer
|
||||
// to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the
|
||||
// module that prints them, an issue against it, not a healer that rotates. A rotation for any other
|
||||
// cause — a credential that stopped working — counts: a schedule or a healer could take it over.
|
||||
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||
DecidedFor: []string{causeLeakedInLogs}},
|
||||
}
|
||||
|
||||
// personsDecision is whether an act in the log is a person's decision by design, by the verb that
|
||||
// recorded it (handActVerbs).
|
||||
func personsDecision(a link.HandAct) bool {
|
||||
for _, v := range handActVerbs {
|
||||
if v.Verb != a.Verb {
|
||||
continue
|
||||
}
|
||||
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// handActFlags are the flags every repairing verb takes.
|
||||
type handActFlags struct {
|
||||
why, cause, condition *string
|
||||
}
|
||||
|
||||
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||
return handActFlags{
|
||||
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||
}
|
||||
}
|
||||
|
||||
// given is whether a reason was given.
|
||||
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||
|
||||
// require refuses an act without a reason, before anything is done.
|
||||
func (f handActFlags) require(verb string) error {
|
||||
if f.given() {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||
}
|
||||
|
||||
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||
// command dials its own.
|
||||
var handActConn *nats.Conn
|
||||
|
||||
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||
func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
|
||||
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
if !f.given() {
|
||||
return
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
act = written
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||
}
|
||||
|
||||
// handActCommand is `hand-act record` and `hand-acts`.
|
||||
func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "record" {
|
||||
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||
if what == "" {
|
||||
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||
}
|
||||
if err := f.require("hand-act record"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||
"act for the same reason will use — it is how a repair done twice is found")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||
}
|
||||
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||
written.Why, written.Cause)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||
}
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
now := time.Now()
|
||||
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeated := link.RepeatedCauses(repairs(acts), now)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Println(")")
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
causes := make([]string, 0, len(repeated))
|
||||
for c, n := range repeated {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// repairs are the acts that are not a person's decision by design: what S15 counts.
|
||||
func repairs(acts []link.HandAct) []link.HandAct {
|
||||
out := make([]link.HandAct, 0, len(acts))
|
||||
for _, a := range acts {
|
||||
if !personsDecision(a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||
// the log could not be read, which status says rather than reading as none.
|
||||
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||
n := -1
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||
n = len(acts)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return -1, err.Error()
|
||||
}
|
||||
return n, ""
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||
// the verb.
|
||||
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor"}},
|
||||
{"push", map[string]any{}},
|
||||
{"plans", map[string]any{"close": "plan-1"}},
|
||||
{"plans", map[string]any{"stop": "plan-1"}},
|
||||
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||
{"command", map[string]any{"command": "push anchor"}},
|
||||
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if c.verb == "plans" && err == nil {
|
||||
// The seat composes the command line; the command refuses it, before opening anything.
|
||||
err = plansCommand(context.Background(), argv[1:])
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("consumer-reset without why: %v", err)
|
||||
}
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("hand-act record without why: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--cause") {
|
||||
t.Errorf("hand-act record without a cause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||
"plans close plan-1 --why the report will not come"},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
var ds []inventory.Duration
|
||||
for i := 1; i <= 10; i++ {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||
got := summarise(ds)
|
||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,876 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The healers (novox/hq to-be 45 §7, ADR 0227 rule 7, Phase 3).
|
||||
//
|
||||
// **A known failure heals itself, under a brake, and every repair is said.** Research 031 counted the
|
||||
// repairs people made by hand in six days: a push to unstick a plan waiting on a report (four times),
|
||||
// a controller restarted to make an object again (twice), a plan closed (twice), a consumer re-made from
|
||||
// now (once). Each was the ordinary path, taken again by a person who noticed. A healer is that act,
|
||||
// registered against the one condition kind it answers, so the mesh takes it itself:
|
||||
//
|
||||
// - **its repair is the ordinary path again** — the send a push makes, the plan's own close, the
|
||||
// assertion every send makes, the consumer reset the verb makes — never a withdrawal, a deletion of
|
||||
// data or a recreation of it;
|
||||
// - **its budget** is how many acts it may take against one thing in a window, and **its settle** how
|
||||
// long after an act it leaves the observation to say whether it worked;
|
||||
// - **success is never the healer's to say.** The condition clears when the watchdog or the probe that
|
||||
// raised it no longer sees it. A healer marking its own work done would be the second opinion of a
|
||||
// fact that rule 1 forbids;
|
||||
// - **a spent budget stops it**: the condition is the operator's, urgent, with every attempt in
|
||||
// `tried`, and no healer touches it again until observation clears it;
|
||||
// - **every act is said**: begun in the store before it is made (so a controller dying mid-act has
|
||||
// still spent it), kept in the condition's `tried` as `healer Hn`, and said on the bus as the
|
||||
// controller seat's `healer-acted`. A heal is not a hand act and is never in the hand-act log —
|
||||
// which is how S15 can tell a repair the mesh made from one a person had to.
|
||||
//
|
||||
// **And the mesh-wide brake**: more than healBrakeLimit acts in an hour, all healers together, and every
|
||||
// healer stops — an urgent condition says so — until an hour has passed with none. A healer looping is
|
||||
// then at most a dozen acts, said, and never the incident itself.
|
||||
//
|
||||
// Only the controller holding the lease heals, under its epoch: a controller serving without the lease
|
||||
// (S12) heals nothing, because a repair is the one act that can always wait for the lease.
|
||||
|
||||
// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour.
|
||||
const (
|
||||
healBrakeLimit = 12
|
||||
healBrakeWindow = time.Hour
|
||||
)
|
||||
|
||||
// healEvery is how often the healers look at what is open.
|
||||
var healEvery = 30 * time.Second
|
||||
|
||||
// What raises the healers' own conditions, and their kinds.
|
||||
const (
|
||||
sourceHealers = "healers"
|
||||
kindHealersBraked = "healers-braked"
|
||||
kindConsumerBehind = "consumer-behind"
|
||||
kindHealersBlind = "probe-failed"
|
||||
)
|
||||
|
||||
// healerRow is one row of the registry.
|
||||
type healerRow struct {
|
||||
ID string
|
||||
// Kinds are the condition kinds it answers: from the signals table, the probe registry, or an event.
|
||||
Kinds []string
|
||||
// Condition, Repair, Then, From are the row in words, as to-be 45 §7 and `healers` say it.
|
||||
Condition string
|
||||
Repair string
|
||||
Then string
|
||||
From string
|
||||
// Budget acts against one budget key within Window; Settle after an act before the next, or the
|
||||
// escalation, so the observation has had its turn to say whether it worked.
|
||||
Budget int
|
||||
Window time.Duration
|
||||
Settle time.Duration
|
||||
// ActsIn is where the repair runs: the controller, or a module that repairs its own (H5).
|
||||
ActsIn string
|
||||
// Event is what each act is said as.
|
||||
Event string
|
||||
// applies says whether this condition is one the healer may act on, and what its budget is
|
||||
// counted against; why when it is not. Reads, never acts. Nil where the repair is a module's.
|
||||
applies func(ctx context.Context, h *healing, c conditions.Condition) (budget string, ok bool, why string, err error)
|
||||
// repair is the act: what it did in words, what came of it, or an error when it could not be done.
|
||||
repair func(ctx context.Context, h *healing, c conditions.Condition) (act, said string, err error)
|
||||
}
|
||||
|
||||
// actsInController is a healer whose repair the controller makes.
|
||||
const actsInController = "controller"
|
||||
|
||||
// healerRegistry is to-be 45 §7's table, compiled in. **The registry is the design's live form**: a
|
||||
// test generated from it holds every row to a condition kind the mesh raises, a budget, a brake and its
|
||||
// event (healers_test.go).
|
||||
var healerRegistry = []healerRow{
|
||||
{ID: "H1", Kinds: []string{"sent-not-reported"},
|
||||
Condition: "a machine was sent a declaration and has not reported it (S2)",
|
||||
Repair: "ask the machine's node-engine to say again what it last applied (the `report` verb); if what " +
|
||||
"comes back is not the declaration it was sent, or nothing comes, send it its current declaration " +
|
||||
"again — what a push of that one machine does, never moving a build a policy or a plan holds back",
|
||||
Then: "resolver operator, urgent", From: "a push by hand to unstick a plan waiting on a report (230, 257, 264, 267)",
|
||||
Budget: 2, Window: 6 * time.Hour, Settle: 3 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAMachine, repair: repairReport},
|
||||
{ID: "H2", Kinds: []string{"stalled"},
|
||||
Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " +
|
||||
"exists — or already finished — every module of every tier built or failed, and every one that " +
|
||||
"rolls out sent (S3); or a delivery held past its state's bound where mesh-delivery's table lets H2 " +
|
||||
"take a transition (D14, novox/hq ADR 0239)",
|
||||
Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " +
|
||||
"done; what it asked still builds and registers — or, for a delivery, mesh-delivery's `close`, which " +
|
||||
"reads its walk again and takes only the transition its table names",
|
||||
Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)",
|
||||
Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAStalePlan, repair: repairPlan},
|
||||
{ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"},
|
||||
Condition: "a seat's holder that does not answer (D3), or a durable consumer the mesh expects and the " +
|
||||
"bus does not hold (D6, S9)",
|
||||
Repair: "assert the bus's streams, consumers and seat workers again — the assertion every send makes " +
|
||||
"(issue 208)",
|
||||
Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)",
|
||||
Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAnObject, repair: repairObjects},
|
||||
{ID: "H4", Kinds: []string{kindConsumerBehind},
|
||||
Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable",
|
||||
Repair: "re-make the consumer to deliver from now — `broker consumer-reset` (issue 248); what it drops " +
|
||||
"is caught up where the table says",
|
||||
Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)",
|
||||
Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAResettableConsumer, repair: repairConsumer},
|
||||
{ID: "H5", Kinds: []string{kindProviderFailing},
|
||||
Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing, " +
|
||||
"credentials-rejected)",
|
||||
Repair: "the provider repairs it itself through the server's own bootstrap command, checks again and " +
|
||||
"says what it did (ADR 0224 §5); the controller keeps its word as the condition",
|
||||
Then: "announced failing by the provider, braked from ten minutes doubling to six hours (ADR 0224 §5)",
|
||||
From: "the identity provider's admin reset through its bootstrap command (179)",
|
||||
// Its budget and brake are the module's own: ten minutes, doubling, to six hours.
|
||||
Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute,
|
||||
ActsIn: "the provider module (keycloak), ADR 0224 §5", Event: "provisioner.failing, provisioner.recovered"},
|
||||
}
|
||||
|
||||
// healerFor is the healer registered for a kind, and false when none acts in the controller.
|
||||
func healerFor(kind string) (healerRow, bool) {
|
||||
for _, r := range healerRegistry {
|
||||
if r.repair != nil && slices.Contains(r.Kinds, kind) {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return healerRow{}, false
|
||||
}
|
||||
|
||||
// healerNamedFor is any healer registered for a kind, wherever it acts: what S15 says beside a cause.
|
||||
func healerNamedFor(kind string) string {
|
||||
for _, r := range healerRegistry {
|
||||
if slices.Contains(r.Kinds, kind) {
|
||||
return r.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// healing is the healers' runner and what their repairs reach.
|
||||
type healing struct {
|
||||
open *stores
|
||||
keeper *conditions.Keeper
|
||||
teller conditions.Teller
|
||||
// js is the bus, for the repairs that assert or reset its objects; nil where there is none.
|
||||
js *broker.JetStream
|
||||
// epoch is the lease's gate; acting says this controller is the one acting, not one standing by.
|
||||
epoch func(ctx context.Context) (uint64, error)
|
||||
acting func() bool
|
||||
now func() time.Time
|
||||
say func(format string, args ...any)
|
||||
|
||||
// The acts, as seams a test replaces: asking a machine to report, sending it again, asserting the
|
||||
// bus's objects, resetting a consumer.
|
||||
askReport func(ctx context.Context, node string) error
|
||||
sendAgain func(ctx context.Context, node string) error
|
||||
assertObjects func(ctx context.Context) error
|
||||
resetConsumer func(stream, name string) (string, error)
|
||||
// reportWait is how long H1 waits for the machine's report after asking.
|
||||
reportWait time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
// declined is why each condition was last passed over, so it is said once and `healers` can show it.
|
||||
declined map[string]string
|
||||
paused string
|
||||
}
|
||||
|
||||
// newHealing is the serving controller's runner, its acts the real ones.
|
||||
func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Teller, js *broker.JetStream) *healing {
|
||||
h := &healing{open: open, keeper: keeper, teller: teller, js: js, acting: link.Holding,
|
||||
epoch: func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) },
|
||||
now: time.Now, say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
reportWait: 45 * time.Second, declined: map[string]string{}}
|
||||
h.askReport = func(ctx context.Context, node string) error {
|
||||
if h.js == nil {
|
||||
return errors.New("this controller is not on the bus")
|
||||
}
|
||||
return askToReport(ctx, h.js.Conn(), node)
|
||||
}
|
||||
h.sendAgain = func(ctx context.Context, node string) error {
|
||||
// **Never what a policy or a plan holds back** (ADR 0221): a send by the mesh itself is a push
|
||||
// that did not name the machine — a person's word sends a held build, a healer's does not.
|
||||
held, err := heldMachines(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if why := held[node]; len(why) > 0 {
|
||||
return fmt.Errorf("not sent again: it would move what a policy or a plan holds back (ADR 0221) — %s; "+
|
||||
"`push %s` sends it, on a person's word", strings.Join(why, "; "), node)
|
||||
}
|
||||
return sendTo(ctx, open, []string{node})
|
||||
}
|
||||
h.assertObjects = func(ctx context.Context) error {
|
||||
if h.js == nil {
|
||||
return errors.New("this controller is not on the bus")
|
||||
}
|
||||
return assertOnSend(ctx, open.inventory, h.js, " ")
|
||||
}
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
if h.js == nil {
|
||||
return "", errors.New("this controller is not on the bus")
|
||||
}
|
||||
before, after, err := h.js.ResetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("it was %d behind with %d unacknowledged; it delivers from now, %d pending", before.Pending,
|
||||
before.AckPending, after.Pending), nil
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// askToReport asks one machine's node-engine to say again what it last applied (to-be 45 §6). On core
|
||||
// NATS, fired and flushed: the answer is the machine's ordinary report, read from the store.
|
||||
func askToReport(ctx context.Context, conn *nats.Conn, node string) error {
|
||||
body, err := json.Marshal(map[string]any{"asked": time.Now().UTC(), "by": "the controller's healer H1"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := conn.Publish(broker.AskReportSubject(node), body); err != nil {
|
||||
return err
|
||||
}
|
||||
flushing, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return conn.FlushWithContext(flushing)
|
||||
}
|
||||
|
||||
// keep runs the healers until ctx ends.
|
||||
func (h *healing) keep(ctx context.Context) {
|
||||
tick := time.NewTicker(healEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
h.tick(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick is one look at what is open, and every act it calls for.
|
||||
func (h *healing) tick(ctx context.Context) {
|
||||
if h.acting != nil && !h.acting() {
|
||||
return
|
||||
}
|
||||
epoch, err := h.epoch(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
h.pause(fmt.Sprintf("this controller may not act: %v", err))
|
||||
return
|
||||
case epoch == 0:
|
||||
h.pause("this controller serves without the lease (S12): a repair waits for it")
|
||||
return
|
||||
}
|
||||
inv := h.open.inventory
|
||||
now := h.now()
|
||||
heals, err := inv.HealsSince(ctx, now.Add(-24*time.Hour))
|
||||
if err != nil {
|
||||
// Blind: nothing is done, and that is said — never read as "no heals, budgets whole".
|
||||
h.reconcile(ctx, []conditions.Observation{{Scope: conditions.ScopeProbe, ID: "healers", Token: "failed",
|
||||
Kind: kindHealersBlind, Severity: conditions.Warning,
|
||||
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
|
||||
Said: firstLine(err.Error())}})
|
||||
return
|
||||
}
|
||||
open, err := h.keeper.Open(ctx)
|
||||
if err != nil {
|
||||
h.say("the healers cannot read the open conditions, and act on nothing: %v", err)
|
||||
return
|
||||
}
|
||||
acts := actsWithin(heals, now.Add(-healBrakeWindow))
|
||||
if braked, said := brakeHolds(acts, open, now); braked {
|
||||
h.reconcile(ctx, []conditions.Observation{brakeObservation(acts, said)})
|
||||
h.pause("the mesh-wide brake holds: " + said)
|
||||
return
|
||||
}
|
||||
h.reconcile(ctx, nil)
|
||||
h.resume()
|
||||
for _, c := range open {
|
||||
row, ok := healerFor(c.Kind)
|
||||
if !ok || c.Escalated() {
|
||||
continue
|
||||
}
|
||||
budget, applies, why, err := row.applies(ctx, h, c)
|
||||
if err != nil {
|
||||
h.decline(c.Key, row.ID, "could not tell whether it applies: "+err.Error())
|
||||
continue
|
||||
}
|
||||
if !applies {
|
||||
h.decline(c.Key, row.ID, why)
|
||||
continue
|
||||
}
|
||||
h.forget(c.Key)
|
||||
spent := spentOn(heals, row, budget, now)
|
||||
if n := len(spent); n > 0 && now.Sub(spent[n-1].At) < row.Settle {
|
||||
continue // its last act is still the observation's to judge
|
||||
}
|
||||
if len(spent) >= row.Budget {
|
||||
h.escalate(ctx, row, c, budget, spent)
|
||||
continue
|
||||
}
|
||||
if len(acts) >= healBrakeLimit {
|
||||
return // the brake takes hold on the next look, said there
|
||||
}
|
||||
if h.act(ctx, row, c, budget, len(spent)) {
|
||||
acts = append(acts, inventory.Heal{At: now})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// act is one healer's act on one condition: begun in the store, made, finished, kept in the
|
||||
// condition's tried and said. False when it could not even be begun.
|
||||
func (h *healing) act(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent int) bool {
|
||||
inv := h.open.inventory
|
||||
begun, err := inv.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
|
||||
BudgetKey: budget, Act: row.Repair, At: h.now()})
|
||||
if err != nil {
|
||||
h.say("healer %s did not act on %s: %v", row.ID, c.Key, err)
|
||||
return false
|
||||
}
|
||||
act, said, err := row.repair(ctx, h, c)
|
||||
outcome := inventory.HealActed
|
||||
if err != nil {
|
||||
outcome, said = inventory.HealFailed, err.Error()
|
||||
}
|
||||
if act == "" {
|
||||
act = row.Repair
|
||||
}
|
||||
finishing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if ferr := inv.FinishHeal(finishing, begun.ID, outcome, act+" — "+said); ferr != nil {
|
||||
h.say("healer %s acted on %s and could not record what came of it: %v", row.ID, c.Key, ferr)
|
||||
}
|
||||
budgetWords := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window)
|
||||
attempt := conditions.Attempt{What: act, Outcome: outcome + ": " + said + " (" + budgetWords + ")",
|
||||
By: "healer " + row.ID}
|
||||
if _, _, terr := h.keeper.Tried(finishing, c.Key, attempt, conditions.ResolverHealer(row.ID)); terr != nil {
|
||||
h.say("healer %s acted on %s and could not keep it in the condition: %v", row.ID, c.Key, terr)
|
||||
}
|
||||
h.tell(finishing, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: act, Outcome: outcome,
|
||||
Said: said, Budget: budgetWords, Epoch: begun.Epoch})
|
||||
h.say("healer %s %s %s: %s — %s (%s)", row.ID, outcome, c.Key, act, said, budgetWords)
|
||||
return true
|
||||
}
|
||||
|
||||
// escalate is a spent budget: the condition is the operator's now, urgent, with what was tried. Said
|
||||
// once: an escalated condition is passed over by every healer until observation clears it.
|
||||
func (h *healing) escalate(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent []inventory.Heal) {
|
||||
var tried []string
|
||||
for _, s := range spent {
|
||||
tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.At.UTC().Format("15:04 MST"), firstLine(s.Said)))
|
||||
}
|
||||
said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window,
|
||||
c.Key, strings.Join(tried, "; "))
|
||||
if _, err := h.open.inventory.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
|
||||
BudgetKey: budget, Act: "handed the condition to the operator", Outcome: inventory.HealEscalated, Said: said,
|
||||
At: h.now()}); err != nil {
|
||||
h.say("healer %s could not record handing %s to the operator, and does not: %v", row.ID, c.Key, err)
|
||||
return
|
||||
}
|
||||
attempt := conditions.Attempt{What: "handed to the operator: nothing in the mesh will repair it now",
|
||||
Outcome: inventory.HealEscalated + ": " + said, By: "healer " + row.ID}
|
||||
if _, _, err := h.keeper.Escalate(ctx, c.Key, attempt); err != nil {
|
||||
h.say("healer %s could not hand %s to the operator: %v", row.ID, c.Key, err)
|
||||
}
|
||||
h.tell(ctx, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator",
|
||||
Outcome: inventory.HealEscalated, Said: said, Budget: fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window)})
|
||||
h.say("healer %s handed %s to the operator: %s", row.ID, c.Key, said)
|
||||
}
|
||||
|
||||
// healerActed is the body of `healer-acted` (to-be 45 §7): the healer, the condition, the act and its
|
||||
// outcome, and where the budget stands. **A contract**, like a condition's events: the operator-channel's
|
||||
// holder may say it.
|
||||
type healerActed struct {
|
||||
Event string `json:"event"`
|
||||
At time.Time `json:"at"`
|
||||
Healer string `json:"healer"`
|
||||
By string `json:"by"`
|
||||
Condition string `json:"condition"`
|
||||
Kind string `json:"kind"`
|
||||
Act string `json:"act"`
|
||||
// Outcome is acted, failed or escalated. Acted says the act was made, not that it worked: the
|
||||
// condition clearing says that.
|
||||
Outcome string `json:"outcome"`
|
||||
Said string `json:"said"`
|
||||
Budget string `json:"budget"`
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// tell says a heal on the bus. Not said is said in the log: the act and the condition's tried still
|
||||
// hold it.
|
||||
func (h *healing) tell(ctx context.Context, e healerActed) {
|
||||
e.Event, e.At, e.By = link.KeyHealerActed, h.now().UTC(), "healer "+e.Healer
|
||||
e.Show = "mesh-controller.conditions key=" + e.Condition
|
||||
if h.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := h.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyHealerActed, body); err != nil {
|
||||
h.say("healer %s %s %s, and that could NOT be said on the bus: %v", e.Healer, e.Outcome, e.Condition, err)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcile keeps the healers' own conditions: the brake, and their being blind.
|
||||
func (h *healing) reconcile(ctx context.Context, observed []conditions.Observation) {
|
||||
if err := h.keeper.Reconcile(ctx, sourceHealers, observed); err != nil {
|
||||
h.say("the healers' own conditions could not be kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *healing) pause(why string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.paused != why {
|
||||
h.say("the healers act on nothing: %s", why)
|
||||
}
|
||||
h.paused = why
|
||||
}
|
||||
|
||||
func (h *healing) resume() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.paused != "" {
|
||||
h.say("the healers act again")
|
||||
}
|
||||
h.paused = ""
|
||||
}
|
||||
|
||||
// decline remembers why a healer passed a condition over, said once.
|
||||
func (h *healing) decline(key, healer, why string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.declined[key] != why {
|
||||
h.say("healer %s leaves %s alone: %s", healer, key, why)
|
||||
}
|
||||
h.declined[key] = why
|
||||
}
|
||||
|
||||
func (h *healing) forget(key string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
delete(h.declined, key)
|
||||
}
|
||||
|
||||
// actsWithin are the heals since a moment that were acts — begun, made or failed; an escalation is a
|
||||
// saying, not an act, and the brake does not count it.
|
||||
func actsWithin(heals []inventory.Heal, since time.Time) []inventory.Heal {
|
||||
var out []inventory.Heal
|
||||
for _, h := range heals {
|
||||
if !h.At.Before(since) && h.Outcome != inventory.HealEscalated {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// spentOn is one healer's acts against one budget key within its window, oldest first.
|
||||
func spentOn(heals []inventory.Heal, row healerRow, budget string, now time.Time) []inventory.Heal {
|
||||
var out []inventory.Heal
|
||||
for _, h := range actsWithin(heals, now.Add(-row.Window)) {
|
||||
if h.Healer == row.ID && h.BudgetKey == budget {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake
|
||||
// already said and an act within the hour still — it lets go an hour after the last act, not the first.
|
||||
func brakeHolds(acts []inventory.Heal, open []conditions.Condition, now time.Time) (bool, string) {
|
||||
said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), healBrakeWindow, healBrakeLimit)
|
||||
if len(acts) >= healBrakeLimit {
|
||||
return true, said
|
||||
}
|
||||
held := slices.ContainsFunc(open, func(c conditions.Condition) bool { return c.Kind == kindHealersBraked })
|
||||
if held && len(acts) > 0 {
|
||||
last := acts[len(acts)-1].At
|
||||
return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said,
|
||||
last.Add(healBrakeWindow).UTC().Format("15:04 MST"))
|
||||
}
|
||||
return false, said
|
||||
}
|
||||
|
||||
// brakeObservation is the brake, as the urgent condition that says it.
|
||||
func brakeObservation(acts []inventory.Heal, said string) conditions.Observation {
|
||||
counts := map[string]int{}
|
||||
for _, a := range acts {
|
||||
if a.Healer != "" {
|
||||
counts[a.Healer+" on "+a.ConditionKey]++
|
||||
}
|
||||
}
|
||||
var most []string
|
||||
for what, n := range counts {
|
||||
most = append(most, fmt.Sprintf("%s ×%d", what, n))
|
||||
}
|
||||
sort.Strings(most)
|
||||
return conditions.Observation{Scope: conditions.ScopeMesh, ID: "healers", Token: "braked", Kind: kindHealersBraked,
|
||||
Severity: conditions.Urgent,
|
||||
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
|
||||
"healer looping, not repairing — " + said,
|
||||
Said: strings.Join(most, ", ")}
|
||||
}
|
||||
|
||||
// --- H1 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// appliesToAMachine is H1's: a machine named, its budget counted against the condition.
|
||||
func appliesToAMachine(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if c.Subject.Machine == "" {
|
||||
return "", false, "it names no machine", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairReport is H1: ask the machine to report; if what comes back is not what it was sent, or nothing
|
||||
// comes, send it again.
|
||||
func repairReport(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
node := c.Subject.Machine
|
||||
inv := h.open.inventory
|
||||
// The store's clock, as the report's time is: not the runner's, which a test moves by hand.
|
||||
asked := time.Now()
|
||||
askErr := h.askReport(ctx, node)
|
||||
current, heard := false, false
|
||||
if askErr == nil {
|
||||
deadline := time.Now().Add(h.reportWait)
|
||||
for {
|
||||
r, found, err := lastReportOf(ctx, inv, node)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if found && r.At != nil && r.At.After(asked) {
|
||||
heard, current = true, r.Current
|
||||
if current {
|
||||
break
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", "", ctx.Err()
|
||||
case <-time.After(min(2*time.Second, h.reportWait/4+time.Millisecond)):
|
||||
}
|
||||
}
|
||||
}
|
||||
if current {
|
||||
return "asked " + node + "'s node-engine to say again what it last applied",
|
||||
"it reported the declaration it was sent: its report had not reached the mesh", nil
|
||||
}
|
||||
why := "it said nothing within " + h.reportWait.String() + " — its node-engine may be older than the report verb"
|
||||
switch {
|
||||
case askErr != nil:
|
||||
why = "it could not be asked: " + askErr.Error()
|
||||
case heard:
|
||||
why = "it reported a declaration other than the one it was sent"
|
||||
}
|
||||
if err := h.sendAgain(ctx, node); err != nil {
|
||||
return "asked " + node + " to report, then sent it its current declaration again", why + "; the send failed",
|
||||
err
|
||||
}
|
||||
return "asked " + node + " to report, then sent it its current declaration again", why + "; sent again", nil
|
||||
}
|
||||
|
||||
// lastReportOf is one machine's last report beside its last send.
|
||||
func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (inventory.Reported, bool, error) {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return inventory.Reported{}, false, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r, true, nil
|
||||
}
|
||||
}
|
||||
return inventory.Reported{}, false, nil
|
||||
}
|
||||
|
||||
// --- H2 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty
|
||||
// when it is neither, which is not H2's to repair.
|
||||
func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) {
|
||||
if p.Release != nil {
|
||||
return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, newer := range recent {
|
||||
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
|
||||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded {
|
||||
continue
|
||||
}
|
||||
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
|
||||
"and branch", newer.ID, newer.Repository, short(newer.Commit)), nil
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil || (s.State != "built" && s.State != "failed") {
|
||||
return "", "", nil
|
||||
}
|
||||
if s.State == "built" && s.SentAt == nil {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if u.RollOut {
|
||||
return "", "", nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return inventory.PlanDone, "finished: every module of every tier is built or failed, and every one that rolls " +
|
||||
"out was sent — nothing is left to wait on", nil
|
||||
}
|
||||
|
||||
// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished.
|
||||
func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if c.Subject.Scope == conditions.ScopeDelivery {
|
||||
return appliesToAStalledDelivery(ctx, h, c)
|
||||
}
|
||||
p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
if !p.Open() {
|
||||
return "", false, "the plan is " + p.State + " already: its condition clears on the next look", nil
|
||||
}
|
||||
state, _, err := planStale(ctx, h.open.inventory, p)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
if state == "" {
|
||||
return "", false, "its wait is neither superseded nor finished: it waits on something still to come, " +
|
||||
"which its own signal says", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set.
|
||||
func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
if c.Subject.Scope == conditions.ScopeDelivery {
|
||||
return repairDelivery(ctx, h, c)
|
||||
}
|
||||
inv := h.open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, c.Subject.ID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if !p.Open() {
|
||||
return "closed nothing", "the plan is " + p.State + " already", nil
|
||||
}
|
||||
state, why, err := planStale(ctx, inv, p)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if state == "" {
|
||||
return "closed nothing", "its wait is no longer superseded or finished", nil
|
||||
}
|
||||
p.State = state
|
||||
p.Note = fmt.Sprintf("closed by healer H2 at tier %d: %s", p.Tier, why)
|
||||
if state != inventory.PlanDone {
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return "closed the plan " + p.ID + " (" + state + ")", why, nil
|
||||
}
|
||||
|
||||
// --- H3 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// appliesToAnObject is H3's: every holder or consumer the probe or the bus names, its budget per object.
|
||||
func appliesToAnObject(_ context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if h.assertObjects == nil {
|
||||
return "", false, "this controller is not on the bus", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairObjects is H3: the send's own assertion of every stream, consumer and seat worker.
|
||||
func repairObjects(ctx context.Context, h *healing, _ conditions.Condition) (string, string, error) {
|
||||
if err := h.assertObjects(ctx); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return "asserted the bus's streams, consumers and seat workers again",
|
||||
"every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there", nil
|
||||
}
|
||||
|
||||
// --- H4 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// resettable is why a consumer may be reset by the mesh itself, from the stream table; empty when not.
|
||||
func resettable(stream, name string) string {
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
if c.Stream == stream && c.Name == name {
|
||||
return c.Resettable
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// consumerOf is the stream and consumer a bus condition names: `<stream>.<consumer>`.
|
||||
func consumerOf(c conditions.Condition) (string, string, bool) {
|
||||
stream, name, found := strings.Cut(c.Subject.ID, ".")
|
||||
return stream, name, found && stream != "" && name != ""
|
||||
}
|
||||
|
||||
// appliesToAResettableConsumer is H4's: only a consumer the stream table marks resettable.
|
||||
func appliesToAResettableConsumer(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
stream, name, ok := consumerOf(c)
|
||||
if !ok {
|
||||
return "", false, "it names no consumer", nil
|
||||
}
|
||||
if resettable(stream, name) == "" {
|
||||
return "", false, fmt.Sprintf("%s on %s is not marked resettable in the stream table: what a reset drops, "+
|
||||
"nothing would catch up", name, stream), nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairConsumer is H4: `broker consumer-reset`, made by the mesh.
|
||||
func repairConsumer(_ context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
stream, name, _ := consumerOf(c)
|
||||
said, err := h.resetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return fmt.Sprintf("re-made %s on %s to deliver from now", name, stream),
|
||||
said + "; " + resettable(stream, name), nil
|
||||
}
|
||||
|
||||
// --- the verb ----------------------------------------------------------------------------------------
|
||||
|
||||
// healersCommand is `healers`: the registry, what the healers did lately, and the brake.
|
||||
func healersCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("healers", flag.ContinueOnError)
|
||||
daysFlag := set.Int("days", 7, "how many days of acts back")
|
||||
jsonFlag := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("healers [--days N] [--json]")
|
||||
}
|
||||
days, asJSON := *daysFlag, *jsonFlag
|
||||
if days <= 0 {
|
||||
return fmt.Errorf("healers --days takes a number of days, not %d", days)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
now := time.Now()
|
||||
heals, err := open.inventory.HealsSince(ctx, now.Add(-time.Duration(days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return fmt.Errorf("what the healers did cannot be read: %w", err)
|
||||
}
|
||||
conds, condErr := openConditions(ctx)
|
||||
braked, said := brakeHolds(actsWithin(heals, now.Add(-healBrakeWindow)), conds, now)
|
||||
brake := map[string]any{"holds": braked, "said": said, "limit": healBrakeLimit, "window": healBrakeWindow.String()}
|
||||
if condErr != nil {
|
||||
brake["conditions"] = "the open conditions could not be read, so whether the brake was said is not known: " +
|
||||
condErr.Error()
|
||||
}
|
||||
var rows []map[string]any
|
||||
for _, r := range healerRegistry {
|
||||
rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair,
|
||||
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
|
||||
"then": r.Then, "acts-in": r.ActsIn, "event": r.Event, "from": r.From})
|
||||
}
|
||||
if heals == nil {
|
||||
heals = []inventory.Heal{}
|
||||
}
|
||||
if asJSON {
|
||||
return printJSON(map[string]any{"healers": rows, "heals": heals, "brake": brake, "days": days,
|
||||
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"})
|
||||
}
|
||||
for _, r := range healerRegistry {
|
||||
fmt.Printf("%s %s\n → %s\n budget %d within %s; then %s (in %s)\n", r.ID, r.Condition, r.Repair, r.Budget,
|
||||
r.Window, r.Then, r.ActsIn)
|
||||
}
|
||||
fmt.Printf("\nthe brake: %s — %s\n\n", map[bool]string{true: "HOLDS, every healer has stopped", false: "off"}[braked], said)
|
||||
if len(heals) == 0 {
|
||||
fmt.Printf("no healer acted in the last %d day(s)\n", days)
|
||||
return nil
|
||||
}
|
||||
for i := len(heals) - 1; i >= 0; i-- {
|
||||
x := heals[i]
|
||||
fmt.Printf("%s %s %-9s %s\n %s\n", x.At.Local().Format("2006-01-02 15:04"), x.Healer, x.Outcome, x.ConditionKey,
|
||||
firstLine(x.Said))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldHeals removes heals past their keeping once a day, by the controller acting only.
|
||||
func forgettingOldHeals(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if link.Holding() {
|
||||
if n, err := inv.ForgetOldHeals(ctx); err != nil {
|
||||
fmt.Printf("heals older than %s could not be removed: %v\n", inventory.HealsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d heal(s) older than %s\n", n, inventory.HealsKeptFor)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// healsCount is what the healers did, counted for `status`.
|
||||
type healsCount struct {
|
||||
Acts int `json:"acts"`
|
||||
Escalated int `json:"escalated"`
|
||||
}
|
||||
|
||||
// countHeals counts acts and escalations.
|
||||
func countHeals(heals []inventory.Heal) *healsCount {
|
||||
out := &healsCount{}
|
||||
for _, h := range heals {
|
||||
if h.Outcome == inventory.HealEscalated {
|
||||
out.Escalated++
|
||||
} else {
|
||||
out.Acts++
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,640 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
|
||||
// checked"): **every row is walked.** Its kinds are ones the mesh raises — a row of the signals table, a
|
||||
// probe of the self-check, or a provider's event — it has a budget, a window and a settle inside it,
|
||||
// what happens when the budget is spent, and the event each act is said as; a healer the controller runs
|
||||
// has its applies and its repair, and an induced failure below that sees it act, say so and brake. A
|
||||
// row added without one fails, so the registry cannot grow a healer nobody has seen act.
|
||||
|
||||
// raisedKinds is every condition kind the mesh raises, and what raises it.
|
||||
func raisedKinds() map[string]string {
|
||||
out := map[string]string{kindProviderFailing: "the provisioner.failing event (ADR 0224)"}
|
||||
for _, r := range signalsTable {
|
||||
for _, k := range kindsOf(r) {
|
||||
out[k] = r.Row
|
||||
}
|
||||
}
|
||||
for _, p := range probeRegistry {
|
||||
out[p.Kind] = p.ID
|
||||
for _, k := range p.Raises {
|
||||
out[k] = p.ID
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inducedFailures are the healers seen acting in this file, by id: a row without one fails.
|
||||
var inducedFailures = map[string]string{
|
||||
"H1": "TestH1AsksAMachineToReportAndSendsItAgain",
|
||||
"H2": "TestH2ClosesAPlanAnotherHasTakenOver",
|
||||
"H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget",
|
||||
"H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers",
|
||||
}
|
||||
|
||||
func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) {
|
||||
kinds := raisedKinds()
|
||||
source, err := os.ReadFile("healers_test.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
answered := map[string]string{}
|
||||
for _, r := range healerRegistry {
|
||||
t.Run(r.ID, func(t *testing.T) {
|
||||
if seen[r.ID] {
|
||||
t.Fatalf("%s is in the registry twice", r.ID)
|
||||
}
|
||||
seen[r.ID] = true
|
||||
if len(r.Kinds) == 0 || r.Condition == "" || r.Repair == "" || r.Then == "" || r.From == "" || r.ActsIn == "" {
|
||||
t.Fatalf("%s does not say what it answers, what it does, what then, and which hand act it replaces: %+v", r.ID, r)
|
||||
}
|
||||
for _, k := range r.Kinds {
|
||||
if _, ok := kinds[k]; !ok {
|
||||
t.Errorf("%s answers %q, which nothing in the mesh raises", r.ID, k)
|
||||
}
|
||||
if other, twice := answered[k]; twice {
|
||||
t.Errorf("%q is answered by %s and %s: one healer per kind", k, other, r.ID)
|
||||
}
|
||||
answered[k] = r.ID
|
||||
}
|
||||
if r.Budget <= 0 || r.Window <= 0 || r.Settle <= 0 || r.Settle > r.Window {
|
||||
t.Errorf("%s has no budget it can spend: %d within %s, settled after %s", r.ID, r.Budget, r.Window, r.Settle)
|
||||
}
|
||||
if r.Event == "" {
|
||||
t.Errorf("%s says nothing when it acts", r.ID)
|
||||
}
|
||||
if r.ActsIn != actsInController {
|
||||
if r.repair != nil || r.applies != nil {
|
||||
t.Errorf("%s acts in %s and the controller would act for it too", r.ID, r.ActsIn)
|
||||
}
|
||||
return
|
||||
}
|
||||
if r.repair == nil || r.applies == nil {
|
||||
t.Fatalf("%s acts in the controller and has no repair or no applies", r.ID)
|
||||
}
|
||||
if r.Event != link.KeyHealerActed || !slices.Contains(broker.ControllerStates, r.Event) {
|
||||
t.Errorf("%s is said as %q, which the controller's grant does not permit", r.ID, r.Event)
|
||||
}
|
||||
if inducedFailures[r.ID] == "" {
|
||||
t.Errorf("%s has no induced failure: a healer nobody has seen act", r.ID)
|
||||
} else if !strings.Contains(string(source), "func "+inducedFailures[r.ID]+"(t *testing.T)") {
|
||||
t.Errorf("%s's induced failure %s is not a test in this file", r.ID, inducedFailures[r.ID])
|
||||
}
|
||||
})
|
||||
}
|
||||
for id := range inducedFailures {
|
||||
if !seen[id] {
|
||||
t.Errorf("an induced failure for %s, which the registry does not have", id)
|
||||
}
|
||||
}
|
||||
if healBrakeLimit <= 0 || healBrakeWindow <= 0 {
|
||||
t.Error("the mesh-wide brake holds nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// heard keeps the healer-acted events said.
|
||||
type heard struct {
|
||||
mu sync.Mutex
|
||||
acts []healerActed
|
||||
}
|
||||
|
||||
func (h *heard) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||
if seat != conditions.Seat || event != link.KeyHealerActed {
|
||||
return errors.New("said under the wrong seat or name: " + seat + " " + event)
|
||||
}
|
||||
var e healerActed
|
||||
if err := json.Unmarshal(body, &e); err != nil {
|
||||
return err
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.acts = append(h.acts, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *heard) said() []healerActed {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]healerActed(nil), h.acts...)
|
||||
}
|
||||
|
||||
// testClock is a moment a test moves by hand.
|
||||
type testClock struct {
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func (c *testClock) now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.at
|
||||
}
|
||||
|
||||
func (c *testClock) pass(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.at = c.at.Add(d)
|
||||
}
|
||||
|
||||
// healingOn is a runner over a mesh's stores and its condition store, under epoch 57, every act a
|
||||
// fake that fails the test unless the test gives it.
|
||||
func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) {
|
||||
t.Helper()
|
||||
if conditionsFrom == nil {
|
||||
t.Fatal("the mesh has no condition store")
|
||||
}
|
||||
told, clock := &heard{}, &testClock{at: time.Now()}
|
||||
// The store's gate, as the serving controller's is the lease's (stores.go).
|
||||
open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
|
||||
h := &healing{open: open, keeper: conditionsFrom, teller: told,
|
||||
epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now,
|
||||
say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond,
|
||||
declined: map[string]string{}}
|
||||
h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil }
|
||||
h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil }
|
||||
h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil }
|
||||
h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil }
|
||||
return h, told, clock
|
||||
}
|
||||
|
||||
// sentNotReported raises S2 for a machine, as the watchdog does.
|
||||
func sentNotReported(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "sent-not-reported", Machine: node,
|
||||
Severity: conditions.Warning, Summary: node + " was sent a declaration and has not reported it", Source: "S2"}
|
||||
if _, err := conditionsFrom.Observe(t.Context(), o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return o.Key()
|
||||
}
|
||||
|
||||
// **H1, the commonest hand act** (031/01 §f: a push by hand to unstick a plan waiting on a report, four
|
||||
// times): the machine is asked to report; a report that names what it was sent is all it takes, and one
|
||||
// that does not — or none — is a send of its current declaration again. Each act kept in `tried` as
|
||||
// `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more.
|
||||
func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := sentNotReported(t, "laptop")
|
||||
|
||||
// First: the report had not reached the mesh, and asking for it brings it.
|
||||
asked := 0
|
||||
h.askReport = func(ctx context.Context, node string) error {
|
||||
asked++
|
||||
if node != "laptop" {
|
||||
t.Errorf("asked %s", node)
|
||||
}
|
||||
_, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied,
|
||||
At: time.Now().Add(time.Second), Declared: "d2"})
|
||||
return err
|
||||
}
|
||||
h.tick(ctx)
|
||||
c, found, err := conditionsFrom.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err)
|
||||
}
|
||||
if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") ||
|
||||
c.Resolver != "healer:H1" {
|
||||
t.Fatalf("after the first act: asked %d, %+v", asked, c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed ||
|
||||
acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 {
|
||||
t.Fatalf("the act was not said as healer-acted: %+v", acts)
|
||||
}
|
||||
|
||||
// Within its settle, nothing more: the observation has its turn.
|
||||
clock.pass(time.Minute)
|
||||
h.tick(ctx)
|
||||
if asked != 1 {
|
||||
t.Fatalf("acted again inside the settle: asked %d", asked)
|
||||
}
|
||||
|
||||
// Second: the machine says nothing, so it is sent again.
|
||||
clock.pass(3 * time.Minute)
|
||||
sent := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(_ context.Context, node string) error { sent++; return nil }
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 {
|
||||
t.Fatalf("the second act: asked %d, sent %d", asked, sent)
|
||||
}
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") {
|
||||
t.Fatalf("tried %+v", c.Tried)
|
||||
}
|
||||
|
||||
// The budget is spent: the operator's, urgent, said; and no healer touches it again.
|
||||
clock.pass(4 * time.Minute)
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 ||
|
||||
!strings.Contains(c.Tried[2].Outcome, "budget of 2") {
|
||||
t.Fatalf("not handed to the operator: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("the escalation was not said: %+v", acts)
|
||||
}
|
||||
clock.pass(time.Hour)
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 || len(told.said()) != 3 {
|
||||
t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said()))
|
||||
}
|
||||
heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed ||
|
||||
heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 {
|
||||
t.Fatalf("the heals kept: %+v %v", heals, err)
|
||||
}
|
||||
// And a heal is not a hand act: what S15 counts never sees it.
|
||||
for _, x := range heals {
|
||||
if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") {
|
||||
t.Errorf("a heal reads as a hand act: %+v", x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing.
|
||||
func TestNoHealerActsWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
sentNotReported(t, "laptop")
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, nil }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 57, nil }
|
||||
h.acting = func() bool { return false }
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted without the lease: %+v", told.said())
|
||||
}
|
||||
}
|
||||
|
||||
// **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour
|
||||
// after the last.
|
||||
func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
for i := 0; i < healBrakeLimit; i++ {
|
||||
if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent",
|
||||
Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed,
|
||||
At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sentNotReported(t, "laptop")
|
||||
h.tick(ctx) // the fakes fail the test if anything acts
|
||||
braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked")
|
||||
if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") {
|
||||
t.Fatalf("the brake was not said: %+v %v", braked, err)
|
||||
}
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted under the brake: %+v", told.said())
|
||||
}
|
||||
// Past the hour of the first act, still held: it lets go an hour after the last.
|
||||
clock.pass(30 * time.Minute)
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held {
|
||||
t.Fatal("the brake let go before an hour had passed since the last act")
|
||||
}
|
||||
clock.pass(31 * time.Minute)
|
||||
asked := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(context.Context, string) error { return nil }
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held {
|
||||
t.Fatal("the brake held an hour after the last act")
|
||||
}
|
||||
if asked != 1 {
|
||||
t.Fatalf("the healers did not act again after the brake let go: asked %d", asked)
|
||||
}
|
||||
}
|
||||
|
||||
// **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is
|
||||
// still to come: that one is its own signal's, not a healer's.
|
||||
func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
created := time.Now().UTC().Add(-time.Hour)
|
||||
older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}}
|
||||
waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}
|
||||
for _, p := range []*inventory.Plan{&older, &waiting, &newer} {
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"plan-old", "plan-other"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled",
|
||||
Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
closed, err := open.inventory.PlanByID(ctx, "plan-old")
|
||||
if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") ||
|
||||
!strings.Contains(closed.Note, "plan-new") {
|
||||
t.Fatalf("the superseded plan: %+v %v", closed, err)
|
||||
}
|
||||
if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding {
|
||||
t.Fatalf("a plan still waiting was closed: %+v", other)
|
||||
}
|
||||
if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf {
|
||||
t.Fatalf("a plan H2 does not repair was touched: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// Finished: every module built, none rolled out unsent — closed as done.
|
||||
done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created,
|
||||
State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") {
|
||||
t.Fatalf("a finished plan reads %q %q %v", state, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said
|
||||
// and left — what a reset drops, nothing would catch up for it.
|
||||
func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, _, _ := healingOn(t, open)
|
||||
marked := 0
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
if c.Resettable != "" {
|
||||
marked++
|
||||
if c.Stream != broker.EventsStream || c.Name != broker.ControllerName {
|
||||
t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
if marked != 1 {
|
||||
t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked)
|
||||
}
|
||||
for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||
Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx) // the fake reset fails the test if it is called
|
||||
reset := ""
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
reset = stream + "." + name
|
||||
return "it was 1500 behind", nil
|
||||
}
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller",
|
||||
Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if reset != "EVENTS.controller" {
|
||||
t.Fatalf("reset %q", reset)
|
||||
}
|
||||
}
|
||||
|
||||
// **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3
|
||||
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
|
||||
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
|
||||
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h, told, clock := healingOn(t, open)
|
||||
h.js = js
|
||||
h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") }
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const key = "bus.NODES.laptop.missing"
|
||||
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" {
|
||||
t.Fatalf("the deleted consumer was not raised: %+v", c)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil {
|
||||
t.Fatalf("H3 did not make the consumer again: %v", err)
|
||||
}
|
||||
c, still, _ := conditionsFrom.Get(ctx, key)
|
||||
if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" {
|
||||
t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c)
|
||||
}
|
||||
probe()
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H3 repaired")
|
||||
}
|
||||
// Kept in the history as the keeper says it, a moment later.
|
||||
var cleared *conditions.Event
|
||||
for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) {
|
||||
history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range history {
|
||||
if history[i].Key == key && history[i].Change == conditions.ChangeCleared {
|
||||
cleared = &history[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 {
|
||||
t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared)
|
||||
}
|
||||
|
||||
// Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops.
|
||||
clock.pass(10 * time.Minute)
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil }
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 {
|
||||
t.Fatalf("the spent budget was not handed to the operator: %+v", c)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
clock.pass(2 * time.Hour)
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 2 {
|
||||
t.Fatal("a healer acted on what the operator holds")
|
||||
}
|
||||
}
|
||||
|
||||
// **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it
|
||||
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
|
||||
// receives what comes next: a reset that left the controller deaf would be the incident.
|
||||
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none.
|
||||
events := make(chan *nats.Msg, 64)
|
||||
sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
followed := broker.ControllerFollows[0]
|
||||
for i := 0; i < consumerFarBehind+200; i++ {
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
probe()
|
||||
const key = "bus.EVENTS.controller.behind"
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind {
|
||||
t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c)
|
||||
}
|
||||
h, told, _ := healingOn(t, open)
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
before, after, err := js.ResetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) +
|
||||
" pending now", nil
|
||||
}
|
||||
h.tick(ctx)
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
if found := probe(); len(found) != 0 {
|
||||
t.Fatalf("after the reset the probe still finds %+v", found)
|
||||
}
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H4 repaired")
|
||||
}
|
||||
// What comes next still reaches the controller.
|
||||
for len(events) > 0 {
|
||||
<-events
|
||||
}
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case m := <-events:
|
||||
if strings.Contains(string(m.Data), "after") {
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
case <-deadline:
|
||||
t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
|
||||
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := askToReport(t.Context(), conn, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err := asked.NextMsg(5 * time.Second)
|
||||
if err != nil || !strings.Contains(string(msg.Data), "healer H1") {
|
||||
t.Fatalf("the machine heard %v, %v", msg, err)
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
|
||||
if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") ||
|
||||
slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") {
|
||||
t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// The core components' health definitions, as probes in the self-check's registry (novox/hq to-be 45
|
||||
// §8, §4 `H-*`). The same definitions judge a core component's new build on its first machine (the
|
||||
// gate, gate.go); here they are run against every machine on the self-check's schedule, so a core
|
||||
// component that stops being healthy between upgrades is said too.
|
||||
//
|
||||
// controller holds the lease, and says it is ready: its self-check ran, `status` answered in bound
|
||||
// node-engine has reported its current declaration, under a build the mesh delivered
|
||||
// node tools announced, and answer the bus's discovery
|
||||
// bus every stream and durable consumer the mesh defines is there, and a request crosses the
|
||||
// bus to every machine's node tools and back
|
||||
const kindCoreUnhealthy = "core-unhealthy"
|
||||
|
||||
// probeControllerHealth is H-controller: the lease is held, fresh, by a controller that says it is
|
||||
// ready — or one that started less than the witness's bound ago.
|
||||
func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
h, found, err := theLease.holder(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
unhealthy := func(why string) []conditions.Observation {
|
||||
node := d.host
|
||||
if found && h.Host != "" {
|
||||
node = h.Host
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: lease.ComponentController + "." + node,
|
||||
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Urgent,
|
||||
Summary: "the controller is not healthy: " + why}}
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
return unhealthy("nobody holds the controller lease"), nil
|
||||
case time.Since(h.Renewed) > lease.FreshWithin:
|
||||
return unhealthy(fmt.Sprintf("the lease was last renewed %s ago", time.Since(h.Renewed).Round(time.Second))), nil
|
||||
case (h.Health == nil || !h.Health.Ready) && time.Since(h.Taken) > lease.ReadyWithin:
|
||||
why := "the controller holding the lease does not say it is ready"
|
||||
if h.Health != nil && h.Health.Why != "" {
|
||||
why += ": " + h.Health.Why
|
||||
}
|
||||
return unhealthy(why), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the
|
||||
// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send.
|
||||
func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
return machinesHealth(ctx, d, hostModule)
|
||||
}
|
||||
|
||||
// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them
|
||||
// announced, answering the bus's discovery.
|
||||
func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
return machinesHealth(ctx, d, broker.RuntimeModule)
|
||||
}
|
||||
|
||||
// machinesHealth judges a component on every machine running it and heard from, by its health
|
||||
// definition, as the gate does — with no condition taken as the build's doing.
|
||||
func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
running, err := inv.Running(ctx, component)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := gatherGateFacts(ctx, d.open, coreComponent(component))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.judged = false
|
||||
heard := heardMachines(d)
|
||||
var out []conditions.Observation
|
||||
for _, node := range running {
|
||||
if !heard[node] {
|
||||
continue // a machine not heard from is S1's
|
||||
}
|
||||
if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound {
|
||||
continue // inside the bound of a send: S2's, and the gate's
|
||||
}
|
||||
// What the machine did with what it was sent is not the component's health: the node-engine's is
|
||||
// that it reported its current declaration at all, the node tools' that they answer.
|
||||
if r := f.reports[node]; r.Current || component == broker.RuntimeModule {
|
||||
now := time.Now()
|
||||
r.Current, r.Outcome = true, inventory.OutcomeApplied
|
||||
if r.At == nil {
|
||||
r.At = &now
|
||||
}
|
||||
f.reports[node] = r
|
||||
}
|
||||
if component == hostModule {
|
||||
// A node-engine reporting a build the mesh delivered, current or previous, is the version
|
||||
// split's (D10), not ill health; a build the mesh did not deliver is.
|
||||
f.engines[node] = deliveredOr(shelf[component], f.engines[node])
|
||||
}
|
||||
h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f)
|
||||
if h == healthGood {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node,
|
||||
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)})
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// deliveredOr is the reported engine build, or the current delivered one when the report names any
|
||||
// build at all: what H-engine judges is that it reports, not which.
|
||||
func deliveredOr(m catalogue.Manifest, reported string) string {
|
||||
if reported == "" {
|
||||
return reported
|
||||
}
|
||||
if v := deliveredVersions(m); len(v) > 0 {
|
||||
return v[0]
|
||||
}
|
||||
return reported
|
||||
}
|
||||
|
||||
// componentWords is a core component as a sentence names it.
|
||||
func componentWords(component string) string {
|
||||
switch component {
|
||||
case hostModule:
|
||||
return "node-engine"
|
||||
case broker.RuntimeModule:
|
||||
return "node tools"
|
||||
case catalogue.ControllerSeatName:
|
||||
return "controller"
|
||||
}
|
||||
return component
|
||||
}
|
||||
|
||||
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a
|
||||
// request crosses it to every machine's node tools and back.
|
||||
func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
problems, err := busHealth(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: "mesh", Token: "unhealthy",
|
||||
Kind: kindCoreUnhealthy, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the bus is not healthy: %s", strings.Join(problems, "; ")),
|
||||
Said: strings.Join(problems, "; ")}}, nil
|
||||
}
|
||||
|
||||
// busHealth is the bus's health definition, as what is wanting: nothing when healthy. What the bus's
|
||||
// planned step checks after the bus is replaced, too.
|
||||
var busHealth = func(ctx context.Context, d *doctor) ([]string, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("this controller has no bus to ask")
|
||||
}
|
||||
streams, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var problems []string
|
||||
for _, s := range streams {
|
||||
if _, err := js.StreamInfo(s.Name, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||
problems = append(problems, "the stream "+s.Name+" is missing")
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
for _, c := range consumers {
|
||||
_, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound) {
|
||||
problems = append(problems, consumerWords(c)+" is missing")
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||
}
|
||||
}
|
||||
// The round trip: every machine heard from that runs the node tools answers across the bus.
|
||||
running, err := d.open.inventory.Running(ctx, broker.RuntimeModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var expected []string
|
||||
for _, n := range running {
|
||||
if heard[n] {
|
||||
expected = append(expected, n)
|
||||
}
|
||||
}
|
||||
if len(expected) > 0 {
|
||||
answered, err := servedOnTheBus(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var silent []string
|
||||
for _, n := range expected {
|
||||
if !answered[n].runtime {
|
||||
silent = append(silent, n)
|
||||
}
|
||||
}
|
||||
// One machine's tools silent is that machine's (H-tools); none answering is the bus.
|
||||
if len(silent) == len(expected) {
|
||||
slices.Sort(silent)
|
||||
problems = append(problems, "no request crossed the bus and came back: no machine's node tools answered ("+
|
||||
strings.Join(silent, ", ")+")")
|
||||
}
|
||||
}
|
||||
return problems, nil
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
|
||||
// issue 259, ADR 0221).
|
||||
//
|
||||
// A named push ends by sending every other machine whose declaration differs from what it was last
|
||||
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
|
||||
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
|
||||
// out makes every machine running it differ from the merge on, and so did a module whose plan was
|
||||
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
|
||||
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
|
||||
// everywhere at once.
|
||||
//
|
||||
// What tells the two apart is which build of each module the machine was last sent, kept with every
|
||||
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
|
||||
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
|
||||
|
||||
// heldBack is why a push that did not name a machine must not send it, empty when it may.
|
||||
//
|
||||
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
|
||||
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
|
||||
// machine was last sent — including a module the machine was never sent at all. A move is held when
|
||||
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
|
||||
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
|
||||
// is not known, so a held upgrade cannot be told from anything else.
|
||||
func heldBack(node string, modules []string, sent map[string]string, known bool,
|
||||
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
|
||||
if !known {
|
||||
return []string{"which builds it was last sent is not known — it was last sent before the " +
|
||||
"mesh kept them, or sent a declaration by hand"}
|
||||
}
|
||||
var why []string
|
||||
for _, m := range modules {
|
||||
now := current[m]
|
||||
was, carried := sent[m]
|
||||
if carried && was == now.Commit {
|
||||
continue
|
||||
}
|
||||
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
|
||||
if !now.RollOut {
|
||||
why = append(why, move+", which its upgrade policy records rather than rolls out")
|
||||
continue
|
||||
}
|
||||
if id := planStillToSend(plans, m, node); id != "" {
|
||||
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
|
||||
}
|
||||
}
|
||||
sort.Strings(why)
|
||||
return why
|
||||
}
|
||||
|
||||
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
|
||||
// one holding the module that has neither finished sending it nor sent it here first, and has not
|
||||
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
|
||||
// per machine.
|
||||
func planStillToSend(plans []inventory.Plan, module, node string) string {
|
||||
for _, p := range plans {
|
||||
s, holds := p.Modules[module]
|
||||
if !p.Open() || !holds {
|
||||
continue
|
||||
}
|
||||
if s == nil {
|
||||
return p.ID
|
||||
}
|
||||
if s.SentAt != nil || s.State == "failed" {
|
||||
continue
|
||||
}
|
||||
first := false
|
||||
for _, n := range s.First {
|
||||
if n == node {
|
||||
first = true
|
||||
}
|
||||
}
|
||||
if !first {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func fromBuild(was string, carried bool) string {
|
||||
if !carried {
|
||||
return "(never sent it) "
|
||||
}
|
||||
return "from " + buildName(was) + " "
|
||||
}
|
||||
|
||||
func buildName(commit string) string {
|
||||
if commit == "" {
|
||||
return "a build with no source"
|
||||
}
|
||||
return shortCommit(commit)
|
||||
}
|
||||
|
||||
// heldMachines reads, for each machine named, why a push that did not name it must not send it
|
||||
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
|
||||
// to the send, which says why.
|
||||
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
|
||||
out := map[string][]string{}
|
||||
if len(names) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
inv := open.inventory
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
modules := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it
|
||||
// runs.
|
||||
same := map[string]string{}
|
||||
for m, was := range sent {
|
||||
same[m] = was
|
||||
if f.identical(m, was, current[m].Commit) {
|
||||
same[m] = current[m].Commit
|
||||
}
|
||||
}
|
||||
why := heldBack(node, modules, same, known, current, plans)
|
||||
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
|
||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||
if planStillToSend(plans, mv.Module, node) == "" {
|
||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
}
|
||||
}
|
||||
if len(why) > 0 {
|
||||
sort.Strings(why)
|
||||
out[node] = why
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
|
||||
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
|
||||
func sayHeld(w io.Writer, node string, why []string) {
|
||||
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
|
||||
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
|
||||
"grant from this push among it — waits with it. `push %s` sends it\n",
|
||||
node, strings.Join(why, "; "), node)
|
||||
}
|
||||
|
||||
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
|
||||
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
|
||||
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
|
||||
//
|
||||
// `handled` is every machine already sent or already said; it is not considered again. Answers the
|
||||
// machines that could not be composed, as refusals.
|
||||
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
||||
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
||||
w io.Writer) ([]string, error) {
|
||||
inv := open.inventory
|
||||
var refusals []string
|
||||
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
||||
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
|
||||
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
|
||||
// cascade legitimately still converging, so it is said rather than passed over in silence.
|
||||
rounds := 0
|
||||
for {
|
||||
would, err := wouldSend(ctx, open, nodes)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
behind, err := inv.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var also []string
|
||||
for _, m := range behind {
|
||||
if !handled[m.Node] {
|
||||
also = append(also, m.Node)
|
||||
}
|
||||
}
|
||||
if len(also) == 0 {
|
||||
return refusals, nil
|
||||
}
|
||||
if rounds++; rounds > len(nodes) {
|
||||
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
|
||||
"should not happen; run `push --behind` to finish\n",
|
||||
rounds-1, strings.Join(also, ", "))
|
||||
return refusals, nil
|
||||
}
|
||||
sort.Strings(also)
|
||||
held, err := heldMachines(ctx, open, also)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var sending []string
|
||||
for _, name := range also {
|
||||
// Every candidate this round is marked handled — the sent ones so they are not
|
||||
// re-listed, the held ones because they stay held, and the refused ones so a machine
|
||||
// that cannot be composed does not make the loop spin on it for ever.
|
||||
handled[name] = true
|
||||
if why, isHeld := held[name]; isHeld {
|
||||
sayHeld(w, name, why)
|
||||
continue
|
||||
}
|
||||
sending = append(sending, name)
|
||||
}
|
||||
if len(sending) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
|
||||
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
|
||||
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
|
||||
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
|
||||
// is left out of it said, and its declaration allocated.
|
||||
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
|
||||
return func(held context.Context, node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
reportKept(held, plan)
|
||||
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
if err == nil {
|
||||
reportLeftOut(node, declared)
|
||||
}
|
||||
return declared, err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
|
||||
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
|
||||
// else that moved is still a consequence the push sends (ADR 0083).
|
||||
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{
|
||||
"resolver": {Commit: "c2c2c2c2c2"},
|
||||
"agent": {Commit: "a2", RollOut: true},
|
||||
"network": {},
|
||||
}
|
||||
modules := []string{"network", "resolver", "agent"}
|
||||
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
|
||||
|
||||
// A module whose policy records moved: held, naming it, both builds and why.
|
||||
why := heldBack("laptop", modules, sent, true, current, nil)
|
||||
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
|
||||
!strings.Contains(why[0], "upgrade policy records") {
|
||||
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
|
||||
}
|
||||
|
||||
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
|
||||
sent["resolver"] = "c2c2c2c2c2"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||
t.Fatalf("a machine whose builds are all current was held: %v", why)
|
||||
}
|
||||
|
||||
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
|
||||
sent["agent"] = "a1"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
|
||||
}
|
||||
|
||||
// The last send's builds are not known: held whole.
|
||||
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
|
||||
!strings.Contains(why[0], "not known") {
|
||||
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
|
||||
}
|
||||
|
||||
// A module the machine was never sent, under a recording policy: held, and said so.
|
||||
delete(sent, "resolver")
|
||||
sent["agent"] = "a2"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
|
||||
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
|
||||
t.Fatalf("a module never sent under a recording policy: %v", why)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
|
||||
// naming some other machine must not send them for it.
|
||||
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
|
||||
at := time.Now()
|
||||
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
|
||||
sent := map[string]string{"agent": "a1"}
|
||||
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
|
||||
|
||||
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
|
||||
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
|
||||
t.Fatalf("a machine the plan has not reached was not held: %v", why)
|
||||
}
|
||||
// The first machine itself was sent by the plan: not held by it.
|
||||
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
|
||||
t.Fatalf("the plan's first machine was held: %v", why)
|
||||
}
|
||||
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
|
||||
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
|
||||
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
|
||||
Modules: map[string]*inventory.PlanModule{"agent": s}}}
|
||||
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
|
||||
t.Errorf("%s: not held: %v", what, why)
|
||||
}
|
||||
}
|
||||
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
|
||||
for what, plans := range map[string][]inventory.Plan{
|
||||
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
|
||||
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "failed"}}}},
|
||||
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built"}}}},
|
||||
} {
|
||||
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
|
||||
t.Errorf("%s: held: %v", what, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A send records the build of each module it carried; a module left out of it keeps the build it
|
||||
// was last sent, since the machine keeps that one.
|
||||
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
|
||||
got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
|
||||
current, map[string]string{"a": "a1", "b": "b1"})
|
||||
if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("carried %v, wanted %v", got, want)
|
||||
}
|
||||
// Not known before: the left-out module is not recorded at all, so it reads as never sent.
|
||||
got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
|
||||
if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("carried %v, wanted %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
|
||||
// reads the machine as current — and writes down which machines it declared.
|
||||
type recordedDelivery struct {
|
||||
inv *inventory.Inventory
|
||||
declared []string
|
||||
}
|
||||
|
||||
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
|
||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
r.declared = append(r.declared, s.node)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
}
|
||||
|
||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||
// says which build it is.
|
||||
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
|
||||
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
|
||||
}}
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// A third machine on the private network: once it is sent, every other machine's peers change with
|
||||
// it, which is a consequence a push must still send — no build moved.
|
||||
func aThirdMachine(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.AddNode(ctx, "spare")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
|
||||
{"name": "systemd", "present": true}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var profile map[string]any
|
||||
if err := json.Unmarshal(reported, &profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
|
||||
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
|
||||
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
aResolver(t, open, "c1c1c1c1c1", asked)
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
|
||||
}
|
||||
digestOfLaptop := func() string {
|
||||
sent, err := inv.Outstanding(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sent
|
||||
}
|
||||
before := digestOfLaptop()
|
||||
|
||||
// The change merges; the policy is record. `push anchor` sends the anchor...
|
||||
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ...and its cascade leaves the laptop, saying so.
|
||||
var said bytes.Buffer
|
||||
d.declared = nil
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
|
||||
if err != nil || len(refused) != 0 {
|
||||
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||
}
|
||||
if len(d.declared) != 0 {
|
||||
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
|
||||
}
|
||||
if digestOfLaptop() != before {
|
||||
t.Fatal("the laptop's last send moved: it was sent the held build")
|
||||
}
|
||||
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
|
||||
"upgrade policy records", "`push laptop` sends it"} {
|
||||
if !strings.Contains(said.String(), want) {
|
||||
t.Errorf("the push did not say %q:\n%s", want, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
|
||||
// is held, and that what else it is owed waits with it.
|
||||
aThirdMachine(t, open)
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
|
||||
}
|
||||
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
|
||||
t.Fatalf("the push did not say both:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
|
||||
// (novox/hq ADR 0236).
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
|
||||
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
|
||||
t.Fatalf("the new send did not record the new build: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
|
||||
// for another reason is sent by a push that names someone else.
|
||||
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
|
||||
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// `push spare`, the machine just placed: the others' peers change with it.
|
||||
aThirdMachine(t, open)
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
var said bytes.Buffer
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if strings.Contains(said.String(), "was not sent") {
|
||||
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
|
||||
record, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||
// question.
|
||||
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
|
||||
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
}
|
||||
@@ -18,16 +18,16 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||
plain := func(context.Context, string) (sendable, error) {
|
||||
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
||||
}
|
||||
failing := func(readyNode, []byte) error { return errors.New("the broker went away") }
|
||||
fine := func(readyNode, []byte) error { return nil }
|
||||
failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
|
||||
fine := &recordingDelivery{}
|
||||
|
||||
for name, round := range map[string]func() error{
|
||||
"a body that cannot be marshalled": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
|
||||
return err
|
||||
},
|
||||
"a send that fails": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
|
||||
return err
|
||||
},
|
||||
} {
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
||||
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
|
||||
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
|
||||
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
|
||||
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
|
||||
//
|
||||
// **The holders on record, and only the sole claimant when there are none** — the same answer the
|
||||
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
|
||||
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
|
||||
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
|
||||
// listed here. A holder off the private network is left out: a resolver named at an address nothing
|
||||
// answers is a lookup that waits out its timeout on every name.
|
||||
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
|
||||
var replicated []catalogue.Seat
|
||||
for _, s := range catalogue.Seats() {
|
||||
if s.Replicated && s.Scope == catalogue.ScopeMesh {
|
||||
replicated = append(replicated, s)
|
||||
}
|
||||
}
|
||||
if len(replicated) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address := map[string]string{}
|
||||
for _, p := range places {
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]map[string]string{}
|
||||
for _, seat := range replicated {
|
||||
var nodes []string
|
||||
for _, h := range recorded {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
|
||||
nodes = append(nodes, h.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
var derived []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||
derived = append(derived, e.On...)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(derived) == 1 {
|
||||
nodes = derived
|
||||
}
|
||||
}
|
||||
at := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
if address[n] != "" {
|
||||
at[overlay.InternalName(n)] = address[n]
|
||||
}
|
||||
}
|
||||
out[seat.Name] = at
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
||||
// one consumer's identity.
|
||||
|
||||
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
||||
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, body string) string {
|
||||
p := filepath.Join(dir, name+".json")
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
objects := write("objects", `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
||||
resolver := write("resolver", `{"module":"resolver","version":"1",
|
||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
||||
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
||||
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
||||
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
||||
}
|
||||
out.Reset()
|
||||
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
||||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
||||
strings.Contains(out.String(), "networkmanager wants") {
|
||||
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
||||
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||
Requires: []string{"wildcard-resolution"}})
|
||||
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||
consumer, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over := consumer.Overflowing()
|
||||
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||
}
|
||||
|
||||
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||
// networkmanager, and no push to the provider could go through.
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||
}
|
||||
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||
}
|
||||
grants, _, _, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keyless bool
|
||||
for _, g := range grants {
|
||||
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||
}
|
||||
if !keyless {
|
||||
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||
}
|
||||
var granted []string
|
||||
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||
granted = append(granted, c.From)
|
||||
}
|
||||
if strings.Join(granted, ",") != "files" {
|
||||
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||
!strings.Contains(said, "left out of anchor's grants") {
|
||||
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||
}
|
||||
|
||||
// And `status` names it, and does not call the mesh well while it stands.
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||
t.Fatalf("status does not say it:\n%s", shown)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||
doc.Overflowing[0].Bound.Max != 20 {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
|
||||
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
|
||||
// declaration composed before an assignment changed and sent after a newer one carried the higher
|
||||
// number — and the machine, which refuses a lower number, took the older content as the mesh's
|
||||
// newest word. Taken before the composition reads anything, the order of numbers is the order of
|
||||
// compositions, and the host's refusal does what it is for.
|
||||
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
|
||||
allot := numbered()
|
||||
var composed []string
|
||||
compose := func(stamp string) func(string) (sendable, error) {
|
||||
return func(node string) (sendable, error) {
|
||||
composed = append(composed, stamp)
|
||||
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
|
||||
}
|
||||
}
|
||||
// Composed first — before an assignment changed — and sent last.
|
||||
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
|
||||
// Composed after the change, sent first.
|
||||
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
|
||||
|
||||
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
|
||||
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
|
||||
stale[0].declared.Sequence, fresh[0].declared.Sequence)
|
||||
}
|
||||
// Sent in the other order, the numbers do not change — so the machine that has applied the
|
||||
// fresh one (2) refuses the stale one (1) when it arrives late.
|
||||
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
|
||||
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
|
||||
}
|
||||
if len(composed) != 2 || composed[0] != "before" {
|
||||
t.Fatalf("compositions happened in an unexpected order: %v", composed)
|
||||
}
|
||||
}
|
||||
|
||||
// The number is taken before the first read of the composition, not after it: an allotter that
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (order, error) {
|
||||
if node == "anchor" {
|
||||
return order{}, context.DeadlineExceeded
|
||||
}
|
||||
return order{sequence: 7}, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
if node == "anchor" {
|
||||
t.Fatal("anchor was composed although its number could not be taken")
|
||||
}
|
||||
return sendable{}, nil
|
||||
})
|
||||
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
|
||||
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
|
||||
}
|
||||
if len(refusals) != 1 {
|
||||
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
|
||||
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
|
||||
// current" over a machine that had just been sent something else.
|
||||
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
t.Fatalf("recording a send after cancellation failed: %v", err)
|
||||
}
|
||||
outstanding, err := inv.Outstanding(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outstanding != digest || digest != digestOf(body) {
|
||||
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// A command run at a shell (novox/hq to-be 45 §6): under the holder's epoch while a controller holds the
|
||||
// lease, read at the moment it acts; under a lease of its own while none does, given back as it ends.
|
||||
func TestACommandActsUnderTheHoldersEpochOrItsOwn(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
ctx := t.Context()
|
||||
|
||||
// Nobody holds it: the command takes it, and gives it back.
|
||||
cmd := &actor{}
|
||||
own, err := cmd.epoch(ctx)
|
||||
if err != nil || own == 0 {
|
||||
t.Fatalf("a command with nobody holding the lease acts as %d (%v)", own, err)
|
||||
}
|
||||
if h, found, _ := lease.Current(ctx, kv); !found || h.Epoch != own {
|
||||
t.Fatalf("the command's lease is not on the bus: %+v", h)
|
||||
}
|
||||
cmd.release()
|
||||
if _, found, _ := lease.Current(ctx, kv); found {
|
||||
t.Fatal("the command did not give its lease back as it ended")
|
||||
}
|
||||
|
||||
// A controller holds it: a command acts under that epoch.
|
||||
l, err := lease.Open(ctx, mustJetStream(t, url), broker.LeaseBucket, lease.Options{Holder: lease.Holder{Instance: "serving"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := l.TryTake(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
borrower := &actor{}
|
||||
defer borrower.release()
|
||||
if got, err := borrower.epoch(ctx); err != nil || got != held {
|
||||
t.Fatalf("a command acts as %d (%v), want the holder's %d", got, err, held)
|
||||
}
|
||||
// The holder lets go: the command does not go on under an epoch nobody holds.
|
||||
l.Release(ctx)
|
||||
if _, err := borrower.epoch(ctx); err == nil {
|
||||
t.Fatal("a command acted under an epoch nobody holds any more")
|
||||
}
|
||||
}
|
||||
|
||||
// mustJetStream is a connection of its own to the test bus.
|
||||
func mustJetStream(t *testing.T, url string) jetstream.JetStream {
|
||||
t.Helper()
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
|
||||
// serving controllers over one store and one bus. The second waits while the first holds the lease; on
|
||||
// a handover it takes it at a higher epoch, the record says which held what and how each ended; and the
|
||||
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
|
||||
// moment it lost it.
|
||||
//
|
||||
// MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus)
|
||||
|
||||
// aBusForTheLease is the test bus with the controller's lease bucket new.
|
||||
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.KeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) })
|
||||
return url, kv
|
||||
}
|
||||
|
||||
func TestTwoControllersOneActs(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Controller A takes the lease.
|
||||
a := &actor{}
|
||||
aCtx, stopA := context.WithCancel(ctx)
|
||||
defer stopA()
|
||||
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epochA, err := a.epoch(ctx)
|
||||
if err != nil || epochA == 0 {
|
||||
t.Fatalf("A holds no epoch: %d, %v", epochA, err)
|
||||
}
|
||||
|
||||
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
|
||||
b := &actor{}
|
||||
bCtx, stopB := context.WithCancel(ctx)
|
||||
defer stopB()
|
||||
tookB := make(chan (<-chan struct{}), 1)
|
||||
go func() {
|
||||
lost, err := b.serveUnderTheLease(bCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Errorf("B: %v", err)
|
||||
close(tookB)
|
||||
return
|
||||
}
|
||||
tookB <- lost
|
||||
}()
|
||||
select {
|
||||
case <-tookB:
|
||||
t.Fatal("B took the lease while A held it")
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B, waiting, may act: %v", err)
|
||||
}
|
||||
|
||||
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
|
||||
stopA()
|
||||
a.release()
|
||||
var lostB <-chan struct{}
|
||||
select {
|
||||
case lostB = <-tookB:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("B did not take the lease A gave back")
|
||||
}
|
||||
select {
|
||||
case <-lostA:
|
||||
default:
|
||||
t.Fatal("A, having given the lease back, is not told it no longer holds it")
|
||||
}
|
||||
epochB, err := b.epoch(ctx)
|
||||
if err != nil || epochB <= epochA {
|
||||
t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err)
|
||||
}
|
||||
if _, err := a.epoch(ctx); err == nil {
|
||||
t.Fatal("A acts after giving the lease back")
|
||||
}
|
||||
ea, _, _ := inv.EpochOf(ctx, epochA)
|
||||
eb, _, _ := inv.EpochOf(ctx, epochB)
|
||||
if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance {
|
||||
t.Fatalf("the record of the handover reads %+v then %+v", ea, eb)
|
||||
}
|
||||
|
||||
// Something else writes the lease's key — a third controller on a clock that read it as expired:
|
||||
// B's next renewal is refused, and B stops acting at once.
|
||||
if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-lostB:
|
||||
case <-time.After(2 * lease.RenewEvery):
|
||||
t.Fatal("B was not told it lost the lease")
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B acts after losing the lease: %v", err)
|
||||
}
|
||||
// Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition
|
||||
// is not raised.
|
||||
inv.ActsUnder(b.epoch)
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
|
||||
State: inventory.PlanBuilding}
|
||||
if err := inv.SavePlan(ctx, &saved); err == nil {
|
||||
t.Fatal("B wrote a plan after losing the lease")
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
|
||||
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
|
||||
defer keeper.Close(context.Background())
|
||||
if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x",
|
||||
Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil {
|
||||
t.Fatal("B raised a condition after losing the lease")
|
||||
}
|
||||
if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost {
|
||||
t.Fatalf("B's epoch does not say it was lost: %+v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease:
|
||||
// it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can.
|
||||
func TestAControllerTheBusRefusesTheLeaseServesUnleasedAndSaysSo(t *testing.T) {
|
||||
a := &actor{serving: true, unleased: "the bus refused the lease's key"}
|
||||
if epoch, err := a.epoch(context.Background()); err != nil || epoch != 0 {
|
||||
t.Fatalf("an unleased controller answers %d, %v: it acts, claiming no epoch", epoch, err)
|
||||
}
|
||||
if st := a.standing(); st.Unleased == "" || st.Held {
|
||||
t.Fatalf("its standing says %+v", st)
|
||||
}
|
||||
// One that neither holds nor is unleased — still waiting — acts on nothing.
|
||||
waiting := &actor{serving: true}
|
||||
if _, err := waiting.epoch(context.Background()); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("a controller waiting for the lease may act: %v", err)
|
||||
}
|
||||
}
|
||||
+126
-7
@@ -8,6 +8,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -54,6 +55,9 @@ func run() error {
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
||||
// 45 §6), so the next controller takes it at once rather than after its age.
|
||||
defer theLease.release()
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
@@ -72,8 +76,28 @@ func run() error {
|
||||
return askCommand(ctx, args[1:])
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return queueCommand(ctx, args[1:])
|
||||
case "cancel":
|
||||
return cancelCommand(ctx, args[1:])
|
||||
case "clear":
|
||||
return clearCommand(ctx, args[1:])
|
||||
case "rebuild":
|
||||
return rebuildCommand(ctx, args[1:])
|
||||
case "replay":
|
||||
return replayCommand(ctx, args[1:])
|
||||
case "kill":
|
||||
return killCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
return collectionCommand(ctx, args[1:])
|
||||
case "plans":
|
||||
return plansCommand(ctx, args[1:])
|
||||
case "delivery":
|
||||
// The verbs the delivery's owner asks with (novox/hq ADR 0239).
|
||||
return deliveryCommand(ctx, args[1:])
|
||||
case "pin":
|
||||
return pinCommand(ctx, args[1:], true)
|
||||
case "unpin":
|
||||
@@ -93,8 +117,20 @@ func run() error {
|
||||
return brokerCommand(ctx, args[1:])
|
||||
case "serve":
|
||||
return serve(ctx)
|
||||
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9).
|
||||
case "facts":
|
||||
return factsCommand(ctx, args[1:])
|
||||
// The merge gate: every machine of the snapshot composed with a change (novox/hq to-be 45 §9).
|
||||
case "merge-gate":
|
||||
return mergeGateCommand(ctx, args[1:])
|
||||
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 286).
|
||||
case "check-here":
|
||||
return checkHereCommand(ctx, args[1:])
|
||||
case "upgrade":
|
||||
return upgradeCommand(ctx, args[1:])
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
|
||||
case "bus":
|
||||
return busCommand(ctx, args[1:])
|
||||
case "declare":
|
||||
return declare(ctx, args[1:])
|
||||
case "overlay":
|
||||
@@ -127,6 +163,31 @@ func run() error {
|
||||
return seatCommand(ctx, args[1:])
|
||||
case "status":
|
||||
return statusCommand(ctx, args[1:])
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
case "hand-act":
|
||||
return handActCommand(ctx, args[1:])
|
||||
case "hand-acts":
|
||||
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||
case "durations":
|
||||
return durationsCommand(ctx, args[1:])
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
case "conditions":
|
||||
return conditionsCommand(ctx, args[1:])
|
||||
case "doctor":
|
||||
return doctorCommand(ctx, args[1:])
|
||||
// What the healers did, and their brake (novox/hq to-be 45 §7).
|
||||
case "healers":
|
||||
return healersCommand(ctx, args[1:])
|
||||
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
||||
// (novox/hq ADR 0230).
|
||||
case "retire":
|
||||
return retireCommand(ctx, args[1:])
|
||||
case "cleanup":
|
||||
return cleanupCommand(ctx, args[1:])
|
||||
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
|
||||
case "data":
|
||||
return dataCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
@@ -173,13 +234,20 @@ func usage() {
|
||||
upgrade <name> roll-out [--together] ...send it to the machines running it
|
||||
upgrade <name> record ...record that they are behind, and send nothing
|
||||
status [--json] what is wrong, what is quiet, and what is out of date
|
||||
retire [--json] every provider waiting for a person to approve a retirement (ADR 0230)
|
||||
retire approve <node> <module> --why <text> retire what it waits with: access off, data kept
|
||||
retire reject <node> <module> --why <text> keep them active; a warning stays open
|
||||
cleanup [list] [--json] every retired consumer per provider: age, size, why
|
||||
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
|
||||
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||
assign <node> <module> put a module on a node
|
||||
unassign <node> <module> take it off
|
||||
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||
unassign <node> <module>... take them off
|
||||
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
||||
what it declares; --yes <digest> cuts it over as previewed
|
||||
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
||||
@@ -199,18 +267,42 @@ func usage() {
|
||||
build --behind build every module the mesh holds older than its source
|
||||
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
||||
builds [<module>] what has been built lately, and what came of it
|
||||
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
|
||||
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
|
||||
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
|
||||
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
|
||||
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||
record an act done by hand outside the mesh (to-be 45 §7)
|
||||
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||
conditions [--scope S] [--severity S] [--machine M] [--json]
|
||||
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
||||
conditions show <key> one condition whole, with its evidence
|
||||
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
||||
conditions history [--days N] [--key K] every raising, change and clearing lately
|
||||
doctor [run|probes|signals] [--json]
|
||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
||||
durations [--kind K] [--days N] [--json]
|
||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||
licence refresh <name> mint a new access token and seal it to every holder
|
||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
||||
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||
pin <node> <provision> <from-node> <module>
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
unpin <node> <provision> put that question back
|
||||
plan <node> [--files|--json] what that node would run, and why
|
||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
||||
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||
that need it; --why records it in the hand-act log
|
||||
version what this binary is
|
||||
|
||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||
@@ -252,26 +344,53 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
||||
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
||||
// became of a build nobody was watching.
|
||||
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||
// recorded or registered.
|
||||
if result.Check != nil || result.Checked != nil {
|
||||
checked(ctx, result)
|
||||
return nil
|
||||
}
|
||||
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
|
||||
// unmerged branch was heard here like any build, registered, and its definition reached a machine
|
||||
// before anyone had reviewed it.
|
||||
if result.DryRun {
|
||||
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||
recordBuildDuration(ctx, b.inv, result, asked)
|
||||
switch {
|
||||
case err != nil && result.Failed != "":
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
if result.Module != "" {
|
||||
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed)
|
||||
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
|
||||
} else {
|
||||
planFailedBuild(ctx, b.open, result)
|
||||
}
|
||||
return nil
|
||||
case errors.Is(err, inventory.ErrSuperseded):
|
||||
// Not a failure: the module is already at what a later request built. A plan that asked
|
||||
// before that later request is answered by it; one that asked after it ignores this.
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
return nil
|
||||
case err != nil:
|
||||
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
|
||||
if manifest.Module != "" {
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error())
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "")
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,414 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The merge gate (novox/hq to-be 45 §9): a change judged against every machine of the snapshot, by the
|
||||
// incidents it is written from. Each case raises a mesh, takes its snapshot, and judges a pull request's
|
||||
// tree against it in throwaway stores of its own.
|
||||
|
||||
// catalogueMesh is two machines and a catalogue repository: a resolver and an object store on the
|
||||
// anchor, and on the laptop a network manager requiring the resolver, an album requiring the object
|
||||
// store, and a login manager. Every module is registered from novox/mesh-catalog, as the mesh's are.
|
||||
func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifests := map[string]string{
|
||||
"objects": `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
|
||||
"resolver": `{"module":"resolver","version":"1",
|
||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`,
|
||||
"networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`,
|
||||
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
|
||||
"lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"],
|
||||
"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`,
|
||||
}
|
||||
for name, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"},
|
||||
{"laptop", "album"}, {"laptop", "lemurs"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f, manifests
|
||||
}
|
||||
|
||||
// aTree is a checkout of the catalogue repository holding these manifests.
|
||||
func aTree(t *testing.T, manifests map[string]string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for name, raw := range manifests {
|
||||
at := filepath.Join(dir, "modules", name)
|
||||
if err := os.MkdirAll(at, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(raw), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict {
|
||||
t.Helper()
|
||||
admin := os.Getenv("MESH_TEST_POSTGRES")
|
||||
in := mergeCheckInput{facts: f, admin: admin, changed: changed}
|
||||
if tree != "" {
|
||||
in.repository, in.tree = "novox/mesh-catalog", tree
|
||||
}
|
||||
v, err := judgeChange(t.Context(), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func withEdit(manifests map[string]string, name, raw string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for k, v := range manifests {
|
||||
out[k] = v
|
||||
}
|
||||
if raw == "" {
|
||||
delete(out, name)
|
||||
} else {
|
||||
out[name] = raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The mesh as it is passes: every machine composes in the gate's store as the controller composed it.
|
||||
func TestTheMeshAsItIsPassesTheGate(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
for _, m := range f.Machines {
|
||||
if !m.Declaration.Composes {
|
||||
t.Fatalf("the mesh itself does not compose: %+v", m.Declaration)
|
||||
}
|
||||
}
|
||||
v := gateJudged(t, f, aTree(t, manifests))
|
||||
if v.Verdict != "pass" {
|
||||
t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report())
|
||||
}
|
||||
for _, m := range v.Machines {
|
||||
if !m.Base.Composes || !m.Change.Composes {
|
||||
t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 263**: a change makes the network manager require the object store's provision; on a machine
|
||||
// whose name is six characters its identity is 26 against a bound of 20. Refused in the pull request,
|
||||
// naming the module, and not on the anchor after it merged.
|
||||
func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
tree := aTree(t, withEdit(manifests, "networkmanager",
|
||||
`{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`))
|
||||
v := gateJudged(t, f, tree)
|
||||
if v.Verdict != "fail" {
|
||||
t.Fatalf("the overflow passed the gate:\n%s", v.Report())
|
||||
}
|
||||
report := v.Report()
|
||||
if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") {
|
||||
t.Errorf("the refusal does not name the module and the provision:\n%s", report)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 236**: a login manager's service that omits its state passed the catalogue check and was
|
||||
// refused whole by the node-engine on the first machine. A change to a module a machine runs, and a
|
||||
// module nobody runs yet, are both refused before merge, naming the machine and the module.
|
||||
func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
broken := `{"module":"lemurs","version":"1","capabilities":["systemd"],
|
||||
"resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}`
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") {
|
||||
t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report())
|
||||
}
|
||||
laptop := snapshot.Pseudonym("machine", "laptop")
|
||||
if !strings.Contains(v.Report(), laptop) {
|
||||
t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report())
|
||||
}
|
||||
|
||||
// And as a new module, which no machine runs: tried on one that could.
|
||||
newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`)
|
||||
newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service")
|
||||
v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") {
|
||||
t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// **Version skew**: a manifest the controller judging it cannot read — the one the mesh runs, for a
|
||||
// catalogue change — fails here, not at registration after the merge.
|
||||
func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album",
|
||||
`{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") {
|
||||
t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// A module a machine runs, removed from its source, fails until it is unassigned (ADR 0236).
|
||||
func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", "")))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") {
|
||||
t.Fatalf("removing a module a machine runs passed:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// **Issues 278 and 280**: a file in no held module's directory read as shared code, and a merge rebuilt the
|
||||
// catalogue. A file is a module's only by being inside it — no build reads one outside — so it rebuilds
|
||||
// nothing, and the gate says why; a merge that does rebuild widely is warned of.
|
||||
func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
was := wideRebuild
|
||||
wideRebuild = 2
|
||||
t.Cleanup(func() { wideRebuild = was })
|
||||
for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} {
|
||||
v := gateJudged(t, f, aTree(t, manifests), file)
|
||||
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" {
|
||||
t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict)
|
||||
}
|
||||
if !strings.Contains(v.Report(), "read by no module's build") {
|
||||
t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report())
|
||||
}
|
||||
}
|
||||
v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go")
|
||||
if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" {
|
||||
t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict)
|
||||
}
|
||||
// With the reference module's definition in the tree, its directory is a module, held or not.
|
||||
withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`)
|
||||
v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts")
|
||||
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 {
|
||||
t.Fatalf("a file of a module nobody holds reads %+v", v.Width)
|
||||
}
|
||||
v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json")
|
||||
if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" {
|
||||
t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
// **A delivery group is judged as one future state** (novox/hq ADR 0239): a catalogue change that needs a
|
||||
// provision only another repository's change adds fails alone and passes composed with it; and a group
|
||||
// whose other head breaks what the first needs fails, naming it.
|
||||
func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) {
|
||||
f, manifests := catalogueMeshWithAnApp(t)
|
||||
needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`)
|
||||
catTree := aTree(t, needsTheApp)
|
||||
alone := gateJudged(t, f, catTree, "modules/album/module.json")
|
||||
if alone.Verdict != "fail" {
|
||||
t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report())
|
||||
}
|
||||
app := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1",
|
||||
"provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog",
|
||||
tree: catTree, changed: []string{"modules/album/module.json"},
|
||||
group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}}
|
||||
together, err := judgeChange(t.Context(), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if together.Verdict == "fail" {
|
||||
t.Fatalf("the group composed together fails:\n%s", together.Report())
|
||||
}
|
||||
if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" {
|
||||
t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
// catalogueMeshWithAnApp is catalogueMesh with an application built from a repository of its own, at its
|
||||
// root, on the anchor.
|
||||
func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifests := map[string]string{
|
||||
"objects": `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
|
||||
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
|
||||
}
|
||||
for name, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f, manifests
|
||||
}
|
||||
|
||||
// busMesh is aMesh with a module on the bus on the laptop, its account issued as `module issue` issues
|
||||
// one: minted, and its credential held as the module's own secret named broker.
|
||||
func busMesh(t *testing.T) snapshot.Facts {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1",
|
||||
"own-secrets":{"broker":"/var/lib/speaker/broker"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "speaker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username()
|
||||
password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule,
|
||||
Node: "laptop", Module: "speaker"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker",
|
||||
`{"user":"`+user+`","password":"`+password+`"}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mc := range f.Machines {
|
||||
if !mc.Declaration.Composes {
|
||||
t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration)
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// **Issue 285**: every machine running a module on the bus failed to compose in the gate's store, with
|
||||
// the change and without — the store held the module's credential and no account for it, which
|
||||
// composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the
|
||||
// mesh issued is raised with its credential, so the machine composes in the gate as on the mesh.
|
||||
func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
||||
f := busMesh(t)
|
||||
v := gateJudged(t, f, "")
|
||||
if v.Verdict != "pass" {
|
||||
t.Fatalf("the mesh as it is does not pass:\n%s", v.Report())
|
||||
}
|
||||
for _, m := range v.Machines {
|
||||
if !m.Base.Composes {
|
||||
t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(v.Summary, "2 of 2 compose") {
|
||||
t.Errorf("the summary does not say every machine composes: %s", v.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 285**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
|
||||
// against a machine that is not the mesh's — broken against broken, which passes whatever the change does.
|
||||
// That is an error, never a pass.
|
||||
func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
|
||||
f := busMesh(t)
|
||||
for i := range f.Machines {
|
||||
// A fact the snapshot does not carry: the module's credential, gone from the laptop's.
|
||||
var kept []snapshot.Accepted
|
||||
for _, a := range f.Machines[i].Accepted {
|
||||
if a.Name != "broker" {
|
||||
kept = append(kept, a)
|
||||
}
|
||||
}
|
||||
f.Machines[i].Accepted = kept
|
||||
}
|
||||
v := gateJudged(t, f, "")
|
||||
if v.Verdict != "error" {
|
||||
t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report())
|
||||
}
|
||||
if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") {
|
||||
t.Errorf("the error does not name what could not be raised: %v", v.Errors)
|
||||
}
|
||||
}
|
||||
|
||||
// A path the snapshot withheld is given a path of its own where an access or a place needs one: a bare
|
||||
// "withheld" is no absolute path, and a machine whose setting composes on the mesh would not in the gate.
|
||||
func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
|
||||
got := standInPaths(map[string]any{
|
||||
"accesses": map[string]any{"races": "withheld", "films": "/media/films", "shows": "/withheld"},
|
||||
"places": map[string]any{"config": map[string]any{"path": "withheld", "owner": "1000:1000"}},
|
||||
"puid": 1000,
|
||||
})
|
||||
accesses := got["accesses"].(map[string]any)
|
||||
if accesses["races"] == accesses["shows"] || !strings.HasPrefix(accesses["races"].(string), "/") ||
|
||||
!strings.HasPrefix(accesses["shows"].(string), "/") || accesses["films"] != "/media/films" {
|
||||
t.Errorf("accesses: %v", accesses)
|
||||
}
|
||||
place := got["places"].(map[string]any)["config"].(map[string]any)
|
||||
if !strings.HasPrefix(place["path"].(string), "/") || place["owner"] != "1000:1000" || got["puid"] != 1000 {
|
||||
t.Errorf("places: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 286**: a check run by hand clones beside a change what the seat clones — one rule, read by the
|
||||
// controller's ask and by the facts — and finds the repository it checks from its origin.
|
||||
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||
for dir, refs := range map[string]map[string]string{
|
||||
"mesh-catalog": {"mesh-catalog": "main"},
|
||||
"mesh-host": {"mesh-host": "c0ffee"},
|
||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
|
||||
} {
|
||||
got := besideRefs(dir, "c0ffee")
|
||||
for d, ref := range refs {
|
||||
if got[d] != ref {
|
||||
t.Errorf("beside %s, %s is cloned at %q, not %q", dir, d, got[d], ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
for owner, want := range map[string][3]string{
|
||||
"ssh://git@git.example:222/novox/mesh-host.git": {"novox", "mesh-host", "ssh://git@git.example:222/novox"},
|
||||
"git@git.example:novox/mesh-tools.git": {"novox", "mesh-tools", "git@git.example:novox"},
|
||||
"http://git.example/novox/hq": {"novox", "hq", "http://git.example/novox"},
|
||||
} {
|
||||
o, r, p := ownerRepoOf(owner)
|
||||
if [3]string{o, r, p} != want {
|
||||
t.Errorf("%s reads as %s %s %s", owner, o, r, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"crypto/ecdh"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -37,6 +40,9 @@ func aMesh(t *testing.T) *stores {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
|
||||
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
|
||||
withConditionsInMemory(t)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -106,3 +112,17 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
||||
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||
}
|
||||
|
||||
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
|
||||
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
|
||||
t.Helper()
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
before := conditionsFrom
|
||||
conditionsFrom = k
|
||||
t.Cleanup(func() {
|
||||
conditionsFrom = before
|
||||
k.Close(context.Background())
|
||||
})
|
||||
return k, store
|
||||
}
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
|
||||
//
|
||||
// The forge's poll announces every merge on the events stream, and the controller acts on what its
|
||||
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
|
||||
// server moved the consumer past it — a fault of consumers with several filters in the server the
|
||||
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
|
||||
// built from that repository stayed behind and were built by hand.
|
||||
//
|
||||
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
|
||||
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
|
||||
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
|
||||
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
|
||||
// would still move something was never acted on — it is said, and acted on now.
|
||||
const (
|
||||
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
|
||||
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
|
||||
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
|
||||
mergeGrace = 10 * time.Minute
|
||||
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
|
||||
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
|
||||
mergeLookBack = 24 * time.Hour
|
||||
// mergeCatchUpEvery is how often the stream is read back.
|
||||
mergeCatchUpEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
// merges is what reads back the forge's announcements; the link server, or a test's list.
|
||||
type merges interface {
|
||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||
}
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := open.inventory.ReadRepositories(ctx)
|
||||
return entries, read, err
|
||||
}
|
||||
failing := ""
|
||||
tick := time.NewTicker(mergeCatchUpEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
watchedMerges.begin()
|
||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||
// says that instead, and leaves what the last one found standing.
|
||||
watchedMerges.end(time.Now(), err)
|
||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("merges the bus may not have handed over are looked for again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
|
||||
// move something is said and acted on, oldest first.
|
||||
//
|
||||
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
|
||||
// because acting on an earlier missed merge of the same repository may have moved what a later one
|
||||
// would have.
|
||||
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
|
||||
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
|
||||
|
||||
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, read, err := catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range all {
|
||||
if now.Sub(a.At) < mergeGrace {
|
||||
continue
|
||||
}
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
moves := wouldMove(a.SourceMoved, entries, read)
|
||||
if len(moves) == 0 {
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moves {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
||||
At: a.At, Modules: names})
|
||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||
isAre(len(names)))
|
||||
if err := act(ctx, a.SourceMoved); err != nil {
|
||||
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
|
||||
a.Owner, a.Repo, a.Commit, err)
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
||||
type missedMerge struct {
|
||||
Owner, Repo, Base, Commit string
|
||||
// At is when the bus took the announcement.
|
||||
At time.Time
|
||||
Modules []string
|
||||
}
|
||||
|
||||
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
||||
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
||||
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
||||
// the fault. The next pass, finding it acted on, clears it.
|
||||
type mergeWatch struct {
|
||||
mu sync.Mutex
|
||||
passed time.Time
|
||||
err error
|
||||
finding []missedMerge
|
||||
missed []missedMerge
|
||||
}
|
||||
|
||||
var watchedMerges = &mergeWatch{}
|
||||
|
||||
func (w *mergeWatch) begin() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = nil
|
||||
}
|
||||
|
||||
func (w *mergeWatch) found(m missedMerge) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = append(w.finding, m)
|
||||
}
|
||||
|
||||
func (w *mergeWatch) end(at time.Time, err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.passed, w.err = at, err
|
||||
if err == nil {
|
||||
w.missed = w.finding
|
||||
}
|
||||
}
|
||||
|
||||
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
||||
// could not read.
|
||||
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// announcedList is the events stream's merges as a test gives them.
|
||||
type announcedList []link.AnnouncedMerge
|
||||
|
||||
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
var out []link.AnnouncedMerge
|
||||
for _, m := range a {
|
||||
if !m.At.Before(since) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
|
||||
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
|
||||
type aCatalogue struct {
|
||||
entries []inventory.Entry
|
||||
acted []string
|
||||
fail error
|
||||
}
|
||||
|
||||
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
return append([]inventory.Entry(nil), c.entries...), nil, nil
|
||||
}
|
||||
|
||||
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
|
||||
return func(_ context.Context, m link.SourceMoved) error {
|
||||
if c.fail != nil {
|
||||
return c.fail
|
||||
}
|
||||
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
|
||||
for _, moved := range wouldMove(m, c.entries, nil) {
|
||||
for i := range c.entries {
|
||||
if c.entries[i].Manifest.Module == moved.Manifest.Module {
|
||||
c.entries[i].Source.Head = m.Commit
|
||||
c.entries[i].Source.Seen = now()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func at(s string) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
|
||||
return link.AnnouncedMerge{
|
||||
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||
MergedAt: mergedAt, Paths: paths,
|
||||
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
|
||||
At: at(onTheBus),
|
||||
}
|
||||
}
|
||||
|
||||
func built(module, repo, path, commit, seen string) inventory.Entry {
|
||||
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
|
||||
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
|
||||
return e
|
||||
}
|
||||
|
||||
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
|
||||
// events stream; the bus never handed it to the controller, which acted on the merges around it and
|
||||
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
|
||||
// move something — so it is said and acted on, once, and only after the controller's own consumer
|
||||
// has had its time with it.
|
||||
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{
|
||||
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
// Acted on when it was announced: looked at after it was merged.
|
||||
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
|
||||
}}
|
||||
stream := announcedList{
|
||||
// Nothing the mesh holds is built from the records repository.
|
||||
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
|
||||
// Acted on: its module was looked at since.
|
||||
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
|
||||
// Never handed over.
|
||||
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
|
||||
"node-tools/internal/console/console.go"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:50:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
pass()
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T22:58:00Z")
|
||||
pass()
|
||||
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
|
||||
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
|
||||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
|
||||
t.Fatalf("the missed merge was not said as one: %q", said)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T23:03:00Z")
|
||||
pass()
|
||||
if len(cat.acted) != 1 || len(said) != 1 {
|
||||
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
|
||||
// that could not be acted on is said and tried again on the next pass.
|
||||
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
|
||||
cat := &aCatalogue{
|
||||
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
|
||||
fail: errors.New("the store is restarting"),
|
||||
}
|
||||
stream := announcedList{
|
||||
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
|
||||
"modules/plex/module.json", "modules/plex/x.ts"),
|
||||
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
|
||||
t.Fatalf("a failed catch-up was not said: %q", said)
|
||||
}
|
||||
cat.fail = nil
|
||||
clock = at("2026-10-05T22:05:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
|
||||
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge older than the look-back is left to the operator: a controller that did not run this
|
||||
// missed it, and acting on it days later would be a surprise rebuild.
|
||||
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
|
||||
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
|
||||
func(string, ...any) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,12 @@ func providedModules() []catalogue.Manifest {
|
||||
// and neither could be swapped for anything, which is the test of whether a thing is a
|
||||
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
|
||||
// to live, and now a module says where it wants it — `facts` in its own manifest.
|
||||
overlay.Manifest(), overlay.DomainManifest(),
|
||||
//
|
||||
// **And the bundle that required it is gone** (novox/hq ADR 0226): `networking` named this
|
||||
// module's requirement and nothing else, so every machine carried two modules for one
|
||||
// network. A machine is assigned the private network itself; what a release stops shipping
|
||||
// is retired at the next start (stores.go, Inventory.RetireUnshipped).
|
||||
overlay.Manifest(),
|
||||
} {
|
||||
var m catalogue.Manifest
|
||||
b, _ := json.Marshal(raw)
|
||||
@@ -59,8 +64,19 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||
if args[0] == "check" {
|
||||
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
||||
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
||||
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
||||
"judge each module's identity on a machine name this many characters long")
|
||||
given, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *longest < 1 {
|
||||
return errors.New("--longest-machine-name is a length, at least 1")
|
||||
}
|
||||
var paths []string
|
||||
for _, a := range args[1:] {
|
||||
for _, a := range given {
|
||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||
under, err := manifestsUnder(a)
|
||||
if err != nil {
|
||||
@@ -71,7 +87,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
paths = append(paths, a)
|
||||
}
|
||||
return moduleCheck(paths, os.Stdout)
|
||||
return moduleCheckFor(paths, *longest, os.Stdout)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -147,6 +163,40 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** (novox/hq ADR 0195): what each module
|
||||
// is, where it runs, whether it is current, and what it says of itself.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Built string `json:"built,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Current bool `json:"current"`
|
||||
Provided bool `json:"provided,omitempty"`
|
||||
// Tools says whether the module answers tools anywhere it runs: a list of its own,
|
||||
// a bundle the runtime serves, or a seat's verbs it claims (novox/hq ADR 0197) —
|
||||
// what the console checks the bus's answers against.
|
||||
Tools bool `json:"tools"`
|
||||
On []string `json:"on"`
|
||||
Provides []string `json:"provides,omitempty"`
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
Claims []string `json:"claims,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
out := make([]listed, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||
for _, c := range m.Claims {
|
||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
fmt.Println("this mesh knows about no modules yet")
|
||||
return nil
|
||||
@@ -300,8 +350,10 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return fmt.Errorf("%s <node> <module>", verb)
|
||||
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
|
||||
// service manager and the package manager — can only go on, or come off, together.
|
||||
if len(args) < 2 {
|
||||
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -315,7 +367,7 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
if verb == "unassign" {
|
||||
act = unassign
|
||||
}
|
||||
said, err := act(ctx, open, args[0], args[1])
|
||||
said, err := act(ctx, open, args[0], args[1:]...)
|
||||
if said != "" {
|
||||
fmt.Println(said)
|
||||
}
|
||||
@@ -458,8 +510,8 @@ const theBrokerSeat = "mesh-broker"
|
||||
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
|
||||
// has adopted it) does the hub stand in, which is where the foundation is by convention.
|
||||
//
|
||||
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the networking
|
||||
// module — not "has an address", which is true of every placed machine and says nothing about
|
||||
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the private network
|
||||
// — not "has an address", which is true of every placed machine and says nothing about
|
||||
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
|
||||
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
|
||||
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
|
||||
@@ -625,7 +677,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
// durable subscription nobody reads.
|
||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
@@ -733,3 +785,23 @@ func claimsFor(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaresTools is whether a module answers tools wherever it runs (novox/hq ADR 0197): it names
|
||||
// tools of its own, its build delivers a bundle the node's runtime serves, or it claims a seat
|
||||
// whose verbs it serves. A module with none is never expected to announce anything.
|
||||
func declaresTools(m catalogue.Manifest) bool {
|
||||
if len(m.Tools) > 0 {
|
||||
return true
|
||||
}
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if len(c.Serves) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -275,25 +275,9 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// No registry trust is composed here any more: the container runtime's module states it, told
|
||||
// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
|
||||
g, err := overlay.From(nodes, cidr, "")
|
||||
if g != nil {
|
||||
// The artifact store, as this network reaches it. Found rather than configured: the
|
||||
// provider is whichever module offers it, on whichever machine holds that module — and if
|
||||
// nothing does yet (genesis raises the registry before the catalogue knows it), there is
|
||||
// no trust to write and nothing is written (novox/hq ADR 0082).
|
||||
//
|
||||
// Refused rather than composed without it when the question could not be answered: a
|
||||
// declaration missing the trust because a lookup failed is a machine that cannot pull,
|
||||
// delivered by a push that reported success — and nothing recomposes it until the next
|
||||
// push (the shape of novox/hq issues 042/048, reappearing as a race).
|
||||
at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
|
||||
if storeErr != nil {
|
||||
return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
|
||||
}
|
||||
if found {
|
||||
g.TrustRegistry(overlay.InternalName(at) + ":" + port)
|
||||
}
|
||||
}
|
||||
if err != nil && len(refused) > 0 {
|
||||
// The network is missing something, and some machines could not be resolved at all. Those
|
||||
// are almost always the same fact: a node that does not resolve contributes nothing, so
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -253,11 +254,10 @@ func theResolver(t *testing.T) catalogue.Manifest {
|
||||
return m
|
||||
}
|
||||
|
||||
// The resolver is handed every machine on the private network as a wildcard, the same set and the
|
||||
// same source as the hosts file, and is handed it again when a machine leaves — through the
|
||||
// module's own manifest asking for the fact, with no module of the mesh's own in between (hal
|
||||
// dnsmasq-app conversion, novox/hq 08-connectivity). The runtime on that machine is pointed at the
|
||||
// machine's own address, where the resolver answers for its containers.
|
||||
// The resolver is handed every machine on the private network as a wildcard, and is handed it again
|
||||
// when a machine leaves — through the module's own manifest asking for the fact, with no module of the
|
||||
// mesh's own in between (hal dnsmasq-app conversion, novox/hq 08-connectivity). It is the mesh's one
|
||||
// resolver (ADR 0194), and the container runtime is given no resolver of its own (ADR 0196).
|
||||
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -265,6 +265,12 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
if _, err := assign(ctx, open, "anchor", "dnsmasq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its bus credential, as assigning issues it where the bus is reachable (novox/hq issue 203):
|
||||
// no bus is known to this test, so it is minted here, or composing refuses the placeholder.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "anchor.dnsmasq", Kind: inventory.BusModule, Node: "anchor", Module: "dnsmasq"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zones := func() string {
|
||||
t.Helper()
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
@@ -286,11 +292,12 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
|
||||
}
|
||||
}
|
||||
// The container runtime is given no resolver of its own (novox/hq ADR 0196): it copies its
|
||||
// machine's, which name the mesh's resolver first. A `dns` key would be a second account of where a
|
||||
// container asks, read only when the runtime starts.
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "dnsmasq.runtime-dns" {
|
||||
if !strings.Contains(r["content"].(string), `"10.77.0.1"`) || r["into"] != "json" {
|
||||
t.Errorf("the runtime is not pointed at this machine's own address, written into its file: %v", r)
|
||||
}
|
||||
t.Errorf("the resolver still writes the runtime's own dns: %v", r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,3 +310,28 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
|
||||
}
|
||||
}
|
||||
|
||||
// The roster is the machines and nothing else (novox/hq ADR 0191): each node's internal domain covers
|
||||
// every route on it, and a node's public domains are public DNS's. A routed name in `.Names` was a
|
||||
// private answer for a public name, handed by a resolver serving a LAN to a phone that could not use it.
|
||||
func TestTheRosterNamesOnlyTheMachines(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(with.Names, with.Machines) {
|
||||
t.Fatalf("%s's roster names more than the machines:\n names %v\n machines %v", node, with.Names, with.Machines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,9 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -44,6 +46,21 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** — the console's discovery reads it
|
||||
// (novox/hq ADR 0195), and a reader that parses a printed column breaks when it is reworded.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Name string `json:"name"`
|
||||
Heard string `json:"heard"`
|
||||
Mode string `json:"mode"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
out := make([]listed, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
out = append(out, listed{Name: n.Name, Heard: heardFrom(n), Mode: modeOf(n), ID: n.ID})
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Said rather than printed as nothing: an empty list and a failed read must never
|
||||
// look the same, and this command answering "none" is only honest because getting
|
||||
@@ -370,8 +387,12 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "accounts" {
|
||||
return busAccounts(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||
return consumerReset(ctx, args[1:])
|
||||
}
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file>")
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||
"broker consumer-reset <stream> <consumer>")
|
||||
}
|
||||
known, err := broker.FromEnvironment()
|
||||
if errors.Is(err, broker.ErrNotConfigured) {
|
||||
@@ -391,6 +412,45 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||
func consumerReset(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
args, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||
}
|
||||
if err := why.require("broker consumer-reset"); err != nil {
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "broker consumer-reset", args)
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
before, after, err := js.ResetConsumer(args[0], args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("consumer %s on %s re-made to deliver from now\n", args[1], args[0])
|
||||
fmt.Printf(" before: delivers %s, delivered to %d, acknowledged to %d, %d pending, %d unacknowledged\n",
|
||||
before.DeliverPolicy, before.Delivered, before.AckFloor, before.Pending, before.AckPending)
|
||||
fmt.Printf(" after: delivers %s, %d pending; what was pending is dropped. A holder bound to it may need its "+
|
||||
"process restarted to bind again\n", after.DeliverPolicy, after.Pending)
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
||||
//
|
||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
||||
@@ -457,6 +517,26 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
fmt.Printf(" public domain %s\n", domain)
|
||||
}
|
||||
|
||||
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
|
||||
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
|
||||
// is something not working now and they are a description. Unreadable is said, not passed over.
|
||||
open, err := openConditions(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
|
||||
}
|
||||
var here []conditions.Condition
|
||||
for _, c := range open {
|
||||
if concerns(c, name) {
|
||||
here = append(here, c)
|
||||
}
|
||||
}
|
||||
if len(here) > 0 {
|
||||
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
|
||||
for _, line := range conditionLines(here, time.Now()) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -500,3 +580,13 @@ func orNotReported(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||
func printJSON(v any) error {
|
||||
body, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -143,10 +144,18 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
||||
}{
|
||||
{"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"},
|
||||
{"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"},
|
||||
{"a file they share", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||
{"a file at the root no build reads (issue 280)", merge([]string{"tsconfig.json"}, false), ""},
|
||||
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
|
||||
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
|
||||
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
|
||||
// novox/hq issue 252: a module the mesh has never registered is still a module, when the merge
|
||||
// shows it is one — and a directory that may be shared code is still shared.
|
||||
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
|
||||
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
|
||||
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
|
||||
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), ""},
|
||||
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""},
|
||||
{"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
@@ -211,3 +220,93 @@ func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a module once built at a commit still follows its branch — a merge into it
|
||||
// matches the module, and a plan re-asks the branch, not the old commit.
|
||||
func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main"}
|
||||
pinned := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a"}
|
||||
if !sourceIs(pinned, m) {
|
||||
t.Error("a module whose record names a commit is left out of a merge into its branch")
|
||||
}
|
||||
full := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a1d2c3b4a5f60718293a4b5c6d7e8f9012"}
|
||||
if !sourceIs(full, m) {
|
||||
t.Error("a full commit hash is read as a branch")
|
||||
}
|
||||
if got := followedBranch("9c97a8a"); got != "" {
|
||||
t.Errorf("a plan would re-ask the old commit %q", got)
|
||||
}
|
||||
if got := followedBranch("release"); got != "release" {
|
||||
t.Errorf("a branch is not followed as named: %q", got)
|
||||
}
|
||||
// A module that follows another branch is still not this merge's.
|
||||
if sourceIs(inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "release"}, m) {
|
||||
t.Error("a module following another branch was matched")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issues 278 and 280: a merge touching the code of a module the mesh does not hold — the
|
||||
// catalogue's reference module, whose manifest it left alone — read as shared and rebuilt every module
|
||||
// built from the repository (103 of them on 2026-10-06, 88 byte-identical); so did a merge-check.sh added at
|
||||
// the root. No build reads a file outside its module's directory, so neither rebuilds anything, said by the
|
||||
// announcer or not.
|
||||
func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
gitea := fromRepo("gitea", repo, "modules/gitea")
|
||||
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
|
||||
known := []inventory.Entry{gitea, keycloak}
|
||||
candidates := []inventory.Entry{gitea, keycloak}
|
||||
merge := func(paths, modules []string, said bool) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Paths: paths,
|
||||
ModuleDirs: modules, ModuleDirsSaid: said}
|
||||
}
|
||||
named := func(entries []inventory.Entry) string {
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
return strings.Join(names, ",")
|
||||
}
|
||||
showcase := []string{"modules/showcase/index.ts"}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
m link.SourceMoved
|
||||
want string
|
||||
}{
|
||||
{"a module held by none, said", merge(showcase, []string{"modules/showcase"}, true), ""},
|
||||
{"beside a held one's change", merge(append([]string{"modules/gitea/index.ts"}, showcase...),
|
||||
[]string{"modules/gitea", "modules/showcase"}, true), "gitea"},
|
||||
{"deeper inside it", merge([]string{"modules/showcase/daemon/index.ts"}, []string{"modules/showcase"}, true), ""},
|
||||
{"a directory holding no manifest, read by no build", merge([]string{"modules/lib/x.go"}, nil, true), ""},
|
||||
{"one of two files in no module", merge([]string{"modules/showcase/index.ts", "modules/lib/x.go"},
|
||||
[]string{"modules/showcase"}, true), ""},
|
||||
{"the root is never a module directory", merge([]string{"tsconfig.json"}, []string{"", "/", "."}, true), ""},
|
||||
{"not said: still no build reads it", merge(showcase, []string{"modules/showcase"}, false), ""},
|
||||
{"an old announcer saying nothing", merge(showcase, nil, false), ""},
|
||||
{"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the announcer says reaches the controller as it is sent: the two fields, by their names on the
|
||||
// wire, and an older announcement without them reads as not said.
|
||||
func TestTheAnnouncerSaysWhichDirectoriesAreModules(t *testing.T) {
|
||||
var m link.SourceMoved
|
||||
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc",`+
|
||||
`"paths":["modules/showcase/index.ts"],"module_dirs":["modules/showcase"],"module_dirs_said":true}`), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.ModuleDirsSaid || !reflect.DeepEqual(m.ModuleDirs, []string{"modules/showcase"}) {
|
||||
t.Fatalf("the announcer's word was lost: %+v", m)
|
||||
}
|
||||
var old link.SourceMoved
|
||||
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc","paths":["x"]}`), &old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if old.ModuleDirsSaid || old.ModuleDirs != nil {
|
||||
t.Fatalf("an older announcement says nothing about module directories: %+v", old)
|
||||
}
|
||||
}
|
||||
|
||||
+386
-128
@@ -8,16 +8,16 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// working out what one machine should be.
|
||||
@@ -90,6 +90,12 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
if err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
// Where each of its consumers of a provision that keeps data was last sent (novox/hq ADR 0232):
|
||||
// a resolution that would answer one from anywhere else keeps it there, and says so.
|
||||
world.Bound, err = inv.BindingsFor(ctx, nodeName)
|
||||
if err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
@@ -129,6 +135,7 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
// a mesh-wide gatherer may pass over — see notResolvable.
|
||||
return catalogue.Resolution{}, nil, notResolvable{err}
|
||||
}
|
||||
logUnheld(nodeName, resolved.Unheld)
|
||||
|
||||
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
|
||||
// password a provider is told to create is the one its consumer was given — and sealed to
|
||||
@@ -371,13 +378,19 @@ func declarationFor(ctx context.Context, open *stores, node string,
|
||||
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
|
||||
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
|
||||
// machine "waiting" means — the read needs no number to be right about that.
|
||||
type Choosing bool
|
||||
type Choosing int
|
||||
|
||||
const (
|
||||
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
||||
Allocating Choosing = true
|
||||
// Reading is every question: what is assigned is used, and nothing is created.
|
||||
Reading Choosing = false
|
||||
Reading Choosing = iota
|
||||
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
||||
Allocating
|
||||
// Foreseeing is Reading, asked ahead of a send (the self-check's D1, novox/hq issue 275): nothing
|
||||
// is created, and an own secret the next send WOULD make is composed with a stand-in and named
|
||||
// (sendable.foreseen) rather than failing the composition — while one the send would be refused
|
||||
// (a bus credential nobody issued, a given secret under an old key) is refused here as it would
|
||||
// be there. Never sent: the stand-in is not a sealed value.
|
||||
Foreseeing
|
||||
)
|
||||
|
||||
func declarationWith(ctx context.Context, open *stores, node string,
|
||||
@@ -396,9 +409,75 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
return sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
if !known {
|
||||
before = nil
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// boundToData is every binding of this machine's consumers to a provision that keeps their data
|
||||
// (novox/hq ADR 0232), as a send records it. Not a consumer left out of the declaration: the machine
|
||||
// is not told anything new about it, so nothing about where it is bound has been sent.
|
||||
func boundToData(plan catalogue.Resolution, leftOut map[string]string) []inventory.Binding {
|
||||
var out []inventory.Binding
|
||||
seen := map[[2]string]bool{}
|
||||
for _, n := range plan.Needs {
|
||||
if !n.KeepsData || n.ByRecord || n.Module == "" {
|
||||
continue
|
||||
}
|
||||
if _, left := leftOut[n.For]; left {
|
||||
continue
|
||||
}
|
||||
key := [2]string{n.For, n.Name}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, inventory.Binding{Machine: plan.Node, Consumer: n.For, Provision: n.Name,
|
||||
Provider: catalogue.Chosen{Node: n.From, Module: n.Module}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
|
||||
// issue 259): the module's current build for each module in it, and for a module left out of it
|
||||
// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
|
||||
// that is not known. Never nil, so a send through here always records what it knows.
|
||||
func carriedBuilds(modules []string, leftOut map[string]string, current map[string]inventory.CurrentBuild,
|
||||
before map[string]string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, m := range modules {
|
||||
if _, left := leftOut[m]; left {
|
||||
if was, kept := before[m]; kept {
|
||||
out[m] = was
|
||||
}
|
||||
continue
|
||||
}
|
||||
out[m] = current[m].Commit
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
|
||||
@@ -423,6 +502,39 @@ func reportLeftOut(node string, declared sendable) {
|
||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||
node, m, declared.leftOutWhy[m])
|
||||
}
|
||||
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||
// 0225): the machine is sent everything else, and the consumer is named.
|
||||
for _, o := range declared.withheld {
|
||||
fmt.Printf("%s: %s\n", node, o)
|
||||
}
|
||||
// And whom it no longer serves because they are bound elsewhere (novox/hq issue 274).
|
||||
for _, u := range declared.unbound {
|
||||
fmt.Printf("%s: %s\n", node, u)
|
||||
}
|
||||
}
|
||||
|
||||
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
|
||||
//
|
||||
// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is
|
||||
// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted
|
||||
// by `module issue` and sealed by it; a push that made a random one would deliver a file the process
|
||||
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
|
||||
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
|
||||
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
|
||||
if name != "broker" {
|
||||
return nil
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return err
|
||||
} else if !minted {
|
||||
return fmt.Errorf(
|
||||
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
||||
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
||||
"`module issue %s --node %s`, then push again",
|
||||
module, node, user, module, node)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
@@ -430,7 +542,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||
inv := open.inventory
|
||||
grants, err := grantsFor(ctx, open, node)
|
||||
grants, withheld, unbound, err := grantsFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
@@ -442,7 +554,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// consumer is told are all derived from it.
|
||||
// What this machine was already given, for a composition that may not allocate.
|
||||
already := map[string]map[int]int{}
|
||||
if choosing == Reading {
|
||||
if choosing != Allocating {
|
||||
held, err := inv.PortsFor(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
@@ -528,6 +640,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// And each module's own secrets — a superuser password, an administrator, an account. Made
|
||||
// per node, so a module running on three machines has three.
|
||||
needed := map[string]map[string]string{}
|
||||
foreseen := map[string]map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
for name := range m.OwnSecrets {
|
||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||
@@ -535,10 +648,29 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
var sealed string
|
||||
var err error
|
||||
if choosing == Allocating {
|
||||
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
|
||||
} else {
|
||||
var held bool
|
||||
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
|
||||
if err == nil && !held && choosing == Foreseeing {
|
||||
// Asked ahead of the send: what the send would do about it, by the send's own
|
||||
// rules. Made by it — a stand-in, named; refused by it — refused here, the same
|
||||
// words (novox/hq issue 275).
|
||||
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if foreseen[m.Module] == nil {
|
||||
foreseen[m.Module] = map[string]bool{}
|
||||
}
|
||||
foreseen[m.Module][name] = true
|
||||
continue
|
||||
}
|
||||
if err == nil && !held {
|
||||
// Never issued, so this machine cannot be running it. Left out rather than
|
||||
// invented: an empty string here would compose a declaration that differs
|
||||
@@ -630,43 +762,38 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
}
|
||||
|
||||
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
||||
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
|
||||
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
|
||||
// to serve and knows nothing about what they mean.
|
||||
// Kept apart from the machines, because a fact about the machines must not be handed the names
|
||||
// the mesh merely serves (novox/hq 04-ISSUES/111).
|
||||
// **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal
|
||||
// domain, `<node>.internal`, and every route on it is a name under that domain (ADR 0151), which
|
||||
// the resolver answers with one wildcard per machine — so no route needs a line of its own. A
|
||||
// node's public domains are the operator's and public DNS answers them; the mesh gives no private
|
||||
// answer for any of them. The roster once carried every routed name, public ones included, and a
|
||||
// resolver that also serves a LAN handed a phone a tunnel address for the mail server.
|
||||
// `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111).
|
||||
machines := make(map[string]string, len(names))
|
||||
for name, at := range names {
|
||||
machines[name] = at
|
||||
}
|
||||
routes, err := routeNamesInTheMesh(ctx, open)
|
||||
|
||||
// And every zone a module in the mesh answers itself (novox/hq ADR 0199), for the mesh's resolver
|
||||
// to forward.
|
||||
zones, err := zonesInTheMesh(ctx, open)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
for name, at := range routes {
|
||||
names[name] = at
|
||||
|
||||
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
|
||||
// lists every holder of the mesh's resolver.
|
||||
replicas, err := replicatedHolders(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// The ports the mesh itself needs open, which no module declares. Read from the broker this
|
||||
// control plane was told about rather than written down twice: the address a node is handed in
|
||||
// its token and the port its machine must accept on are the same fact.
|
||||
//
|
||||
// **Only on the node that listens on it** (novox/hq issue: the broker opening leaked onto
|
||||
// every node). The opening exists to WIDEN the broker's port to from-anywhere — a machine
|
||||
// enrolling is not on the mesh yet, so the broker's own `from: mesh` listen would refuse its
|
||||
// first dial. That widening belongs on the broker's host and nowhere else: a node that only
|
||||
// dials out needs no incoming rule, and an opening for a port nothing here listens on is a
|
||||
// from-anywhere hole for a dead port. So the foundation port is kept only when a module
|
||||
// resolved onto THIS node actually listens on it.
|
||||
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
||||
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
||||
// before it had an address on the private network. A machine joins through the tunnel now, and
|
||||
// every link to the bus crosses it, so its reach is what the `nats` module declares: the mesh.
|
||||
// Nothing the mesh itself needs is opened beyond what a module declares.
|
||||
var foundation []int
|
||||
if b, err := broker.FromEnvironment(); err == nil {
|
||||
if _, port, err := net.SplitHostPort(b.Address); err == nil {
|
||||
if n, err := strconv.Atoi(port); err == nil {
|
||||
foundation = foundationPortsFor(n, plan.Modules)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
|
||||
// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The
|
||||
@@ -726,38 +853,36 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
|
||||
// 0222): the artifact store as this network reaches it, which the container runtime is told to
|
||||
// trust (ADR 0082). The same address composed into every reference the mesh built.
|
||||
var reach map[string]string
|
||||
if artifactStore != "" {
|
||||
reach = map[string]string{"mesh-artifact-store": artifactStore}
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Machines: machines, Zones: zones, Holders: replicas,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
BusUsers: busUsers,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||
BusUsers: busUsers, Withheld: withheld, Unbound: unbound,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
|
||||
// ADR 0066).
|
||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||
// answering listen is published on there.
|
||||
//
|
||||
// **Mesh-wide, so any container resolves any routed name to its proxy** — including an internal
|
||||
// ACME validator, which cannot complete a challenge for a name it cannot reach. A routed name is
|
||||
// composed on the consumer's node (from its label and that node's public domain) and served by the
|
||||
// node answering the consumer's route requirement; this gathers both.
|
||||
//
|
||||
// It reads route names off resolutions rather than a table because there is no table: a route is a
|
||||
// contribution, computed from what each node runs. Name-agnostic — a contribution counts as a
|
||||
// routed name only because it carried a label the mesh composed, never because the mesh knows what
|
||||
// "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
|
||||
// the rest their names.
|
||||
//
|
||||
// **A node that could not be READ is a different matter and is raised.** Skipping one states, to
|
||||
// every machine at once, that its names do not exist — and since the roster is part of every
|
||||
// container's identity, that withdraws them and replaces every container (novox/hq 04-ISSUES/152,
|
||||
// 151). So every failure here says which machine and which read, because the alternative is a
|
||||
// mesh-wide refusal with nothing named in it.
|
||||
func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
|
||||
// Read across every machine's resolution, as the roster once read routed names: a node whose set does
|
||||
// not compose declares nothing and is passed over, so one broken machine does not cost the rest their
|
||||
// zones; a store that cannot be read is raised, naming the machine, because returning the zones
|
||||
// without it would withdraw them from the resolver as if the operator had (novox/hq 04-ISSUES/152).
|
||||
// What the mesh refuses about the zones together — one declared twice, one shadowing the mesh's
|
||||
// suffix or a node's public domain — is refused here, by name.
|
||||
func zonesInTheMesh(ctx context.Context, open *stores) ([]catalogue.ZoneAt, error) {
|
||||
inv := open.inventory
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
@@ -769,43 +894,47 @@ func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string,
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
|
||||
}
|
||||
|
||||
// Every machine's resolution first, then the names across them at once: which node serves a
|
||||
// name is a question about the graph — the consumer on one machine, the provider on another —
|
||||
// and answered wrongly by looking at one contribution at a time (novox/hq issue 178).
|
||||
plans := map[string]catalogue.Resolution{}
|
||||
settings := map[string]catalogue.SettingsBy{}
|
||||
var zones []catalogue.ZoneAt
|
||||
var public []string
|
||||
for _, n := range nodes {
|
||||
plan, layers, err := planFor(ctx, open, n.Name)
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
switch {
|
||||
case unresolvable(err):
|
||||
// Their set does not compose, so they serve no names. Passed over, so one machine's
|
||||
// broken set does not cost the rest theirs.
|
||||
continue
|
||||
case err != nil:
|
||||
// The mesh could not be asked. Returning the roster without this machine's names would
|
||||
// state that they do not exist — to every machine, and indistinguishably from the
|
||||
// operator having withdrawn them (novox/hq 04-ISSUES/152).
|
||||
return nil, fmt.Errorf("the names %s serves cannot be read: %w", n.Name, err)
|
||||
return nil, fmt.Errorf("the zones %s answers cannot be read: %w", n.Name, err)
|
||||
}
|
||||
plans[n.Name], settings[n.Name] = plan, layers
|
||||
}
|
||||
served, err := catalogue.NamesServed(plans, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for name, node := range served {
|
||||
if at := address[node]; at != "" {
|
||||
out[name] = at
|
||||
if plan.PublicDomain != "" {
|
||||
public = append(public, plan.PublicDomain)
|
||||
}
|
||||
for _, m := range plan.Modules {
|
||||
if m.Zone == nil {
|
||||
continue
|
||||
}
|
||||
at := address[n.Name]
|
||||
if at == "" {
|
||||
// Not on the private network yet: nothing could reach its answerer.
|
||||
continue
|
||||
}
|
||||
published, layers, err := portsGivenOn(ctx, inv, n.Name, m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the zone %s declares on %s cannot be read: %w", m.Module, n.Name, err)
|
||||
}
|
||||
z, err := catalogue.ZoneOn(m, layers, published, n.Name, at)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zones = append(zones, *z)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
if problems := catalogue.ZonesProblems(zones, overlay.Suffix(), public); len(problems) > 0 {
|
||||
return nil, fmt.Errorf("the mesh's zones cannot be forwarded:\n - %s", strings.Join(problems, "\n - "))
|
||||
}
|
||||
return zones, nil
|
||||
}
|
||||
|
||||
// certificateFor is what the mesh certifies about one machine's internal name.
|
||||
@@ -875,28 +1004,45 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
|
||||
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
||||
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
||||
// the mesh hands over something it cannot itself use.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
||||
//
|
||||
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
|
||||
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
|
||||
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
|
||||
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
|
||||
//
|
||||
// **A provider is granted exactly the consumers whose own resolution binds them to it** (novox/hq
|
||||
// issue 274). The pair credentials on record say only whom this node was ever asked by: after a
|
||||
// consumer of a provision that keeps its data was moved and pinned back (issue 273, ADR 0232), the
|
||||
// credential from the provider it left was still on record, so that provider went on being asked for
|
||||
// five databases nobody used and never retired them. A credential whose consumer is bound elsewhere
|
||||
// is withdrawn here like one nobody asks for — the provider retires it and keeps its data (ADR 0230)
|
||||
// — and returned beside the grants, for plan and push to say. It stays on record: it is the key to
|
||||
// the login the provider keeps until `cleanup delete`, and to that data should a person pin it back.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) (
|
||||
[]catalogue.Grant, []catalogue.Overflow, []catalogue.Unbound, error) {
|
||||
inv := open.inventory
|
||||
issued, err := inv.SecretsFrom(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
||||
// the mesh names machines.
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
||||
// from a record beside it. A provider told to create a password and not what to create it for
|
||||
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
||||
out := make([]catalogue.Grant, 0, len(issued))
|
||||
var withheld []catalogue.Overflow
|
||||
var unbound []catalogue.Unbound
|
||||
for _, s := range issued {
|
||||
plan, settings, err := planFor(ctx, open, s.Consumer)
|
||||
switch {
|
||||
@@ -909,11 +1055,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
||||
// nothing — and withholding a grant on that reading takes a consumer's access away
|
||||
// (novox/hq 04-ISSUES/152).
|
||||
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
return nil, nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
}
|
||||
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// A port in there is the consumer's software port until this. The consumer is on another
|
||||
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
||||
@@ -921,7 +1067,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
||||
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
||||
from := s.ConsumerModule
|
||||
@@ -931,10 +1077,20 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// working for ever after its consumer went away.
|
||||
from = ""
|
||||
}
|
||||
if bound := plan.BindsFrom(s.Name, s.ConsumerModule, s.Local); from != "" && !slices.Contains(bound, node) {
|
||||
// It still asks, and not of this node: its resolution — the same one read above, so an
|
||||
// unreadable one is the error above and never an empty answer here (issue 152) — binds
|
||||
// this credential to another provider, or under this local name to none. Withdrawn
|
||||
// exactly as a credential nobody asks for, and said.
|
||||
from = ""
|
||||
unbound = append(unbound, catalogue.Unbound{Provision: s.Name, Provider: node,
|
||||
Consumer: s.Consumer, Module: s.ConsumerModule, Local: s.Local, BoundTo: bound})
|
||||
}
|
||||
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
||||
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
||||
// remedy a short slug rather than a login a provider silently shortened.
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
|
||||
// this provision, as the consumer's resolution states it from the provider's offer (ADR
|
||||
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
|
||||
// provider silently shortened — and rather than this whole machine refused for it.
|
||||
slug := ""
|
||||
for _, mm := range plan.Modules {
|
||||
if mm.Module == s.ConsumerModule {
|
||||
@@ -943,15 +1099,32 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
}
|
||||
}
|
||||
if from != "" {
|
||||
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
||||
return nil, err
|
||||
bound := boundOfGrant(plan, s, node)
|
||||
source := catalogue.IdentitySource(slug, s.ConsumerModule)
|
||||
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
|
||||
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
|
||||
Consumer: s.Consumer, Module: s.ConsumerModule,
|
||||
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, catalogue.Grant{
|
||||
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
||||
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
||||
}
|
||||
return out, nil
|
||||
return out, withheld, unbound, nil
|
||||
}
|
||||
|
||||
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
|
||||
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
|
||||
// than to none: what the provider keeps of it is not known here.
|
||||
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
|
||||
for _, n := range consumer.Needs {
|
||||
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
|
||||
return n.Identity
|
||||
}
|
||||
}
|
||||
return catalogue.DefaultIdentityBound
|
||||
}
|
||||
|
||||
// listensLines is what a person is told about what this module would open, and why — the same
|
||||
@@ -1027,6 +1200,11 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
left := plan.LeftOut(settings, record.Adopted)
|
||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||
}
|
||||
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||
for _, o := range plan.Overflowing() {
|
||||
fmt.Printf("%s: %s\n", args[0], o)
|
||||
}
|
||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||
// stored before its definition moved from under it.
|
||||
for _, m := range plan.Modules {
|
||||
@@ -1340,6 +1518,20 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||
//
|
||||
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
|
||||
// resource of that module, so the question is whether it is here.
|
||||
list, missing, err := busUserList(ctx, inv, onThisNode)
|
||||
if len(missing) > 0 {
|
||||
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
return list, err
|
||||
}
|
||||
|
||||
// busUserList is composeBusUsers without saying anything: the list, and the users left out of it
|
||||
// for want of a credential. Asked on every send to decide whether the machine holding the bus must
|
||||
// go first (novox/hq issue 249), where saying the same missing users each time would bury them.
|
||||
func busUserList(ctx context.Context, inv *inventory.Inventory,
|
||||
onThisNode []catalogue.Manifest) (string, []string, error) {
|
||||
holdsTheBus := false
|
||||
for _, m := range onThisNode {
|
||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||
@@ -1347,54 +1539,33 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||
}
|
||||
}
|
||||
if !holdsTheBus {
|
||||
return "", nil
|
||||
return "", nil, nil
|
||||
}
|
||||
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
hashes := make(map[string]string, len(kept))
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
filled, missing := broker.WithPasswords(users, hashes)
|
||||
if len(missing) > 0 {
|
||||
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
if len(filled) == 0 {
|
||||
return "", fmt.Errorf(
|
||||
return "", missing, fmt.Errorf(
|
||||
"this machine runs the bus and not one user has a credential, so the composed list " +
|
||||
"would refuse every connection in the mesh")
|
||||
}
|
||||
return broker.ComposeAccounts(filled)
|
||||
}
|
||||
|
||||
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
|
||||
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
|
||||
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
|
||||
// not on the mesh yet and its first dial would be refused. That widening belongs on the broker's
|
||||
// host alone: a node that only dials out needs no incoming rule, and an opening for a port
|
||||
// nothing here listens on is a from-anywhere hole for a dead port.
|
||||
func foundationPortsFor(brokerPort int, modules []catalogue.Manifest) []int {
|
||||
for _, m := range modules {
|
||||
for _, l := range m.Listens {
|
||||
if l.Port == brokerPort {
|
||||
return []int{brokerPort}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
list, err := broker.ComposeAccounts(filled)
|
||||
return list, missing, err
|
||||
}
|
||||
|
||||
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
||||
@@ -1416,3 +1587,90 @@ func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.C
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
|
||||
// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
||||
//
|
||||
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
||||
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
||||
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
||||
// burying the line about the node it acted on. A command says what concerns its own act instead
|
||||
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
||||
var (
|
||||
unheldLogged = map[string]string{}
|
||||
unheldLoggedMu sync.Mutex
|
||||
logUnheldChanges bool
|
||||
)
|
||||
|
||||
// logUnheld logs a node's unmet seat dependencies when they differ from what was last logged for
|
||||
// it, including when they become none — in the serving controller, and nowhere else.
|
||||
func logUnheld(node string, unheld []catalogue.Unheld) {
|
||||
if !logUnheldChanges {
|
||||
return
|
||||
}
|
||||
lines := make([]string, 0, len(unheld))
|
||||
for _, u := range unheld {
|
||||
lines = append(lines, u.String())
|
||||
}
|
||||
now := strings.Join(lines, "\n")
|
||||
unheldLoggedMu.Lock()
|
||||
before, seen := unheldLogged[node]
|
||||
unheldLogged[node] = now
|
||||
unheldLoggedMu.Unlock()
|
||||
if (seen && before == now) || (!seen && now == "") {
|
||||
return
|
||||
}
|
||||
if now == "" {
|
||||
fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
||||
node, len(lines), strings.Join(lines, "\n "))
|
||||
}
|
||||
|
||||
// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over
|
||||
// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not —
|
||||
// which includes every one of a module just assigned — and each now met that was not. Nothing about
|
||||
// any other node, and nothing that was already true before the act.
|
||||
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
|
||||
judge := func(names []string) map[string]catalogue.Unheld {
|
||||
var set []catalogue.Manifest
|
||||
for _, n := range names {
|
||||
if m, known := shelf[n]; known {
|
||||
set = append(set, m)
|
||||
}
|
||||
}
|
||||
out := map[string]catalogue.Unheld{}
|
||||
for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
|
||||
out[u.Module+" "+u.Seat] = u
|
||||
}
|
||||
return out
|
||||
}
|
||||
was, now := judge(before), judge(after)
|
||||
var lines []string
|
||||
for _, k := range sortedNames(now) {
|
||||
if _, already := was[k]; !already {
|
||||
lines = append(lines, "but "+now[k].String())
|
||||
}
|
||||
}
|
||||
for _, k := range sortedNames(was) {
|
||||
if _, still := now[k]; still {
|
||||
continue
|
||||
}
|
||||
u := was[k]
|
||||
if !slices.Contains(after, u.Module) {
|
||||
continue // went with its module, which says nothing about the seat
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("and %s on %s now has %s held", u.Module, node, u.Seat))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func sortedNames[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A plan follows what is done to the build queue (novox/hq ADR 0219).
|
||||
//
|
||||
// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
|
||||
//
|
||||
// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
|
||||
// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
|
||||
// sends a reader looking for a fault that is a decision. It says what it waits on instead.
|
||||
// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
|
||||
// failed plan's failed modules and the plan goes on from that tier as if they had built the
|
||||
// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
|
||||
// beside it — beside it, the plan would either ask it again or stay failed on an outcome the
|
||||
// rebuild has already replaced.
|
||||
|
||||
// pauseView is whether the build seat takes work: the holders that said they are paused, and
|
||||
// whether that is every holder.
|
||||
type pauseView struct {
|
||||
Nodes []string
|
||||
All bool
|
||||
}
|
||||
|
||||
// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
|
||||
// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
|
||||
func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
|
||||
if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
|
||||
return "", false
|
||||
}
|
||||
var asked *time.Time
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
|
||||
if asked == nil || s.AskedAt.Before(*asked) {
|
||||
asked = s.AskedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
if asked == nil {
|
||||
return "", false
|
||||
}
|
||||
waited := now.Sub(*asked)
|
||||
said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
|
||||
strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
|
||||
// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
|
||||
// longer than a day is named.
|
||||
if waited > pausedTooLong {
|
||||
said += " — PAUSED OVER A DAY: `resume` takes builds again"
|
||||
}
|
||||
return said, true
|
||||
}
|
||||
|
||||
// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
|
||||
const pausedTooLong = 24 * time.Hour
|
||||
|
||||
// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
|
||||
// where the seat being paused changes what a plan says.
|
||||
func awaitsABuild(plans []inventory.Plan) bool {
|
||||
for _, p := range plans {
|
||||
if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
|
||||
// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
|
||||
// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
|
||||
// reads as late, which is what it said before this existed.
|
||||
func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
|
||||
if !awaitsABuild(plans) {
|
||||
return pauseView{}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
seat := buildSeatAmong(entries)
|
||||
holders := holdersAmong(entries, seat)
|
||||
if len(holders) == 0 {
|
||||
return pauseView{}
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
}
|
||||
defer js.Close()
|
||||
said, err := link.PausedSaid(js, seat, holders)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
}
|
||||
return pauseOf(holders, said)
|
||||
}
|
||||
|
||||
// pauseOf is the view from what each holder said.
|
||||
func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
|
||||
var v pauseView
|
||||
for _, n := range holders {
|
||||
if said[n].Paused {
|
||||
v.Nodes = append(v.Nodes, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(v.Nodes)
|
||||
v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
|
||||
return v
|
||||
}
|
||||
|
||||
// failedIn is the modules of a plan's current tier that failed to build, sorted.
|
||||
func failedIn(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
|
||||
// that holds what this one held now (issue 254, ADR 0218).
|
||||
func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
return q, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
switch {
|
||||
case p.State == inventory.PlanDone:
|
||||
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||
case p.State == inventory.PlanSuperseded:
|
||||
return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
|
||||
case p.Open():
|
||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||
}
|
||||
if len(failedIn(p)) > 0 {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
stopped := stoppedRollouts(p)
|
||||
if len(stopped) == 0 {
|
||||
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
|
||||
p.Tier, p.ID, p.Note)
|
||||
}
|
||||
// **A build that failed its gate is not sent again** (novox/hq ADR 0236): it was put back on its first
|
||||
// machine, and retrying would judge the build the mesh put back, or send the failed one by hand.
|
||||
for _, m := range stopped {
|
||||
if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed && !noVerdictOnItsBuild(g) {
|
||||
return fmt.Errorf("%s failed its gate on %s (%s) and was put back: a build that failed its gate is not "+
|
||||
"sent again — a newer merge, or `rebuild %s`, makes a new build, judged at the gate again",
|
||||
m, strings.Join(g.Machines, ", "), g.Why, m)
|
||||
}
|
||||
}
|
||||
// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
|
||||
// sent — a newer build, and sending this one again would put the older build back on its machines.
|
||||
for _, m := range stopped {
|
||||
if q, found := newerPlanFor(m, p, plans); found {
|
||||
return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
|
||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
|
||||
// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
|
||||
func stoppedRollouts(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
|
||||
len(s.First) > 0 && s.Why != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
|
||||
func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
for _, tier := range q.Tiers {
|
||||
for _, m := range tier {
|
||||
if m == module {
|
||||
return q, true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
|
||||
// variable so a test of a retried rollout needs no machine.
|
||||
var sendRollout = sendToEach
|
||||
|
||||
// resumed sets a failed plan building again once nothing in its tier is failed.
|
||||
func resumed(p *inventory.Plan, why string) {
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = why
|
||||
}
|
||||
}
|
||||
|
||||
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||
// that tier: what follows is the plan going on as if they had built the first time.
|
||||
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans = append(plans, recent...)
|
||||
if err := retryRefusal(p, plans); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
|
||||
return retryTierWhole(ctx, open, &p, again)
|
||||
}
|
||||
if len(failedIn(p)) == 0 {
|
||||
return retryRollouts(ctx, open, &p)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
failed := failedIn(p)
|
||||
var asked []string
|
||||
for _, m := range failed {
|
||||
askModule(ctx, &p, m, byName)
|
||||
if s := p.Modules[m]; s.State == "asked" {
|
||||
asked = append(asked, m+" as "+s.Build)
|
||||
}
|
||||
}
|
||||
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.State != inventory.PlanBuilding {
|
||||
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
|
||||
p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
|
||||
}
|
||||
|
||||
// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
|
||||
// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
|
||||
func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
defer release()
|
||||
openPlans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 20)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
p, found := planHolding(module, openPlans, recent)
|
||||
if !found {
|
||||
return false, "", nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
was := p.State
|
||||
askModule(ctx, &p, module, byName)
|
||||
s := p.Modules[module]
|
||||
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if s.State != "asked" {
|
||||
return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
|
||||
}
|
||||
said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
|
||||
s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
|
||||
switch {
|
||||
case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
|
||||
said += "; the plan had failed and builds again from this tier"
|
||||
case was == inventory.PlanFailed:
|
||||
said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
|
||||
}
|
||||
return true, said, nil
|
||||
}
|
||||
|
||||
// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
|
||||
// yet built, else the newest failed plan whose current tier does, and that no open plan of its
|
||||
// repository supersedes.
|
||||
func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
|
||||
holds := func(p inventory.Plan) bool {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if m == module {
|
||||
s := p.Modules[m]
|
||||
return s == nil || s.State != "built"
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, p := range openPlans {
|
||||
if holds(p) {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
sorted := append([]inventory.Plan(nil), recent...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
|
||||
for _, p := range sorted {
|
||||
if p.State != inventory.PlanFailed || !holds(p) {
|
||||
continue
|
||||
}
|
||||
if _, superseded := newerOpenPlan(p, openPlans); superseded {
|
||||
continue
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
|
||||
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
|
||||
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
|
||||
func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
|
||||
// Every machine once, for all the modules stopped there (novox/hq issue 281).
|
||||
stopped := stoppedRollouts(*p)
|
||||
var machines []string
|
||||
for _, m := range stopped {
|
||||
for _, n := range p.Modules[m].First {
|
||||
if !slices.Contains(machines, n) {
|
||||
machines = append(machines, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(machines)
|
||||
sent, err := sendRollout(ctx, open, machines)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
|
||||
strings.Join(stopped, ", "), strings.Join(machines, ", "), p.ID, err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
var said []string
|
||||
for _, m := range stopped {
|
||||
s := p.Modules[m]
|
||||
var again []string
|
||||
for _, n := range sent {
|
||||
if slices.Contains(s.First, n) {
|
||||
again = append(again, n)
|
||||
}
|
||||
}
|
||||
s.First, s.FirstAt, s.Why = again, &now, ""
|
||||
said = append(said, m+" to "+strings.Join(again, ", "))
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
|
||||
"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
|
||||
}
|
||||
|
||||
// noVerdictOnItsBuild says a failed gate said nothing about the module's build (novox/hq issue 281): its
|
||||
// send changed nothing of the module there — the machine already ran that build, carried there by an
|
||||
// earlier send of the same tier, or one identical to it. Such a module was blamed for its machine.
|
||||
func noVerdictOnItsBuild(g *inventory.PlanGate) bool {
|
||||
return g.Rollback == gateUnchanged || (g.From != "" && sameCommit(g.From, g.To))
|
||||
}
|
||||
|
||||
// unjudgedAtGate is the modules of a plan's current tier stopped at a gate that was no verdict on their
|
||||
// build, sorted.
|
||||
func unjudgedAtGate(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.Gate != nil && s.Gate.Verdict == inventory.GateFailed && noVerdictOnItsBuild(s.Gate) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// retryTierWhole retries a plan stopped at a gate that judged no build of the module it stopped on
|
||||
// (issue 281): that module is asked again under a new id — its old build may be marked failed, and a new
|
||||
// verdict is what takes the gate's condition away — and every module of the tier sent first and never
|
||||
// passed is sent again, the tier whole, one send per machine, judged again. What passed stays passed.
|
||||
func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again []string) (string, error) {
|
||||
inv := open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
var resent []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
s := p.Modules[m]
|
||||
if s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(again, m) ||
|
||||
(s.Gate != nil && s.Gate.Verdict == inventory.GatePassed) {
|
||||
continue
|
||||
}
|
||||
sendAgain(s)
|
||||
resent = append(resent, m)
|
||||
}
|
||||
var asked []string
|
||||
for _, m := range again {
|
||||
sendAgain(p.Modules[m])
|
||||
askModule(ctx, p, m, byName)
|
||||
if s := p.Modules[m]; s.State == "asked" {
|
||||
asked = append(asked, m+" as "+s.Build)
|
||||
}
|
||||
}
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
|
||||
strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.State != inventory.PlanBuilding {
|
||||
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; %s sent again with the tier, one send per machine, "+
|
||||
"judged again", p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", "), orNone(strings.Join(resent, ", "))), nil
|
||||
}
|
||||
|
||||
// sendAgain forgets a module's first send, so its plan sends it again.
|
||||
func sendAgain(s *inventory.PlanModule) {
|
||||
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 213: for the moment a machine hands its controller over, the container and the
|
||||
// process both run the plan timer on one store. Only the one holding the plans moves them; the other
|
||||
// leaves them alone, and moves them once they are let go.
|
||||
func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now().UTC()
|
||||
// Every tier done: the next step is the plan's last, and needs nothing but the store.
|
||||
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
|
||||
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The other controller: its own connections to the same store, holding the plans.
|
||||
other, err := inventory.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(other.Close)
|
||||
release, err := other.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(release) // before the close above: a pool waits for a connection still held
|
||||
|
||||
advancePlans(ctx, open)
|
||||
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() {
|
||||
t.Fatalf("a controller moved a plan another held: %+v %v", p, err)
|
||||
}
|
||||
|
||||
release()
|
||||
advancePlans(ctx, open)
|
||||
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone {
|
||||
t.Fatalf("the plan did not move once it was let go: %+v %v", p, err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+740
-217
File diff suppressed because it is too large
Load Diff
@@ -17,7 +17,7 @@ import (
|
||||
// the wrong machine no longer refuses the whole node), applied one level up.
|
||||
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
sending, refusals := composeEach(
|
||||
[]string{"anchor", "home-server", "laptop"},
|
||||
[]string{"anchor", "home-server", "laptop"}, numbered(),
|
||||
func(node string) (sendable, error) {
|
||||
if node == "anchor" {
|
||||
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||
@@ -43,7 +43,7 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||
sending, refusals := composeEach([]string{"spare"},
|
||||
sending, refusals := composeEach([]string{"spare"}, numbered(),
|
||||
func(string) (sendable, error) { return sendable{}, nil })
|
||||
if len(sending) != 1 || len(refusals) != 0 {
|
||||
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||
@@ -74,3 +74,9 @@ func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||
func numbered() func(string) (order, error) {
|
||||
var n int64
|
||||
return func(string) (order, error) { n++; return order{sequence: n}, nil }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,709 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
//
|
||||
// Seen whole — what waits, what runs where and for how long, what was handed out as often as it
|
||||
// may be and never settled — and changed through the controller: an ask cancelled, the queue
|
||||
// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's
|
||||
// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`).
|
||||
//
|
||||
// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that
|
||||
// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
|
||||
// rather than waiting for ever on an answer nobody will give.
|
||||
|
||||
// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
|
||||
// than the holder's worst case (its two passes removing containers and its wait between, 50s).
|
||||
const (
|
||||
holderAsks = 15 * time.Second
|
||||
killAnswer = 75 * time.Second
|
||||
)
|
||||
|
||||
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
|
||||
func dialTheBus() (*broker.JetStream, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
return js, nil
|
||||
}
|
||||
|
||||
// queueCommand prints every ask in the build seat's work queue.
|
||||
func queueCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("queue", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "the queue as JSON")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(q, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
fmt.Print(queueText(q, time.Now()))
|
||||
return nil
|
||||
}
|
||||
|
||||
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
|
||||
// Never what an ask carries as `held` — that is every artifact the mesh has built.
|
||||
func queueText(q link.Queue, now time.Time) string {
|
||||
var b strings.Builder
|
||||
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||
fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead))
|
||||
ago := func(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "asked at an unknown time"
|
||||
}
|
||||
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
|
||||
}
|
||||
what := func(a link.QueuedAsk) string {
|
||||
s := a.Repository
|
||||
if a.Path != "" {
|
||||
s += " at " + a.Path
|
||||
}
|
||||
if a.Ref != "" {
|
||||
s += " on " + a.Ref
|
||||
}
|
||||
return s
|
||||
}
|
||||
if len(running) > 0 {
|
||||
fmt.Fprintln(&b, "\nin flight:")
|
||||
for _, a := range running {
|
||||
where := "taken, not yet said where"
|
||||
switch {
|
||||
case a.On != "":
|
||||
where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
fmt.Fprintln(&b, "\nwaiting:")
|
||||
for _, a := range waiting {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(dead) > 0 {
|
||||
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
|
||||
for _, a := range dead {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(q.Asks) == 0:
|
||||
fmt.Fprintln(&b, "nothing is asked of it")
|
||||
default:
|
||||
fmt.Fprintln(&b, "\n`cancel <id>` drops a waiting or dead ask, `clear` every waiting one, `kill <id>` ends one in flight")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// cancelCommand drops one waiting or dead ask.
|
||||
func cancelCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("cancel <build id>")
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ask, found := q.Find(args[0])
|
||||
if !found {
|
||||
return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+
|
||||
"what came of it", args[0], seat)
|
||||
}
|
||||
said, err := cancelAsk(ctx, js, open, seat, ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
|
||||
//
|
||||
// **In this order, and each step for a reason** (novox/hq ADR 0219):
|
||||
// 1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
|
||||
// is running — that is `kill`, on the machine running it;
|
||||
// 2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
|
||||
// 3. for a waiting ask, the worker is read again: one handed out since the queue was read was
|
||||
// taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
|
||||
// read the mark first and ends it as cancelled, or is building it;
|
||||
// 4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
|
||||
// handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
|
||||
// at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
|
||||
// that took it before the mark is building it, and only `kill` ends that;
|
||||
// 5. the message is deleted by its sequence;
|
||||
// 6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
|
||||
func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
|
||||
if ask.State == link.AskInFlight && ask.On != "" {
|
||||
return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
|
||||
"ends the build where it runs", ask.ID, ask.On, ask.ID)
|
||||
}
|
||||
if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
withdraw := func() {
|
||||
if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
|
||||
}
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
switch ask.State {
|
||||
case link.AskWaiting:
|
||||
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||
withdraw()
|
||||
return "", err
|
||||
} else if taken {
|
||||
withdraw()
|
||||
return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
|
||||
"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
|
||||
"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
|
||||
}
|
||||
case link.AskInFlight:
|
||||
if started, err := startedSince(ctx, js, seat, ask); err != nil {
|
||||
withdraw()
|
||||
return "", err
|
||||
} else if started != "" {
|
||||
withdraw()
|
||||
return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
|
||||
"ends it there", ask.ID, started, ask.ID)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
|
||||
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
|
||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||
}
|
||||
recordCancelled(ctx, open, ask)
|
||||
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
|
||||
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
|
||||
}
|
||||
|
||||
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
|
||||
// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
|
||||
var holderLooks = 5 * time.Second
|
||||
|
||||
// startedSince waits out a holder's look at the cancelled set and says the machine that started the
|
||||
// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
|
||||
// a start of a build.
|
||||
func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(holderLooks):
|
||||
}
|
||||
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||
if !ask.AskedAt.IsZero() {
|
||||
since = ask.AskedAt.Add(-time.Minute)
|
||||
}
|
||||
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
s, ok := heard.Started[ask.ID]
|
||||
if !ok || heard.Outcomes[ask.ID] {
|
||||
return "", nil
|
||||
}
|
||||
at, _ := time.Parse(time.RFC3339Nano, s.At)
|
||||
if ask.Started.IsZero() || at.After(ask.Started) {
|
||||
return s.On, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// takenSince is whether the worker has handed out the ask at this sequence.
|
||||
func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
|
||||
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
|
||||
if errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot read the worker of the queue again: %w", err)
|
||||
}
|
||||
return info.Delivered.Stream >= seq, nil
|
||||
}
|
||||
|
||||
// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded,
|
||||
// then the plan that asked for it — by the id it asked with, or by repository and path.
|
||||
func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) {
|
||||
r := ask.Request
|
||||
result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref,
|
||||
Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand}
|
||||
_ = builds{open.inventory, open}.Built(ctx, result)
|
||||
}
|
||||
|
||||
// clearCommand cancels every waiting ask, and with --dead every dead one too.
|
||||
func clearCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("clear", flag.ContinueOnError)
|
||||
dead := set.Bool("dead", false, "the dead asks too")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
said, err := clearQueue(ctx, js, open, seat, *dead)
|
||||
fmt.Print(said)
|
||||
return err
|
||||
}
|
||||
|
||||
// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight.
|
||||
func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) {
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var b strings.Builder
|
||||
cancelled, refused := 0, 0
|
||||
for _, a := range q.Asks {
|
||||
if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) {
|
||||
continue
|
||||
}
|
||||
said, err := cancelAsk(ctx, js, open, seat, a)
|
||||
if err != nil {
|
||||
refused++
|
||||
fmt.Fprintf(&b, " %s: %v\n", a.ID, err)
|
||||
continue
|
||||
}
|
||||
cancelled++
|
||||
fmt.Fprintf(&b, " %s\n", said)
|
||||
}
|
||||
what := "waiting"
|
||||
if dead {
|
||||
what = "waiting and dead"
|
||||
}
|
||||
fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what)
|
||||
if refused > 0 {
|
||||
fmt.Fprintf(&b, ", %d could not be", refused)
|
||||
}
|
||||
fmt.Fprintln(&b)
|
||||
if n := len(q.Of(link.AskInFlight)); n > 0 {
|
||||
fmt.Fprintf(&b, "%d in flight, left running: `kill <id>` ends one where it runs\n", n)
|
||||
}
|
||||
if n := len(q.Of(link.AskDead)); n > 0 && !dead {
|
||||
fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n)
|
||||
}
|
||||
if refused > 0 {
|
||||
return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused)
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// rebuildCommand asks the module's current source again — or, given a build's id, that build's
|
||||
// repository, path and ref — under a new id.
|
||||
func rebuildCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("rebuild <module | build id>")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var source buildSource
|
||||
var path, ref, module string
|
||||
if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
source, module = sourceOfBuild(b, entries)
|
||||
path, ref = b.Path, b.Ref
|
||||
// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
|
||||
// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
|
||||
// that commit out over everything since. Building that commit again is `replay`, which says
|
||||
// what it would do and refuses to register it while anything newer is asked or registered.
|
||||
if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
|
||||
ref = followedBranch(e.Source.Ref)
|
||||
follows := ref
|
||||
if follows == "" {
|
||||
follows = "the repository's default branch"
|
||||
}
|
||||
fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
|
||||
"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
|
||||
}
|
||||
} else if e, known := entryNamed(entries, args[0]); known {
|
||||
// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
|
||||
source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
|
||||
} else {
|
||||
return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
|
||||
}
|
||||
|
||||
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
|
||||
// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
|
||||
if module != "" {
|
||||
joined, said, err := joinAPlan(ctx, open, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if joined {
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
id, err := askABuild(ctx, source, path, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("rebuild asked as %s\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// entryNamed is the catalogue's entry for a module.
|
||||
func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == name {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Entry{}, false
|
||||
}
|
||||
|
||||
// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
|
||||
// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
|
||||
// (ADR 0111) — else the repository as it was cloned.
|
||||
func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) {
|
||||
for _, e := range entries {
|
||||
if (b.Module != "" && e.Manifest.Module == b.Module) ||
|
||||
(b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) {
|
||||
return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module
|
||||
}
|
||||
}
|
||||
return buildSource{Repository: b.Repository}, b.Module
|
||||
}
|
||||
|
||||
// replayCommand asks a recorded build's repository and path again at the commit it built.
|
||||
func replayCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("replay", flag.ContinueOnError)
|
||||
register := set.Bool("register", false, "register what it builds, as any build is")
|
||||
older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("replay <build id> [--register [--older]]")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
b, found, err := inv.BuildByID(ctx, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no build %s is recorded", positionals[0])
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, module := sourceOfBuild(b, entries)
|
||||
var history []inventory.Build
|
||||
if module != "" {
|
||||
if history, err = inv.Builds(ctx, module, 100); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// What is asked of the module and not yet answered, when the replay would be registered.
|
||||
var outstanding []string
|
||||
if *register {
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
|
||||
js.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
|
||||
}
|
||||
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(replaySaid(b, module, id, *register))
|
||||
return nil
|
||||
}
|
||||
|
||||
// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
|
||||
//
|
||||
// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
|
||||
// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
|
||||
// older commit is newer than everything since, and would roll that older commit out as the module's
|
||||
// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
|
||||
// --register says otherwise, and --register is refused when a newer build of a different commit is
|
||||
// registered, unless --older says that is the point.
|
||||
//
|
||||
// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
|
||||
// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
|
||||
// and their builds — made from newer source — would be recorded and never registered (issue 219
|
||||
// orders by ask), the plan waiting on them sending the older commit instead.
|
||||
func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
|
||||
outstanding []string) error {
|
||||
if b.Commit == "" {
|
||||
return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
|
||||
"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
|
||||
}
|
||||
if older && !register {
|
||||
return errors.New("--older only says what --register may do; a dry run registers nothing")
|
||||
}
|
||||
if register && len(outstanding) > 0 {
|
||||
return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
|
||||
"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
|
||||
orNone(module), strings.Join(outstanding, "; "), b.ID)
|
||||
}
|
||||
if !register || older {
|
||||
return nil
|
||||
}
|
||||
for _, h := range history {
|
||||
if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
|
||||
continue
|
||||
}
|
||||
if h.AskedOrAt().After(b.AskedOrAt()) {
|
||||
return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
|
||||
"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
|
||||
"without --register looks at it; --register --older registers it anyway",
|
||||
module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome.
|
||||
func replaySaid(b inventory.Build, module, id string, register bool) string {
|
||||
what := b.Repository
|
||||
if module != "" {
|
||||
what = module
|
||||
}
|
||||
said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id)
|
||||
if !register {
|
||||
return said + "\n a dry run: the outcome is looked at and not taken in — nothing is recorded or " +
|
||||
"registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it"
|
||||
}
|
||||
return said + "\n registered when it is built, as the module's current version — newer than every build " +
|
||||
"asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it"
|
||||
}
|
||||
|
||||
// killCommand finds the machine running a build and asks its holder to end it.
|
||||
func killCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("kill <build id>")
|
||||
}
|
||||
id := args[0]
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||
if at, ok := link.BuildAskedAt(id); ok {
|
||||
since = at.Add(-time.Minute)
|
||||
}
|
||||
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started, ok := heard.Started[id]
|
||||
if !ok {
|
||||
return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id)
|
||||
}
|
||||
if heard.Outcomes[id] {
|
||||
return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
|
||||
}
|
||||
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return printHolderAnswer(started.On, answer)
|
||||
}
|
||||
|
||||
// pauseCommand asks one machine's holder, or every holder's, to pause or resume.
|
||||
func pauseCommand(ctx context.Context, verb string, args []string) error {
|
||||
if len(args) > 1 {
|
||||
return fmt.Errorf("%s [node]", verb)
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
nodes := args
|
||||
if len(nodes) == 0 {
|
||||
if nodes, err = buildSeatHolders(ctx, seat); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb)
|
||||
}
|
||||
}
|
||||
failed := 0
|
||||
for _, node := range nodes {
|
||||
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks)
|
||||
if err != nil {
|
||||
fmt.Printf("%s: %v\n", node, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
if err := printHolderAnswer(node, answer); err != nil {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printHolderAnswer prints what a holder said, or its refusal as the command's failure.
|
||||
func printHolderAnswer(node string, answer link.Answer) error {
|
||||
if answer.Error != "" {
|
||||
fmt.Printf("%s: %s\n", node, answer.Error)
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
var said struct {
|
||||
Said string `json:"said"`
|
||||
}
|
||||
if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" {
|
||||
fmt.Printf("%s: %s\n", node, said.Said)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s: %s\n", node, string(answer.Result))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildSeatHolders is every machine an assigned module holding the build seat runs on.
|
||||
func buildSeatHolders(ctx context.Context, seat string) ([]string, error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return holdersAmong(entries, seat), nil
|
||||
}
|
||||
|
||||
// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each.
|
||||
func holdersAmong(entries []inventory.Entry, seat string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if !e.Manifest.ClaimsSeat(seat) {
|
||||
continue
|
||||
}
|
||||
for _, n := range e.On {
|
||||
if !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
|
||||
// by repository and path, and every open plan holding it not yet built.
|
||||
func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
|
||||
var out []string
|
||||
for _, a := range q.Asks {
|
||||
if repositoryMatches(a.Repository, repository) && a.Path == path {
|
||||
out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
|
||||
}
|
||||
}
|
||||
if module == "" {
|
||||
return out
|
||||
}
|
||||
for _, p := range plans {
|
||||
if !p.Open() {
|
||||
continue
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
if m == module {
|
||||
if st := p.Modules[m]; st == nil || st.State != "built" {
|
||||
out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,772 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
|
||||
//
|
||||
// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
|
||||
// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
|
||||
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused' (each test on a bus of its own)
|
||||
|
||||
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
|
||||
func asksRecorded(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond))
|
||||
asked = append(asked, source.Repository+"#"+id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
return &asked
|
||||
}
|
||||
|
||||
// twoTiers registers two modules, b standing on a, each with a source a plan asks.
|
||||
func twoTiers(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"a", "b"} {
|
||||
if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A failed plan is retried: its failed module asked again under a new id, the plan building at that
|
||||
// tier, and when that build comes in the plan goes on and asks its next tier.
|
||||
func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.KilledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said, err := retryPlan(ctx, open, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := p.Modules["a"]
|
||||
if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
|
||||
t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
|
||||
}
|
||||
if !strings.Contains(said, a.Build) {
|
||||
t.Errorf("retry does not say the new id: %q", said)
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
|
||||
// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
|
||||
planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
|
||||
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
|
||||
t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
|
||||
}
|
||||
asking, _ := link.BuildAskedAt(a.Build)
|
||||
planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
|
||||
p, err = open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
|
||||
t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// What retry refuses, and says why.
|
||||
func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
plan := func(id, state string, created time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee",
|
||||
Created: created, State: state, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||
}
|
||||
failed := plan("plan-1", inventory.PlanFailed, at)
|
||||
for _, c := range []struct {
|
||||
p inventory.Plan
|
||||
others []inventory.Plan
|
||||
says string
|
||||
}{
|
||||
{plan("plan-d", inventory.PlanDone, at), nil, "is done"},
|
||||
{plan("plan-s", inventory.PlanSuperseded, at), nil, "was"},
|
||||
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
||||
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
||||
} {
|
||||
err := retryRefusal(c.p, c.others)
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
||||
}
|
||||
}
|
||||
stopped := failed
|
||||
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
stopped.Note = "a stopped at its first machine"
|
||||
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||
}
|
||||
// Another branch's newer plan, an older one, and a failed one do not supersede it.
|
||||
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||
other.Branch = "release"
|
||||
if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
|
||||
plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
|
||||
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
|
||||
func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
|
||||
t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
|
||||
if err := rebuildCommand(ctx, []string{"b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
|
||||
t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
|
||||
func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
|
||||
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
|
||||
t.Fatal("joined a failed plan a newer open one supersedes")
|
||||
}
|
||||
if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
|
||||
t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
|
||||
}
|
||||
built := failed
|
||||
built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
|
||||
t.Fatal("joined a plan that has the module built")
|
||||
}
|
||||
}
|
||||
|
||||
// replay is a dry run unless registered, and registering an older commit than one registered since
|
||||
// is refused unless --older says it is meant (novox/hq issue 207).
|
||||
func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
|
||||
newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
|
||||
history := []inventory.Build{newer, old}
|
||||
|
||||
if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
|
||||
t.Errorf("a dry run was refused: %v", err)
|
||||
}
|
||||
err := replayRefusal(old, "a", history, true, false, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
|
||||
t.Errorf("registering an older commit than the one registered was not refused: %v", err)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
|
||||
t.Errorf("--register --older was refused: %v", err)
|
||||
}
|
||||
if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
|
||||
t.Errorf("registering the newest build again was refused: %v", err)
|
||||
}
|
||||
// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
|
||||
same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
|
||||
broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
|
||||
if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
|
||||
t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
|
||||
}
|
||||
if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
|
||||
t.Error("a build that recorded no commit was replayed")
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
|
||||
t.Error("--older without --register was taken")
|
||||
}
|
||||
// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
|
||||
// holding it unbuilt, would be replaced by a replay asked now (issue 219).
|
||||
q := link.Queue{Asks: []link.QueuedAsk{
|
||||
{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
|
||||
{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
|
||||
{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
|
||||
}}
|
||||
plans := []inventory.Plan{
|
||||
{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
|
||||
{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
}
|
||||
outstanding := outstandingFor("a", "novox/a", "", q, plans)
|
||||
if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
|
||||
t.Fatalf("outstanding: %v", outstanding)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
|
||||
t.Errorf("registered over an outstanding ask: %v", err)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
|
||||
t.Errorf("a dry run was refused for what is outstanding: %v", err)
|
||||
}
|
||||
if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
|
||||
t.Errorf("a dry replay does not say what it is: %q", said)
|
||||
}
|
||||
if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
|
||||
t.Errorf("a registered replay does not say what it does: %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
|
||||
// paused on some holders only is not a reason the plan is waiting.
|
||||
func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
||||
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
asked := now.Add(-12 * time.Minute)
|
||||
p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding,
|
||||
Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||
|
||||
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
|
||||
line := planLineWith(p, now, all)
|
||||
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan on a paused seat reads %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
||||
t.Errorf("status --json says %+v", st)
|
||||
}
|
||||
if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
|
||||
t.Errorf("a plan waiting on a paused seat counted late")
|
||||
}
|
||||
|
||||
longAgo := now.Add(-25 * time.Hour)
|
||||
forgotten := p
|
||||
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
|
||||
if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan paused over a day reads %q", line)
|
||||
}
|
||||
|
||||
some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
|
||||
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
|
||||
t.Fatalf("%+v", some)
|
||||
}
|
||||
if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
|
||||
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
|
||||
t.Errorf("status --json: %+v", st)
|
||||
}
|
||||
// A plan rolling out, or with nothing asked, is not waiting on the seat.
|
||||
rolling := p
|
||||
rolling.State = inventory.PlanRolling
|
||||
if _, paused := pausedWaiting(rolling, all, now); paused {
|
||||
t.Error("a rolling plan reads as waiting on the build seat")
|
||||
}
|
||||
}
|
||||
|
||||
// The queue's verbs are the controller seat's, each to the command it names.
|
||||
func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{"queue", nil, []string{"queue"}},
|
||||
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
||||
{"clear", nil, []string{"clear"}},
|
||||
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
||||
{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
|
||||
{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
|
||||
{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
|
||||
{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
|
||||
{"pause", nil, []string{"pause"}},
|
||||
{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}},
|
||||
} {
|
||||
got, err := argvFor(c.verb, c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("cancel", nil); err == nil {
|
||||
t.Error("cancel without an id was taken")
|
||||
}
|
||||
declared := map[string]bool{}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
declared[v.Name] = true
|
||||
}
|
||||
for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} {
|
||||
if !declared[v] {
|
||||
t.Errorf("%s is not a verb of the controller seat", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- against a real bus -----------------------------------------------------------------------
|
||||
|
||||
// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller
|
||||
// pointed at it, and a function that asks the seat one build.
|
||||
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
|
||||
t.Helper()
|
||||
url := testbus.URL(t)
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
|
||||
Emits: []string{"started", "built", "log.*", "paused.*"}}
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||
if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
|
||||
link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||
_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
|
||||
_ = js.Context().PurgeStream(broker.EventsStream)
|
||||
})
|
||||
ask := func(id, repository string) link.BuildRequest {
|
||||
r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
|
||||
body, _ := json.Marshal(r)
|
||||
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
return js, ask
|
||||
}
|
||||
|
||||
// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows.
|
||||
func deadOne(t *testing.T, js *broker.JetStream) {
|
||||
t.Helper()
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
for i := 0; i < worker.MaxDeliver; i++ {
|
||||
msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("delivery %d: %v", i+1, err)
|
||||
}
|
||||
_ = msgs[0].Nak()
|
||||
}
|
||||
// One more pull, as a holder always has one waiting: the server finds the ask past its deliveries
|
||||
// then, and stops counting it pending.
|
||||
if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 {
|
||||
t.Fatalf("an ask past its deliveries was delivered again")
|
||||
}
|
||||
}
|
||||
|
||||
// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan
|
||||
// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for
|
||||
// is still found.
|
||||
func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
twoTiers(t, open)
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
asked, _ := link.BuildAskedAt(id)
|
||||
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
|
||||
t.Fatalf("the queue reads %+v", q)
|
||||
}
|
||||
if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
|
||||
strings.Contains(text, "secret-ish") {
|
||||
t.Errorf("the queue says:\n%s", text)
|
||||
}
|
||||
|
||||
if err := cancelCommand(ctx, []string{id}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||
t.Fatalf("the ask is still queued: %+v", q.Asks)
|
||||
}
|
||||
if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
|
||||
t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
|
||||
}
|
||||
b, found, err := open.inventory.BuildByID(ctx, id)
|
||||
if err != nil || !found || b.Failed != link.CancelledByHand {
|
||||
t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, plan.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
|
||||
t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
|
||||
}
|
||||
if err := cancelCommand(ctx, []string{id}); err == nil {
|
||||
t.Error("an ask cancelled already was cancelled again")
|
||||
}
|
||||
}
|
||||
|
||||
// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight.
|
||||
func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
start := time.Now()
|
||||
dead := link.NewBuildID(start)
|
||||
ask(dead, "https://forge.example/novox/dead.git")
|
||||
deadOne(t, js)
|
||||
var waiting []string
|
||||
for i := 1; i <= 2; i++ {
|
||||
id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
|
||||
waiting = append(waiting, id)
|
||||
ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
|
||||
t.Fatalf("the queue reads %+v", q)
|
||||
}
|
||||
said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
|
||||
t.Errorf("clear said:\n%s", said)
|
||||
}
|
||||
q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
|
||||
t.Fatalf("after clear the queue is %+v", q.Asks)
|
||||
}
|
||||
if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||
t.Fatalf("clear --dead left %+v", q.Asks)
|
||||
}
|
||||
for _, id := range append(waiting, dead) {
|
||||
if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
|
||||
t.Errorf("%s is recorded as %+v", id, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An ask in flight is not cancelled: kill ends it where it runs.
|
||||
func TestCancelRefusesAnAskInFlight(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := q.Find(id)
|
||||
if a.State != link.AskInFlight || a.On != "ace" {
|
||||
t.Fatalf("the taken ask reads %+v", a)
|
||||
}
|
||||
_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
|
||||
if err == nil || !strings.Contains(err.Error(), "kill "+id) {
|
||||
t.Fatalf("an ask in flight was cancelled: %v", err)
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||
t.Error("a refused cancel recorded an outcome")
|
||||
}
|
||||
}
|
||||
|
||||
// kill finds the machine from the build's start and asks that machine's holder; pause asks the
|
||||
// machine named. Each prints what the holder answered, and a refusal is the command's failure.
|
||||
func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
||||
js, _ := aBuildQueue(t)
|
||||
ctx := t.Context()
|
||||
asked := map[string]string{}
|
||||
handlers := map[string]link.ToolHandler{
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct{ ID string }
|
||||
_ = json.Unmarshal(raw, &args)
|
||||
asked["kill"] = args.ID
|
||||
if args.ID != "build-running" {
|
||||
return nil, fmt.Errorf("ace is not building %s", args.ID)
|
||||
}
|
||||
return map[string]any{"said": "killed " + args.ID}, nil
|
||||
},
|
||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||
asked["pause"] = "ace"
|
||||
return map[string]any{"said": "ace is paused"}, nil
|
||||
},
|
||||
}
|
||||
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
for _, id := range []string{"build-running", "build-other"} {
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-running"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked["kill"] != "build-running" {
|
||||
t.Fatalf("the holder was asked %v", asked)
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-other"}); err == nil {
|
||||
t.Error("the holder's refusal was not the command's")
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
|
||||
t.Errorf("a build nobody started: %v", err)
|
||||
}
|
||||
if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
|
||||
t.Fatalf("pause: %v %v", err, asked)
|
||||
}
|
||||
if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
|
||||
t.Error("a machine nothing answers on was resumed")
|
||||
}
|
||||
}
|
||||
|
||||
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
var sentTo [][]string
|
||||
was := sendRollout
|
||||
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||
sentTo = append(sentTo, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
|
||||
// a records: a person's choice, since the default rolls out (novox/hq ADR 0236).
|
||||
if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
long := time.Now().UTC().Add(-2 * time.Hour)
|
||||
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &stopped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A newer plan holding a refuses it: sending the older build would put it back.
|
||||
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
|
||||
t.Fatalf("retried under a newer plan: %v", err)
|
||||
}
|
||||
newer.State = inventory.PlanSuperseded
|
||||
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said, err := retryPlan(ctx, open, stopped.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
|
||||
t.Fatalf("sent %v; said %q", sentTo, said)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, stopped.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := p.Modules["a"]
|
||||
if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
|
||||
t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
|
||||
}
|
||||
// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
|
||||
advancePlans(ctx, open)
|
||||
if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
|
||||
t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
|
||||
// the plan, asked later, never answers it (novox/hq ADR 0219).
|
||||
func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := time.Now().UTC().Add(-time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
|
||||
p, _ := open.inventory.PlanByID(ctx, plan.ID)
|
||||
if p.Modules["a"].State != "asked" {
|
||||
t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
|
||||
}
|
||||
// From the records too: a later build of the module recorded is not the plan's.
|
||||
recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
|
||||
if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
|
||||
t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
|
||||
}
|
||||
planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
|
||||
if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
|
||||
t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
|
||||
}
|
||||
}
|
||||
|
||||
// rebuild of a build made at a commit asks what the module follows now, never the commit.
|
||||
func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
var refs []string
|
||||
was := askABuild
|
||||
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||
refs = append(refs, ref)
|
||||
return was(c, source, path, ref)
|
||||
}
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
|
||||
Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
|
||||
t.Fatalf("asked %v at %v", *asked, refs)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask the worker counts out that no machine said it started is cancelled only if no start comes
|
||||
// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
|
||||
func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
was := holderLooks
|
||||
holderLooks = 300 * time.Millisecond
|
||||
t.Cleanup(func() { holderLooks = was })
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := q.Find(id)
|
||||
if a.State != link.AskInFlight || a.On != "" {
|
||||
t.Fatalf("the taken ask reads %+v", a)
|
||||
}
|
||||
// The holder that took it says it started, while the cancel waits.
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
_, _ = js.Context().Publish(link.BuildStarted(), started)
|
||||
}()
|
||||
if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
|
||||
t.Fatalf("a build that started was cancelled: %v", err)
|
||||
}
|
||||
if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
|
||||
t.Error("the withdrawn cancel is still marked")
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||
t.Error("a withdrawn cancel recorded an outcome")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -73,6 +75,32 @@ type meshStatus struct {
|
||||
// alone. A document without this called a machine well while a predecessor's chain refused
|
||||
// what the mesh declared open.
|
||||
Filtered []machineFiltered `json:"filtered,omitempty"`
|
||||
// Unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||
// dependency is met. Reported, not refused, until the switch.
|
||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
|
||||
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
|
||||
// HandActsUnread says why when it could not be read, rather than reading as none.
|
||||
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
|
||||
HandActsUnread string `json:"handActsUnread,omitempty"`
|
||||
// HealsThisWeek is what the healers did in the last seven days (novox/hq to-be 45 §7); HealsUnread
|
||||
// why it could not be read.
|
||||
HealsThisWeek *healsCount `json:"healsThisWeek,omitempty"`
|
||||
HealsUnread string `json:"healsUnread,omitempty"`
|
||||
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
|
||||
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
|
||||
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
|
||||
Conditions []conditions.Condition `json:"conditions"`
|
||||
ConditionsUnread string `json:"conditionsUnread,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
|
||||
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
|
||||
// provider says so. A document without it called the mesh well while the identity provider
|
||||
// refused every consumer for a day (04-ISSUES/179).
|
||||
Failing []conditions.Condition `json:"failing,omitempty"`
|
||||
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
|
||||
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
|
||||
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
@@ -171,7 +199,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now())}
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
|
||||
// In a stated order, so two readings of an unchanged mesh are the same document.
|
||||
untakenNodes := make([]string, 0, len(asked.untaken))
|
||||
for name := range asked.untaken {
|
||||
@@ -204,6 +232,15 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
||||
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
|
||||
if out.Conditions == nil {
|
||||
out.Conditions = []conditions.Condition{}
|
||||
}
|
||||
out.Failing = providerStandings(asked.conditions)
|
||||
out.Overflowing = asked.overflowing
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -0,0 +1,667 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// No build reaches a machine without a gate (novox/hq ADR 0236).
|
||||
//
|
||||
// **A send carries the machine's whole declaration.** A plan sending one module to its first machine
|
||||
// sends every other module whose build moved there too; a cascade, a healer's resend and a whole-mesh
|
||||
// push do the same. Under the old default (`record`) builds were registered and sent nowhere, so on the
|
||||
// day the default became `roll` every machine was behind on dozens of builds no gate had seen — and the
|
||||
// next send of anything would have restarted all of them at once, on every machine.
|
||||
//
|
||||
// So **a build moves on a machine only through a send that judges it there**, or a person's:
|
||||
//
|
||||
// - a gated send — a plan's first machine, a release plan's machine — carries every move waiting on that
|
||||
// machine, and its gate judges each of them; a pass is each build's verdict, a failure puts back what
|
||||
// failed;
|
||||
// - a module a send exists for may move anywhere it is sent: a policy of *together*, a rollback, a build
|
||||
// whose gate already passed;
|
||||
// - a send naming a machine by a person (`push <node>`, the bus step) carries what it carries;
|
||||
// - every other send — a plan's "rest", a cascade, a healer's, the bus's user list carried — is refused,
|
||||
// or leaves the machine, while a move there waits for a gate. A rebuild that made the same artifacts
|
||||
// from the same manifest is no move.
|
||||
//
|
||||
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
|
||||
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
|
||||
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
|
||||
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
|
||||
|
||||
// sendScope is what a send may carry that no gate has seen.
|
||||
type sendScope struct {
|
||||
// judged are the machines whose every move this send's gate judges.
|
||||
judged map[string]bool
|
||||
// modules are those a send exists for, which may move wherever it goes.
|
||||
modules map[string]bool
|
||||
// person is a person's act: it carries what it carries.
|
||||
person bool
|
||||
}
|
||||
|
||||
type sendScopeKey struct{}
|
||||
|
||||
func withScope(ctx context.Context, s sendScope) context.Context {
|
||||
return context.WithValue(ctx, sendScopeKey{}, s)
|
||||
}
|
||||
|
||||
func scopeOf(ctx context.Context) sendScope {
|
||||
s, _ := ctx.Value(sendScopeKey{}).(sendScope)
|
||||
return s
|
||||
}
|
||||
|
||||
// errUngated is a send refused because it would carry a build no gate has seen.
|
||||
var errUngated = errors.New("a build waits there for its gate")
|
||||
|
||||
// errWalkedElsewhere is a gated send refused because a plan that has started walks a move there.
|
||||
var errWalkedElsewhere = errors.New("a plan already walking a build there sends it")
|
||||
|
||||
// moveFacts is what tells a move from a rebuild, and a gated build from one no gate has seen.
|
||||
type moveFacts struct {
|
||||
current map[string]inventory.CurrentBuild
|
||||
fps map[string]map[string]string
|
||||
// srcs is, per module, per commit, its build's source fingerprint (novox/hq issue 280).
|
||||
srcs map[string]map[string]string
|
||||
passed map[string]map[string]bool
|
||||
plans []inventory.Plan
|
||||
}
|
||||
|
||||
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
|
||||
var f moveFacts
|
||||
var err error
|
||||
if f.current, err = inv.CurrentBuilds(ctx); err != nil {
|
||||
return f, err
|
||||
}
|
||||
if f.fps, err = inv.Fingerprints(ctx); err != nil {
|
||||
return f, err
|
||||
}
|
||||
if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
|
||||
return f, err
|
||||
}
|
||||
if f.passed, err = inv.PassedCommits(ctx); err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.plans, err = inv.OpenPlans(ctx)
|
||||
return f, err
|
||||
}
|
||||
|
||||
// identical is whether two builds of a module put the same thing on a machine: the same commit,
|
||||
// builds made from the same source (novox/hq issue 280) — the module's tree, the contexts it read, its
|
||||
// bases and toolchains, whatever digests an image rebuild made of them — or builds that made the same
|
||||
// artifacts from the same manifest.
|
||||
func (f moveFacts) identical(module, a, b string) bool {
|
||||
if a == b || sameCommit(a, b) {
|
||||
return true
|
||||
}
|
||||
if sa := f.srcs[module][a]; sa != "" && sa == f.srcs[module][b] {
|
||||
return true
|
||||
}
|
||||
fa := f.fps[module][a]
|
||||
return fa != "" && fa == f.fps[module][b]
|
||||
}
|
||||
|
||||
// gated is whether a build of a module from this commit — or one identical to it — passed a gate.
|
||||
func (f moveFacts) gated(module, commit string) bool {
|
||||
for c := range f.passed[module] {
|
||||
if f.identical(module, c, commit) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// unchangedOnEvery is whether a plan's build of a module was made from the source of the build every
|
||||
// machine running it was last sent (novox/hq issue 280): no move on any of them. False when no machine
|
||||
// runs it, when what one was sent is not known, or when the build stands for itself.
|
||||
func unchangedOnEvery(ctx context.Context, inv *inventory.Inventory, module, build string, running []string) (bool, error) {
|
||||
if build == "" || len(running) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
same, err := inv.SameSourceCommits(ctx, module, build)
|
||||
if err != nil || len(same) == 0 {
|
||||
return false, err
|
||||
}
|
||||
for _, n := range running {
|
||||
sent, known, err := inv.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
was, carried := sent[module]
|
||||
if !known || !carried || !inRun(same, was) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// inRun is whether a commit is one of a run's, however either is abbreviated.
|
||||
func inRun(run map[string]bool, commit string) bool {
|
||||
for c := range run {
|
||||
if sameCommit(c, commit) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out
|
||||
// (a recorded one is a person's push), that the machine was sent before, moving to a build not identical
|
||||
// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names
|
||||
// none: the cascade holds it whole (ADR 0221).
|
||||
//
|
||||
// With all, a move to a build that passed a gate elsewhere counts too: what a gated send carries.
|
||||
func (f moveFacts) moves(node string, modules []string, sent map[string]string, known, all bool) []inventory.CarriedMove {
|
||||
if !known {
|
||||
return nil
|
||||
}
|
||||
var out []inventory.CarriedMove
|
||||
for _, m := range modules {
|
||||
now := f.current[m]
|
||||
was, carried := sent[m]
|
||||
if !now.RollOut || !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) {
|
||||
continue
|
||||
}
|
||||
out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// walkedBy is the open plan that has started walking a module's build — sent it to a first machine,
|
||||
// not yet passed — other than to this machine; empty when none does.
|
||||
func (f moveFacts) walkedBy(module, node string) string {
|
||||
for _, p := range f.plans {
|
||||
s, holds := p.Modules[module]
|
||||
if !p.Open() || !holds || s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(s.First, node) {
|
||||
continue
|
||||
}
|
||||
if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed {
|
||||
continue
|
||||
}
|
||||
return p.ID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// machineMoves is the moves no gate has seen on one machine, read from what it would be sent now.
|
||||
var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, nil // it cannot be worked out: the send says why
|
||||
}
|
||||
modules := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
|
||||
// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the
|
||||
// machines named, and why; the machine holding the bus, added only for its user list, is left instead.
|
||||
func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) {
|
||||
scope := scopeOf(ctx)
|
||||
if scope.person {
|
||||
return names, nil
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var kept, refused []string
|
||||
for _, n := range names {
|
||||
moves, err := machineMoves(ctx, open, f, n, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var waiting []string
|
||||
for _, mv := range moves {
|
||||
if !scope.judged[n] && !scope.modules[mv.Module] {
|
||||
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(waiting) == 0:
|
||||
kept = append(kept, n)
|
||||
case n == addedHolder:
|
||||
fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s wait there for a gate "+
|
||||
"— the bus may refuse what was newly granted until it is\n", n, strings.Join(waiting, ", "))
|
||||
default:
|
||||
refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", ")))
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
return nil, fmt.Errorf("%w: %s — a release plan sends them, one machine at a time, each judged (novox/hq ADR "+
|
||||
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// gatedSend sends one machine everything waiting there, under a gate that judges it all: what moved is
|
||||
// answered, with the build each moved to, for the gate to judge and to put back. Refused while a plan that
|
||||
// has started walks one of those builds elsewhere: that plan sends it here once its gate passed.
|
||||
//
|
||||
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
|
||||
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
|
||||
// module was the same declaration sent again and again, each one setting aside the one before.
|
||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) {
|
||||
inv := open.inventory
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
moves, err := machineMoves(ctx, open, f, node, true)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
own := func(module string) bool {
|
||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||
}
|
||||
for _, mv := range moves {
|
||||
if own(mv.Module) {
|
||||
continue
|
||||
}
|
||||
if id := f.walkedBy(mv.Module, node); id != "" {
|
||||
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
mv.Module, short(mv.To), node, id)
|
||||
}
|
||||
}
|
||||
for _, o := range owns {
|
||||
i := slices.IndexFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == o.Module })
|
||||
switch {
|
||||
case i < 0:
|
||||
moves = append(moves, o)
|
||||
case moves[i].Build == "":
|
||||
moves[i].Build = o.Build
|
||||
}
|
||||
}
|
||||
if len(moves) == 0 && len(owns) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
for i := range moves {
|
||||
if moves[i].Build == "" {
|
||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return moves, sent, nil
|
||||
}
|
||||
|
||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
||||
seen := map[string]bool{}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] {
|
||||
continue
|
||||
}
|
||||
seen[c.Build] = true
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
|
||||
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why,
|
||||
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
|
||||
fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// failCarried puts back every build the gate carried that it found wanting, on the machines that were
|
||||
// sent it, once each.
|
||||
func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
||||
var notes []string
|
||||
if p.Note != "" {
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
done := map[string]bool{except: true}
|
||||
for _, c := range g.Carried {
|
||||
if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) {
|
||||
continue
|
||||
}
|
||||
done[c.Module] = true
|
||||
machines, err := sentTheBuild(ctx, open, c.Module, c.To)
|
||||
if err != nil || len(machines) == 0 {
|
||||
machines = []string{c.Node}
|
||||
}
|
||||
state := &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To}
|
||||
// A module of the plan's tier sent in the same send (issue 281) keeps its own record of it.
|
||||
if s := p.Modules[c.Module]; s != nil && except != "" && s.GatedBy == except && s.Build == c.Build {
|
||||
state = s
|
||||
}
|
||||
p.Note = ""
|
||||
gateFailed(ctx, open, p, c.Module, state, machines, g.Why)
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = strings.Join(notes, "; ")
|
||||
}
|
||||
|
||||
// sentTheBuild is every machine running a module that was last sent this build of it.
|
||||
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
|
||||
running, err := open.inventory.Running(ctx, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, n := range running {
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if known && sameCommit(sent[module], commit) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// releaseRepository is what a release plan says it is for, where a merge's says its repository.
|
||||
const releaseRepository = "the builds waiting for a gate"
|
||||
|
||||
// releaseHeard is the machines a release plan may send: heard within their heartbeat's bound. A machine
|
||||
// away is left, not judged against a bound it cannot meet. A variable so a test says who is heard.
|
||||
var releaseHeard = func(ctx context.Context, open *stores) (map[string]bool, error) {
|
||||
if d := doctorFrom; d != nil && d.watchdogs != nil {
|
||||
return heardMachines(d), nil
|
||||
}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for _, r := range reports {
|
||||
if r.At != nil && time.Since(*r.At) < 15*time.Minute {
|
||||
out[r.Node] = true
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// backlogNow is what the newest look found waiting, for `upgrade backlog` and the gate's probe.
|
||||
var backlogNow struct {
|
||||
held string
|
||||
waiting map[string][]inventory.CarriedMove
|
||||
}
|
||||
|
||||
// waitingMoves is every machine's moves no gate has seen and no started plan walks.
|
||||
func waitingMoves(ctx context.Context, open *stores, all bool) (map[string][]inventory.CarriedMove, error) {
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]inventory.CarriedMove{}
|
||||
for _, n := range nodes {
|
||||
moves, err := machineMoves(ctx, open, f, n.Name, all)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, mv := range moves {
|
||||
if f.walkedBy(mv.Module, n.Name) == "" {
|
||||
out[n.Name] = append(out[n.Name], mv)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// releaseBacklog opens a release plan when builds wait for a gate and none is open; not after a release
|
||||
// plan failed, until a person releases one (by). Called with the plans held.
|
||||
func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Plan, error) {
|
||||
inv := open.inventory
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range plans {
|
||||
if p.Release != nil {
|
||||
if by != "" {
|
||||
return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
waiting, err := waitingMoves(ctx, open, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
backlogNow.waiting, backlogNow.held = waiting, ""
|
||||
if len(waiting) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
// Opened by what no gate has seen; it walks every machine where anything of the release waits — a
|
||||
// build that passed on the first machine still goes to the next one by this plan, judged there too.
|
||||
if waiting, err = waitingMoves(ctx, open, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if by == "" {
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Release == nil {
|
||||
continue
|
||||
}
|
||||
if p.State == inventory.PlanFailed {
|
||||
backlogNow.held = fmt.Sprintf("%s failed (%s); what waits is released again by a person", p.ID, p.Note)
|
||||
return nil, nil
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
controllers, err := inv.Running(ctx, catalogue.ControllerSeatName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var order, last []string
|
||||
modules := map[string]bool{}
|
||||
for node, moves := range waiting {
|
||||
if !heard[node] {
|
||||
continue
|
||||
}
|
||||
for _, mv := range moves {
|
||||
modules[mv.Module] = true
|
||||
}
|
||||
if slices.Contains(controllers, node) {
|
||||
last = append(last, node)
|
||||
} else {
|
||||
order = append(order, node)
|
||||
}
|
||||
}
|
||||
if len(order)+len(last) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
sort.Strings(order)
|
||||
sort.Strings(last)
|
||||
order = append(order, last...)
|
||||
names := make([]string, 0, len(modules))
|
||||
for m := range modules {
|
||||
names = append(names, m)
|
||||
}
|
||||
sort.Strings(names)
|
||||
now := time.Now().UTC()
|
||||
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
|
||||
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
|
||||
Release: &inventory.PlanRelease{Order: order, By: by},
|
||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
|
||||
strings.Join(order, ", "))}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
|
||||
// gate, or the machine being judged judged once more; the plan done when every machine is.
|
||||
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
||||
r := p.Release
|
||||
now := time.Now().UTC()
|
||||
if r.Gate == nil {
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for r.Next < len(r.Order) {
|
||||
node := r.Order[r.Next]
|
||||
if !heard[node] {
|
||||
r.Skipped = append(r.Skipped, node)
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
moves, sent, err := gatedSend(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
changed := p.Note != note
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||
}
|
||||
if len(moves) == 0 {
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves}
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||
p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
|
||||
if len(r.Skipped) > 0 {
|
||||
p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
g := r.Gate
|
||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g))
|
||||
changed := p.Note != note
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
case inventory.GatePassed:
|
||||
passCarried(ctx, open, p, g, "")
|
||||
r.Done = append(r.Done, firstOf(g.Machines))
|
||||
r.Next++
|
||||
r.Gate = nil
|
||||
return true, nil
|
||||
}
|
||||
p.Note = ""
|
||||
batched, back := batchingRollbacks(ctx)
|
||||
failCarried(batched, open, p, g, "")
|
||||
sendRollbacks(ctx, open, p, back)
|
||||
p.Note = fmt.Sprintf("failed its gate on %s: %s — %s", strings.Join(g.Machines, ", "), g.Why, p.Note)
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// backlogObservation is what the gate's probe says of a release held after a failure.
|
||||
func backlogObservation() []conditions.Observation {
|
||||
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
|
||||
return nil
|
||||
}
|
||||
n := 0
|
||||
var machines []string
|
||||
for node, moves := range backlogNow.waiting {
|
||||
n += len(moves)
|
||||
machines = append(machines, node)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
return []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held",
|
||||
Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%d build move(s) on %s wait for a gate and are not released: %s — `upgrade backlog` lists "+
|
||||
"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}}
|
||||
}
|
||||
|
||||
// backlogCommand is `upgrade backlog`, read-only, and `upgrade release-backlog --why`.
|
||||
func backlogCommand(ctx context.Context, sub string, args []string) error {
|
||||
set := flag.NewFlagSet("upgrade "+sub, flag.ContinueOnError)
|
||||
why := addHandActFlags(set)
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("upgrade backlog | upgrade release-backlog --why <text>")
|
||||
}
|
||||
if sub == "release-backlog" {
|
||||
if err := why.require("upgrade release-backlog"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if sub == "release-backlog" {
|
||||
release, err := open.inventory.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
why.record(ctx, "upgrade release-backlog", nil)
|
||||
p, err := releaseBacklog(ctx, open, link.Caller())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p == nil {
|
||||
fmt.Println("nothing waits for a gate on any machine heard from: nothing to release")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s releases it; `plans %s` says where it is\n", p.ID, p.ID)
|
||||
return nil
|
||||
}
|
||||
waiting, err := waitingMoves(ctx, open, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(waiting) == 0 {
|
||||
fmt.Println("no build waits for a gate on any machine")
|
||||
return nil
|
||||
}
|
||||
nodes := make([]string, 0, len(waiting))
|
||||
for n := range waiting {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
fmt.Printf("%s: %d build(s) wait for a gate\n", n, len(waiting[n]))
|
||||
for _, mv := range waiting[n] {
|
||||
fmt.Printf(" %-28s %s → %s\n", mv.Module, short(mv.From), short(mv.To))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -115,3 +115,142 @@ func TestACycleIsOneLastTierAndSaidSo(t *testing.T) {
|
||||
t.Fatalf("a cycle should be one tier of two, said: %v", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the
|
||||
// bundle a tier after the toolchain, not beside it.
|
||||
func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) {
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}}
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"},
|
||||
[]string{"mesh-tools", "node-tools"}, edges)
|
||||
if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" {
|
||||
t.Fatalf("the toolchain, then the bundle: %v", p.Tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/214: a plan whose build outcome was recorded while no controller followed it —
|
||||
// the controller rebuilding itself — settles from the build records instead of waiting for ever.
|
||||
func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 3, 19, 20, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Tiers: [][]string{{"mesh-controller", "builder"}, {"route-proxy"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"mesh-controller": {State: "asked", AskedAt: &asked},
|
||||
"builder": {State: "asked", AskedAt: &asked},
|
||||
}}
|
||||
records := map[string][]inventory.Build{
|
||||
// Newest first, as Builds answers: the build after the ask is the outcome.
|
||||
"mesh-controller": {
|
||||
{ID: "build-2", Commit: "2ebbb799", At: asked.Add(4 * time.Minute)},
|
||||
{ID: "build-1", Commit: "06ea2168", At: asked.Add(-10 * time.Minute)},
|
||||
},
|
||||
// Only a build from before the ask: not this ask's outcome.
|
||||
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
|
||||
}
|
||||
if !settleFromRecords(&p, p.Tiers[0], records, nil) {
|
||||
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
|
||||
}
|
||||
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
|
||||
t.Errorf("the controller's ask is %+v, want built from 2ebbb799", s)
|
||||
}
|
||||
if s := p.Modules["builder"]; s.State != "asked" {
|
||||
t.Errorf("an ask with no record after it was settled: %+v", s)
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: a build recorded after the ask but asked before it — an earlier
|
||||
// plan's late outcome — is not this ask's, built or failed.
|
||||
r := inventory.Plan{ID: "plan-3", Tiers: [][]string{{"postgres"}},
|
||||
Modules: map[string]*inventory.PlanModule{"postgres": {State: "asked", AskedAt: &asked}}}
|
||||
late := map[string][]inventory.Build{"postgres": {
|
||||
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
|
||||
}}
|
||||
if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
|
||||
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
|
||||
}
|
||||
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
|
||||
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
|
||||
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
|
||||
if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
|
||||
r.Modules["postgres"].Commit != "4bcd5f73" {
|
||||
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
|
||||
}
|
||||
|
||||
// A failure recorded after the ask fails the plan, as hearing it would have.
|
||||
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
|
||||
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
|
||||
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
|
||||
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
|
||||
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
|
||||
}
|
||||
}
|
||||
|
||||
// The first machine's report, made between the send to it and the send to the rest, opens the gate for it:
|
||||
// each machine is judged from its own send, not from the last one (novox/hq issue 256).
|
||||
func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
|
||||
built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC)
|
||||
firstSent := built.Add(31 * time.Second)
|
||||
firstReported := built.Add(43 * time.Second)
|
||||
restSent := built.Add(58 * time.Second)
|
||||
restReported := built.Add(74 * time.Second)
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", At: &firstReported},
|
||||
{Node: "g14", At: &restReported},
|
||||
}
|
||||
since := func(node string) time.Time {
|
||||
if node == "ace" {
|
||||
return firstSent
|
||||
}
|
||||
return restSent
|
||||
}
|
||||
if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok {
|
||||
t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting)
|
||||
}
|
||||
// The old reading, every machine from the last send, is what held the plan.
|
||||
if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok {
|
||||
t.Fatal("the single-moment reading should hold the first machine back")
|
||||
}
|
||||
early := built.Add(10 * time.Second)
|
||||
if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" {
|
||||
t.Fatal("a report from before the machine was sent opened the gate")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 278 and ADR 0236's open question: a change to the build agent — its manifest alone, as
|
||||
// the catalogue's data sections were, or its program — rebuilds the build agent and nothing it builds.
|
||||
// What it builds is ordered after it in a plan that holds both, never added to one for its sake. The
|
||||
// merge that rebuilt 103 modules with the agent in tier 0 rebuilt them for a file read as shared code;
|
||||
// the agent stood first only because everything else is built by it.
|
||||
func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
agent := fromRepo("build-agent", repo, "modules/build-agent")
|
||||
redis := fromRepo("redis", repo, "modules/redis")
|
||||
postgres := fromRepo("postgres", repo, "modules/postgres")
|
||||
entries := []inventory.Entry{agent, redis, postgres}
|
||||
edges := []inventory.Edge{
|
||||
{From: "redis", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||
{From: "redis", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "postgres", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "mesh-tools", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
||||
}
|
||||
for _, paths := range [][]string{
|
||||
{"modules/build-agent/module.json"}, // its manifest alone
|
||||
{"modules/build-agent/cmd/agent/main.go", "modules/build-agent/module.json"}, // its program too
|
||||
} {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", Paths: paths,
|
||||
ModuleDirs: []string{"modules/build-agent"}, ModuleDirsSaid: true}
|
||||
touched := whatTheMergeTouched(entries, entries, m)
|
||||
if len(touched) != 1 || touched[0].Manifest.Module != "build-agent" {
|
||||
t.Fatalf("%v touched %v", paths, touched)
|
||||
}
|
||||
p := planOfMerge(m, []string{"build-agent"}, edges)
|
||||
if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "build-agent" || len(p.Modules) != 1 {
|
||||
t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers)
|
||||
}
|
||||
}
|
||||
// With what it builds moved beside it, the agent comes first and they follow: an order, not a widening.
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"},
|
||||
[]string{"build-agent", "redis"}, edges)
|
||||
if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 {
|
||||
t.Fatalf("the agent, then what moved beside it: %v", p.Tiers)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// The backlog the old default left — builds registered and sent nowhere — is released by a plan, one
|
||||
// machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236).
|
||||
|
||||
type backlogMesh struct {
|
||||
open *stores
|
||||
sent [][]string
|
||||
broken map[string]bool
|
||||
}
|
||||
|
||||
// aBacklog is a mesh where `app` moved c1 → c2 on anchor and laptop, `late` moved on laptop only, and
|
||||
// `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once.
|
||||
func aBacklog(t *testing.T) *backlogMesh {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
b := &backlogMesh{open: open, broken: map[string]bool{}}
|
||||
withConditionsInMemory(t)
|
||||
was := doctorFrom
|
||||
doctorFrom = nil
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
build := func(module, commit, made string, asked time.Time) {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}}
|
||||
for _, m := range []string{"app", "late", "same"} {
|
||||
build(m, "c1", "sha256:"+m+"-1", old)
|
||||
for _, n := range on[m] {
|
||||
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
sent := map[string]string{}
|
||||
for m, nodes := range on {
|
||||
for _, x := range nodes {
|
||||
if x == n {
|
||||
sent[m] = "c1"
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
build("app", "c2", "sha256:app-2", now)
|
||||
build("late", "c2", "sha256:late-2", now)
|
||||
build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes
|
||||
|
||||
assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
||||
machineMoves = assigned
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true}, nil
|
||||
}
|
||||
n := 0
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
b.sent = append(b.sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
modules, _ := open.inventory.Assigned(ctx, node)
|
||||
carried := map[string]string{}
|
||||
for _, m := range modules {
|
||||
carried[m] = current[m].Commit
|
||||
}
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if b.broken[r.Node] {
|
||||
r.Outcome = inventory.OutcomeFailed
|
||||
}
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery = 0, time.Hour
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
||||
t.Helper()
|
||||
plans, err := b.open.inventory.RecentPlans(t.Context(), 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range plans {
|
||||
if p.Release != nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
t.Fatal("no release plan")
|
||||
return inventory.Plan{}
|
||||
}
|
||||
|
||||
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
||||
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
||||
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moves, _ := machineMoves(ctx, b.open, f, "laptop", false)
|
||||
var names []string
|
||||
for _, mv := range moves {
|
||||
names = append(names, mv.Module)
|
||||
}
|
||||
if !reflect.DeepEqual(names, []string{"app", "late"}) {
|
||||
t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names)
|
||||
}
|
||||
// Nothing else may carry them: a send that judges nothing is refused.
|
||||
if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) {
|
||||
t.Fatalf("a send that judges nothing would carry them: %v", err)
|
||||
}
|
||||
|
||||
advancePlans(ctx, b.open)
|
||||
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent)
|
||||
}
|
||||
p := b.release(t)
|
||||
if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil {
|
||||
t.Fatalf("the release plan is %+v", p.Release)
|
||||
}
|
||||
gateEvery = 0
|
||||
for i := 0; i < 4; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v", b.sent)
|
||||
}
|
||||
p = b.release(t)
|
||||
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, build := range []string{"build-app-c2", "build-late-c2"} {
|
||||
if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err)
|
||||
}
|
||||
}
|
||||
// Nothing waits now, and nothing opens again.
|
||||
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
|
||||
t.Fatalf("a release opened with nothing waiting: %+v %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A release that fails on its first machine puts back what it carried there, goes no further, and the
|
||||
// next one waits for a person, said as a condition; a person releases it with why.
|
||||
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
gateEvery = 0
|
||||
b.broken["anchor"] = true
|
||||
advancePlans(ctx, b.open)
|
||||
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the release is %s: %s", p.State, p.Note)
|
||||
}
|
||||
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) {
|
||||
t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("app is at %s, not put back", current["app"].Commit)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
|
||||
t.Fatal("app's build is not marked failed at its gate")
|
||||
}
|
||||
// late still waits on laptop, and is not released on its own after a failure.
|
||||
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
|
||||
t.Fatalf("a release opened after a failed one: %+v %v", p, err)
|
||||
}
|
||||
if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") {
|
||||
t.Fatalf("the held release is not said: %+v", obs)
|
||||
}
|
||||
b.broken["anchor"] = false
|
||||
if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" {
|
||||
t.Fatalf("the person's release is %s (%+v)", p.State, p.Release)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||
t.Fatalf("late was not released to laptop: %v", sent)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
||||
// happened, asserting the rule's outcome rather than the fix's mechanism, so it can be run against the
|
||||
// commit before the fix and fail there, and against the fix and pass. **Written only with what the
|
||||
// controller had before each fix** — the stores, register, assign, planFor, declarationFor — so the
|
||||
// prover (mesh-lab replays/cmd/prove) can lay this file over the older commit and run it there.
|
||||
//
|
||||
// Registered in mesh-lab's replays/register.go with the fix each one proves; run by this repository's
|
||||
// merge check on every pull request with the rest of the suite.
|
||||
|
||||
// **R263 — a consumer's identity never refuses its provider's machine.** On 2026-10-06 the network
|
||||
// manager came to require the mesh's resolver; on a machine whose name made its identity 23 to 26
|
||||
// characters against the one global bound of 20, the anchor — the resolver's holder, carrying every
|
||||
// consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the
|
||||
// provider's machine composes whatever its consumers are called, and a provision that mints no
|
||||
// credential holds no consumer to a key's length.
|
||||
func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||
Requires: []string{"wildcard-resolution"}})
|
||||
for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
// The consumer's machine composes first, as a push does: what it is sent is what its provider grants.
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatalf("%s does not resolve: %v", node, err)
|
||||
}
|
||||
if _, err := declarationFor(ctx, open, node, plan, settings); err != nil {
|
||||
t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+
|
||||
"mesh_laptop_networkmanager (26 characters), refused it: %v", node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for
|
||||
// the resolver re-bound every database consumer on the home server — which runs its own store, beside
|
||||
// its applications' data — to the store seat's holder on the control node, which made each a new empty
|
||||
// database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a
|
||||
// machine running its own store stays bound to it while another machine holds the store's seat; the
|
||||
// resolver, which every holder answers alike, follows its seat.
|
||||
func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []catalogue.Manifest{
|
||||
{Module: "store", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
|
||||
{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
|
||||
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
|
||||
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
|
||||
} {
|
||||
register(t, open, m)
|
||||
}
|
||||
assignAll := func(pairs ...[2]string) {
|
||||
for _, a := range pairs {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The control node holds the mesh's store and resolver seats; the home server runs its own of each.
|
||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
||||
[2]string{"laptop", "board"})
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var board, network string
|
||||
for _, n := range plan.Needs {
|
||||
switch {
|
||||
case n.For == "board" && n.Name == "postgres-database":
|
||||
board = n.From
|
||||
case n.For == "network" && n.Name == "wildcard-resolution":
|
||||
network = n.From
|
||||
}
|
||||
}
|
||||
if board != "laptop" {
|
||||
t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+
|
||||
"database there (issue 273)", board)
|
||||
}
|
||||
if network != "anchor" {
|
||||
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The verbs a person answers a provider's retirement with, and cleans up with (novox/hq ADR 0230).
|
||||
//
|
||||
// **The controller asks; the provider acts.** Approving, rejecting and deleting are each a question to
|
||||
// the provider on the machine it runs on — its `provisioner_*` tools — because the provider owns its
|
||||
// backend and the controller touches none. Every one says why, is written in the hand-act log before
|
||||
// it is asked, and the provider says what it did as its `provisioner.retirement` event.
|
||||
|
||||
const retireUsage = "retire [--json] | retire approve <node> <module> --why <text> | retire reject <node> <module> --why <text>"
|
||||
|
||||
const cleanupUsage = "cleanup [list] [--json] | cleanup delete <node> <module> <consumer> --why <text> | " +
|
||||
"cleanup delete --older-than <days> --why <text> [--confirm]"
|
||||
|
||||
func isNothingServes(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
||||
|
||||
func unmarshalAnswer(a link.Answer, v any) error {
|
||||
if len(a.Result) == 0 {
|
||||
return errors.New("an empty answer")
|
||||
}
|
||||
return json.Unmarshal(a.Result, v)
|
||||
}
|
||||
|
||||
// retireCommand is `retire`, `retire approve` and `retire reject`.
|
||||
func retireCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
set := flag.NewFlagSet("retire", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error { return listRetiring(ctx, open.inventory, conn, *asJSON) })
|
||||
case "approve", "reject":
|
||||
set := flag.NewFlagSet("retire "+sub, flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 2 {
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
if err := f.require("retire " + sub); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return answerRetirement(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, sub == "approve", f)
|
||||
})
|
||||
}
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
|
||||
// retiringRow is one provider's waiting or rejected set, as `retire` lists it.
|
||||
type retiringRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Waiting []link.RetiredConsumer `json:"waiting,omitempty"`
|
||||
Since string `json:"since,omitempty"`
|
||||
Held int `json:"held,omitempty"`
|
||||
Bound string `json:"bound,omitempty"`
|
||||
Rejected []link.RetiredConsumer `json:"rejected,omitempty"`
|
||||
RejectWhy string `json:"rejected-why,omitempty"`
|
||||
Unasked string `json:"unasked,omitempty"`
|
||||
}
|
||||
|
||||
func listRetiring(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, asJSON bool) error {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var rows []retiringRow
|
||||
for _, p := range instances {
|
||||
row := retiringRow{Node: p.Node, Module: p.Module}
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
switch {
|
||||
case isNothingServes(err):
|
||||
row.Unasked = "answers no retirement question: it predates ADR 0230"
|
||||
case err != nil:
|
||||
row.Unasked = err.Error()
|
||||
default:
|
||||
if state.Waiting != nil {
|
||||
row.Waiting, row.Since, row.Held = state.Waiting.Consumers, state.Waiting.Since, state.Waiting.Held
|
||||
}
|
||||
if state.Rejected != nil {
|
||||
row.Rejected, row.RejectWhy = state.Rejected.Consumers, orWhy(state.Rejected.By, state.Rejected.Why)
|
||||
}
|
||||
row.Bound = state.Bound
|
||||
if row.Waiting == nil && row.Rejected == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
if asJSON {
|
||||
if rows == nil {
|
||||
rows = []retiringRow{}
|
||||
}
|
||||
return printJSON(map[string]any{"providers": rows})
|
||||
}
|
||||
said := false
|
||||
for _, r := range rows {
|
||||
switch {
|
||||
case r.Unasked != "":
|
||||
fmt.Printf("%s on %s: not asked — %s\n", r.Module, r.Node, r.Unasked)
|
||||
default:
|
||||
if r.Waiting != nil {
|
||||
said = true
|
||||
fmt.Printf("%s on %s WAITS since %s to retire %d of the %d it holds (%s): %s\n"+
|
||||
" retire approve %s %s --why … | retire reject %s %s --why …\n",
|
||||
r.Module, r.Node, r.Since, len(r.Waiting), r.Held, orBound(r.Bound), consumerList(r.Waiting),
|
||||
r.Node, r.Module, r.Node, r.Module)
|
||||
}
|
||||
if r.Rejected != nil {
|
||||
said = true
|
||||
fmt.Printf("%s on %s keeps active, by a rejection (%s): %s\n", r.Module, r.Node, r.RejectWhy,
|
||||
consumerList(r.Rejected))
|
||||
}
|
||||
}
|
||||
}
|
||||
if !said {
|
||||
fmt.Println("no provider waits for a person to approve a retirement")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// answerRetirement approves or rejects what one provider waits with. **The set sent is the set the
|
||||
// provider says it waits with, read now**, and the provider refuses any other: a person approves what
|
||||
// they were shown, never a set that moved since.
|
||||
func answerRetirement(ctx context.Context, conn *nats.Conn, p providerInstance, approve bool, f handActFlags) error {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
verb, tool := "retire reject", link.ToolRetireReject
|
||||
var set []link.RetiredConsumer
|
||||
if state.Waiting != nil {
|
||||
set = state.Waiting.Consumers
|
||||
}
|
||||
if approve {
|
||||
verb, tool = "retire approve", link.ToolRetireApprove
|
||||
if set == nil && state.Rejected != nil {
|
||||
// A rejection can be taken back: the consumers it kept are retired after all.
|
||||
set = state.Rejected.Consumers
|
||||
}
|
||||
}
|
||||
if len(set) == 0 {
|
||||
return fmt.Errorf("%s on %s waits for nobody to approve or reject a retirement. Nothing was done", p.Module, p.Node)
|
||||
}
|
||||
names := make([]string, 0, len(set))
|
||||
for _, c := range set {
|
||||
names = append(names, c.Consumer)
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
*f.cause = kindRetireWaiting
|
||||
}
|
||||
if strings.TrimSpace(*f.condition) == "" {
|
||||
*f.condition = retireWaitingKey(p.Module, p.Node)
|
||||
}
|
||||
f.record(ctx, verb, append([]string{p.Node, p.Module}, names...))
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, tool, p.Node, map[string]any{
|
||||
"consumers": names, "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController,
|
||||
}, retirementAsk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s on %s refused: %s", p.Module, p.Node, answer.Error)
|
||||
}
|
||||
if approve && state.RetiresBy == "mark-only" {
|
||||
fmt.Printf("%s on %s marked %s retired, MARK ONLY: this provider cannot disable a consumer, so they keep "+
|
||||
"their access until `cleanup delete`; `cleanup list` shows them\n", p.Module, p.Node, strings.Join(names, ", "))
|
||||
} else if approve {
|
||||
fmt.Printf("%s on %s retired %s: access disabled, data kept; `cleanup list` shows them\n", p.Module, p.Node,
|
||||
strings.Join(names, ", "))
|
||||
} else {
|
||||
fmt.Printf("%s on %s keeps %s active; the warning stays open until the mesh asks for them again or the "+
|
||||
"retirement is approved\n", p.Module, p.Node, strings.Join(names, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupCommand is `cleanup list` and `cleanup delete`.
|
||||
func cleanupCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
set := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
listing, err := gatherRetired(ctx, open.inventory, conn, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return printRetired(listing, *asJSON)
|
||||
})
|
||||
case "delete":
|
||||
set := flag.NewFlagSet("cleanup delete", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
olderThan := set.Int("older-than", 0, "every retired consumer older than this many days")
|
||||
confirm := set.Bool("confirm", false, "with --older-than: delete what is listed, rather than only list it")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.require("cleanup delete"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
*f.cause = kindCleanupWaiting
|
||||
}
|
||||
switch {
|
||||
case *olderThan > 0 && len(rest) == 0:
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now())
|
||||
})
|
||||
case *olderThan == 0 && len(rest) == 3 && !*confirm:
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
// A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a
|
||||
// provider's retired consumer otherwise.
|
||||
if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil &&
|
||||
r.RetiredAt != nil {
|
||||
return deleteRetiredData(ctx, conn, open, r, f)
|
||||
}
|
||||
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
|
||||
})
|
||||
}
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
|
||||
// retiredRow is one retired consumer, as `cleanup list` shows it.
|
||||
type retiredRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Consumer string `json:"consumer"`
|
||||
// ConsumerNode is where the consumer was, when the provider knows.
|
||||
ConsumerNode string `json:"consumer-node,omitempty"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
RetiredAt string `json:"retired-at,omitempty"`
|
||||
// AgeDays is whole days since it was retired; -1 when the provider could not say when.
|
||||
AgeDays int `json:"age-days"`
|
||||
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
|
||||
Access string `json:"access,omitempty"`
|
||||
// Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is
|
||||
// kept on its machine, and its class. Consumer is then the item.
|
||||
Path string `json:"path,omitempty"`
|
||||
Class string `json:"class,omitempty"`
|
||||
}
|
||||
|
||||
// retiredListing is every provider's retired consumers, and the providers that could not say.
|
||||
type retiredListing struct {
|
||||
Retired []retiredRow `json:"retired"`
|
||||
// Unasked names each provider not asked, and why: one older than the question, or one that failed.
|
||||
Unasked []string `json:"unasked,omitempty"`
|
||||
}
|
||||
|
||||
func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, now time.Time) (retiredListing, error) {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return retiredListing{}, err
|
||||
}
|
||||
listing := retiredOf(ctx, conn, instances, now)
|
||||
// And every module's own data retired on its machine (novox/hq ADR 0233).
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
return retiredListing{}, err
|
||||
}
|
||||
listing.Retired = append(listing.Retired, retiredData(records, now)...)
|
||||
sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays })
|
||||
return listing, nil
|
||||
}
|
||||
|
||||
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
|
||||
out := retiredListing{Retired: []retiredRow{}}
|
||||
for _, p := range instances {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
if isNothingServes(err) {
|
||||
out.Unasked = append(out.Unasked, fmt.Sprintf("%s on %s answers no retirement question: it predates ADR 0230", p.Module, p.Node))
|
||||
} else {
|
||||
out.Unasked = append(out.Unasked, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, c := range state.Retired {
|
||||
row := retiredRow{Node: p.Node, Module: p.Module, Consumer: c.Consumer, ConsumerNode: c.Node, Kind: c.Kind,
|
||||
RetiredAt: c.RetiredAt, AgeDays: -1, SizeBytes: c.SizeBytes, Why: c.Why, Access: c.Access}
|
||||
if at := retiredAt(c); !at.IsZero() {
|
||||
row.AgeDays = int(now.Sub(at).Hours() / 24)
|
||||
}
|
||||
out.Retired = append(out.Retired, row)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out.Retired, func(i, j int) bool { return out.Retired[i].AgeDays > out.Retired[j].AgeDays })
|
||||
return out
|
||||
}
|
||||
|
||||
func printRetired(l retiredListing, asJSON bool) error {
|
||||
if asJSON {
|
||||
return printJSON(l)
|
||||
}
|
||||
if len(l.Retired) == 0 {
|
||||
fmt.Println("no provider holds a retired consumer, and no machine holds retired data")
|
||||
}
|
||||
for _, r := range l.Retired {
|
||||
age := "age unknown"
|
||||
if r.AgeDays >= 0 {
|
||||
age = strconv.Itoa(r.AgeDays) + " day(s)"
|
||||
}
|
||||
kind := ""
|
||||
if r.Kind != "" && r.Kind != "consumer" {
|
||||
kind = " [" + r.Kind + "]"
|
||||
}
|
||||
if r.Access == "kept" {
|
||||
kind += " [MARK ONLY: access kept until deleted]"
|
||||
}
|
||||
if r.Kind == retiredDataKind {
|
||||
fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer,
|
||||
r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
|
||||
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
||||
}
|
||||
for _, u := range l.Unasked {
|
||||
fmt.Printf("not asked: %s\n", u)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteRetired asks one provider to delete one consumer it holds retired — never an active one: the
|
||||
// provider refuses that, and this refuses it first, from what the provider says it holds.
|
||||
func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, consumer string, f handActFlags) error {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found := false
|
||||
for _, c := range state.Retired {
|
||||
found = found || c.Consumer == consumer
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("%s on %s holds no retired consumer %s — only a retired consumer is deleted. Nothing was done",
|
||||
p.Module, p.Node, consumer)
|
||||
}
|
||||
f.record(ctx, "cleanup delete", []string{p.Node, p.Module, consumer})
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetiredDelete, p.Node, map[string]any{
|
||||
"consumer": consumer, "confirm": consumer, "why": strings.TrimSpace(*f.why), "by": link.Caller(),
|
||||
"via": link.ViaController,
|
||||
}, retirementAsk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s on %s refused to delete %s: %s", p.Module, p.Node, consumer, answer.Error)
|
||||
}
|
||||
var done struct {
|
||||
FreedBytes *int64 `json:"freed_bytes"`
|
||||
}
|
||||
_ = unmarshalAnswer(answer, &done)
|
||||
fmt.Printf("%s on %s deleted %s (%s freed)\n", p.Module, p.Node, consumer, sizeWords(done.FreedBytes))
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
|
||||
// One whose age the provider cannot say is never in it.
|
||||
func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool,
|
||||
f handActFlags, now time.Time) error {
|
||||
listing, err := gatherRetired(ctx, open.inventory, conn, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteFrom(ctx, conn, open, listing, days, confirm, f)
|
||||
}
|
||||
|
||||
func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
||||
var due []retiredRow
|
||||
unknown := 0
|
||||
for _, r := range listing.Retired {
|
||||
switch {
|
||||
case r.AgeDays < 0:
|
||||
unknown++
|
||||
case r.AgeDays > days:
|
||||
due = append(due, r)
|
||||
}
|
||||
}
|
||||
for _, u := range listing.Unasked {
|
||||
fmt.Printf("not asked: %s\n", u)
|
||||
}
|
||||
if unknown > 0 {
|
||||
fmt.Printf("%d retired consumer(s) whose age their provider cannot say are left out\n", unknown)
|
||||
}
|
||||
if len(due) == 0 {
|
||||
fmt.Printf("nothing has been retired more than %d day(s)\n", days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("retired more than %d day(s):\n", days)
|
||||
for _, r := range due {
|
||||
fmt.Printf(" %s on %s: %s — %d day(s), %s\n", r.Module, r.Node, r.Consumer, r.AgeDays, sizeWords(r.SizeBytes))
|
||||
}
|
||||
if !confirm {
|
||||
fmt.Printf("nothing was deleted: add --confirm to delete these %d\n", len(due))
|
||||
return nil
|
||||
}
|
||||
var failed []string
|
||||
for _, r := range due {
|
||||
var err error
|
||||
if r.Kind == retiredDataKind {
|
||||
var rec inventory.DataRecord
|
||||
if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil {
|
||||
err = deleteRetiredData(ctx, conn, open, rec, f)
|
||||
}
|
||||
} else {
|
||||
err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f)
|
||||
}
|
||||
if err != nil {
|
||||
failed = append(failed, err.Error())
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d of %d not deleted: %s", len(failed), len(due), strings.Join(failed, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
|
||||
// (novox/hq ADR 0230, the operator's decision of 2026-10-06; it replaces ADR 0229's withdrawal brake).
|
||||
//
|
||||
// A provider retires a consumer — disables its access, keeps its data, marks it with when and why —
|
||||
// once it has seen the same consumers go unasked for in five passes. More than three at once, or more
|
||||
// than half of those it holds where it holds more than one, is a person actively working on the mesh,
|
||||
// so the provider retires nothing and waits: the controller keeps that as an urgent condition naming
|
||||
// what would go and the two verbs that answer it. A retired consumer is deleted only by `cleanup
|
||||
// delete`, which the provider executes on its own backend — the controller never touches one — and
|
||||
// one retired more than thirty days is a warning that cleanup is waiting (D11).
|
||||
|
||||
// The kinds a provider's retirement word raises.
|
||||
const (
|
||||
kindRetireWaiting = "retire-waiting"
|
||||
kindRetireRejected = "retire-rejected"
|
||||
kindCleanupWaiting = "cleanup-waiting"
|
||||
// sourceRetirement is what raised a retirement condition: the provider's own event.
|
||||
sourceRetirement = "provisioner.retirement"
|
||||
)
|
||||
|
||||
// cleanupAfter is how long a consumer may stay retired before cleanup is said to be waiting (D11).
|
||||
const cleanupAfter = 30 * 24 * time.Hour
|
||||
|
||||
// retirementAsk is how long one provider is given to answer one question about its retired consumers.
|
||||
var retirementAsk = 20 * time.Second
|
||||
|
||||
func retireWaitingKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeProvider, module+"."+node, "retire")
|
||||
}
|
||||
|
||||
func retireRejectedKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeProvider, module+"."+node, "retire-rejected")
|
||||
}
|
||||
|
||||
// retirements keeps what providers say about consumers they retire, as conditions.
|
||||
type retirements struct {
|
||||
keeper func() *conditions.Keeper
|
||||
// record writes an act done by hand that reached a provider some other way than this controller's
|
||||
// verbs; nil records nothing (a test).
|
||||
record func(ctx context.Context, act link.HandAct) error
|
||||
}
|
||||
|
||||
func consumerList(cs []link.RetiredConsumer) string {
|
||||
parts := make([]string, 0, len(cs))
|
||||
for _, c := range cs {
|
||||
p := c.Consumer
|
||||
if c.Node != "" {
|
||||
p += " (" + c.Node + ")"
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// waitingObservation is a provider waiting for a person to approve or reject a retirement.
|
||||
func waitingObservation(r link.Retirement) conditions.Observation {
|
||||
since := r.Since
|
||||
if since.IsZero() {
|
||||
since = r.At
|
||||
}
|
||||
summary := fmt.Sprintf("%s on %s would retire %d consumer(s) the mesh no longer asks for — %s — more than its "+
|
||||
"bound (%s), so it retires nothing until a person answers: `retire approve %s %s --why …` or `retire reject "+
|
||||
"%s %s --why …`", r.Module, r.ProviderNode, len(r.Consumers), consumerList(r.Consumers), orBound(r.Bound),
|
||||
r.ProviderNode, r.Module, r.ProviderNode, r.Module)
|
||||
said := fmt.Sprintf("waiting since %s, %d held: %s", since.UTC().Format("2006-01-02 15:04 MST"), r.Held,
|
||||
consumerList(r.Consumers))
|
||||
return conditions.Observation{Scope: conditions.ScopeProvider, ID: r.Module + "." + r.ProviderNode,
|
||||
Token: "retire", Kind: kindRetireWaiting, Machine: r.ProviderNode, Also: consumerNodes(r),
|
||||
Severity: conditions.Urgent, Summary: summary, Said: said, Source: sourceRetirement,
|
||||
Resolver: conditions.ResolverOperator}
|
||||
}
|
||||
|
||||
// rejectedObservation is consumers kept active by a person's rejection though the mesh asks for them no more.
|
||||
func rejectedObservation(r link.Retirement) conditions.Observation {
|
||||
summary := fmt.Sprintf("%s on %s keeps %s active although the mesh no longer asks for them: a person rejected "+
|
||||
"their retirement (%s). Assign them again, or `retire approve %s %s --why …`", r.Module, r.ProviderNode,
|
||||
consumerList(r.Consumers), orWhy(r.By, r.Why), r.ProviderNode, r.Module)
|
||||
return conditions.Observation{Scope: conditions.ScopeProvider, ID: r.Module + "." + r.ProviderNode,
|
||||
Token: "retire-rejected", Kind: kindRetireRejected, Machine: r.ProviderNode, Also: consumerNodes(r),
|
||||
Severity: conditions.Warning, Summary: summary, Said: "rejected: " + orWhy(r.By, r.Why),
|
||||
Source: sourceRetirement, Resolver: conditions.ResolverOperator}
|
||||
}
|
||||
|
||||
func orBound(b string) string {
|
||||
if b == "" {
|
||||
return "more than 3, or more than half of those held"
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func orWhy(by, why string) string {
|
||||
switch {
|
||||
case by != "" && why != "":
|
||||
return by + ": " + why
|
||||
case why != "":
|
||||
return why
|
||||
case by != "":
|
||||
return "by " + by
|
||||
}
|
||||
return "no reason given"
|
||||
}
|
||||
|
||||
// consumerNodes are the other machines a retirement concerns: where its consumers are.
|
||||
func consumerNodes(r link.Retirement) []string {
|
||||
seen := map[string]bool{r.ProviderNode: true}
|
||||
var out []string
|
||||
for _, c := range r.Consumers {
|
||||
if c.Node != "" && !seen[c.Node] {
|
||||
seen[c.Node] = true
|
||||
out = append(out, c.Node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Retired keeps one word. Waiting raises the urgent condition; a rejection turns it into a warning; a
|
||||
// retirement, an approval or the set settling clears both. An approval, a rejection or a deletion that
|
||||
// did not come through this controller's verbs is recorded in the hand-act log here, so every one is.
|
||||
func (s retirements) Retired(ctx context.Context, r link.Retirement) error {
|
||||
k := s.keeper()
|
||||
if k == nil {
|
||||
return fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
|
||||
}
|
||||
waiting, rejected := retireWaitingKey(r.Module, r.ProviderNode), retireRejectedKey(r.Module, r.ProviderNode)
|
||||
clear := func(keys ...string) error {
|
||||
for _, key := range keys {
|
||||
if _, err := k.Clear(ctx, key, fmt.Sprintf("%s on %s says %s", r.Module, r.ProviderNode, r.Change)); err != nil {
|
||||
return storeAway(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
switch r.Change {
|
||||
case link.RetireWaiting:
|
||||
if _, err = k.Observe(ctx, waitingObservation(r)); err != nil {
|
||||
return storeAway(err)
|
||||
}
|
||||
case link.RetireRejected:
|
||||
if err = clear(waiting); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = k.Observe(ctx, rejectedObservation(r)); err != nil {
|
||||
return storeAway(err)
|
||||
}
|
||||
case link.RetireApproved, link.RetireRetired, link.RetireSettled:
|
||||
if err = clear(waiting, rejected); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch r.Change {
|
||||
case link.RetireApproved, link.RetireRejected, link.RetireDeleted:
|
||||
if r.Via != link.ViaController && s.record != nil {
|
||||
verb := map[string]string{link.RetireApproved: "retire approve", link.RetireRejected: "retire reject",
|
||||
link.RetireDeleted: "cleanup delete"}[r.Change]
|
||||
cause := kindRetireWaiting
|
||||
if r.Change == link.RetireDeleted {
|
||||
cause = kindCleanupWaiting
|
||||
}
|
||||
why := r.Why
|
||||
if strings.TrimSpace(why) == "" {
|
||||
why = "no reason given to the provider"
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: append([]string{r.ProviderNode, r.Module}, r.Names()...),
|
||||
Why: why + " (asked of the provider directly, not through the controller)", Cause: cause,
|
||||
By: r.By, At: r.At}
|
||||
if act.By == "" {
|
||||
act.By = "unknown, asked of " + r.Module + " on " + r.ProviderNode + " directly"
|
||||
}
|
||||
if err := s.record(ctx, act); err != nil {
|
||||
return fmt.Errorf("%s on %s %s by hand, and it could not be recorded: %v: %w", r.Module,
|
||||
r.ProviderNode, r.Change, err, link.ErrTryAgain)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordHandActOnTheBus writes an act through the serving controller's connection.
|
||||
func recordHandActOnTheBus(ctx context.Context, act link.HandAct) error {
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// providerInstance is one provider module on one machine.
|
||||
type providerInstance struct {
|
||||
Node, Module string
|
||||
}
|
||||
|
||||
// providerInstances are every module assigned on every machine whose manifest receives contributions:
|
||||
// every provider, which serves the retirement tools (ADR 0230) — or is older than them.
|
||||
func providerInstances(ctx context.Context, inv *inventory.Inventory) ([]providerInstance, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []providerInstance
|
||||
for _, n := range nodes {
|
||||
modules, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what %s is assigned cannot be read: %w", n.Name, err)
|
||||
}
|
||||
for _, m := range modules {
|
||||
if man, ok := shelf[m]; ok && len(man.Receives) > 0 {
|
||||
out = append(out, providerInstance{Node: n.Name, Module: m})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Node != out[j].Node {
|
||||
return out[i].Node < out[j].Node
|
||||
}
|
||||
return out[i].Module < out[j].Module
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// askRetirement asks one provider for what it holds retired and what waits.
|
||||
func askRetirement(ctx context.Context, conn *nats.Conn, p providerInstance) (link.RetirementState, error) {
|
||||
var state link.RetirementState
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetirement, p.Node, map[string]any{}, retirementAsk)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return state, fmt.Errorf("%s on %s answered %s with an error: %s", p.Module, p.Node, link.ToolRetirement, answer.Error)
|
||||
}
|
||||
if err := unmarshalAnswer(answer, &state); err != nil {
|
||||
return state, fmt.Errorf("%s on %s answered %s with something unreadable: %w", p.Module, p.Node, link.ToolRetirement, err)
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// retiredAt reads a retired consumer's moment; zero when the provider could not say.
|
||||
func retiredAt(c link.RetiredConsumer) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, c.RetiredAt)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// cleanupObservation is a provider holding consumers retired longer than cleanupAfter.
|
||||
func cleanupObservation(p providerInstance, old []link.RetiredConsumer, now time.Time) conditions.Observation {
|
||||
parts := make([]string, 0, len(old))
|
||||
for _, c := range old {
|
||||
parts = append(parts, fmt.Sprintf("%s (%d days, %s)", c.Consumer, int(now.Sub(retiredAt(c)).Hours()/24),
|
||||
sizeWords(c.SizeBytes)))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeProvider, ID: p.Module + "." + p.Node, Token: "cleanup",
|
||||
Kind: kindCleanupWaiting, Machine: p.Node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s on %s holds %d consumer(s) retired more than %d days, waiting for a person to "+
|
||||
"delete or bring them back: %s — `cleanup list`, then `cleanup delete %s %s <consumer> --why …`",
|
||||
p.Module, p.Node, len(old), int(cleanupAfter.Hours()/24), strings.Join(parts, ", "), p.Node, p.Module),
|
||||
Resolver: conditions.ResolverOperator}
|
||||
}
|
||||
|
||||
func sizeWords(size *int64) string {
|
||||
if size == nil || *size < 0 {
|
||||
return "size unknown"
|
||||
}
|
||||
b := float64(*size)
|
||||
for _, unit := range []string{"B", "KB", "MB", "GB"} {
|
||||
if b < 1024 || unit == "GB" {
|
||||
if unit == "B" {
|
||||
return fmt.Sprintf("%d B", *size)
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", b, unit)
|
||||
}
|
||||
b /= 1024
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// probeRetired is D11: no provider holds a consumer retired more than thirty days. Every provider
|
||||
// assigned is asked what it holds retired; one that does not serve the question — an older build, a
|
||||
// TypeScript provider not yet on the SDK that retires — has nothing it can say and is passed over,
|
||||
// which is not a failure of the probe. A provider that cannot be asked otherwise is.
|
||||
func probeRetired(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, fmt.Errorf("no bus to ask the providers over")
|
||||
}
|
||||
instances, err := providerInstances(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return retiredTooLong(ctx, d.js.Conn(), instances, time.Now())
|
||||
}
|
||||
|
||||
// retiredTooLong asks each provider and answers one observation per provider holding anything retired
|
||||
// longer than cleanupAfter.
|
||||
func retiredTooLong(ctx context.Context, conn *nats.Conn, instances []providerInstance,
|
||||
now time.Time) ([]conditions.Observation, error) {
|
||||
var out []conditions.Observation
|
||||
var problems []string
|
||||
for _, p := range instances {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
if isNothingServes(err) {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, err.Error())
|
||||
continue
|
||||
}
|
||||
var old []link.RetiredConsumer
|
||||
for _, c := range state.Retired {
|
||||
if at := retiredAt(c); !at.IsZero() && now.Sub(at) > cleanupAfter {
|
||||
old = append(old, c)
|
||||
}
|
||||
}
|
||||
if len(old) > 0 {
|
||||
out = append(out, cleanupObservation(p, old, now))
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
// Not "nothing retired": a provider that could not be asked may hold the oldest of all.
|
||||
return nil, fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
|
||||
// (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that
|
||||
// answer it ask the provider — never anything else — for exactly what it said.
|
||||
|
||||
func waitingWord(module, node string, names ...string) link.Retirement {
|
||||
r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting,
|
||||
Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()}
|
||||
for _, n := range names {
|
||||
r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"})
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
all, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range all {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
var recorded []link.HandAct
|
||||
r := retirements{keeper: func() *conditions.Keeper { return k },
|
||||
record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }}
|
||||
ctx := t.Context()
|
||||
waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d")
|
||||
if err := r.Retired(ctx, waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open := openKeys(t, k)
|
||||
c, ok := open["provider.postgres.anchor.retire"]
|
||||
if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent {
|
||||
t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open)
|
||||
}
|
||||
for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres",
|
||||
"retire reject anchor postgres", "more than 3"} {
|
||||
if !strings.Contains(c.Summary, want) {
|
||||
t.Errorf("the condition does not say %q: %s", want, c.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here —
|
||||
// the verb recorded it before it asked.
|
||||
rejected := waiting
|
||||
rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController
|
||||
if err := r.Retired(ctx, rejected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open = openKeys(t, k)
|
||||
if _, still := open["provider.postgres.anchor.retire"]; still {
|
||||
t.Fatal("a rejection left the provider waiting")
|
||||
}
|
||||
if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning ||
|
||||
c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") {
|
||||
t.Fatalf("a rejection is not a warning saying why: %+v", open)
|
||||
}
|
||||
if len(recorded) != 0 {
|
||||
t.Fatalf("an act through the controller was recorded twice: %+v", recorded)
|
||||
}
|
||||
|
||||
// Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here.
|
||||
approved := waiting
|
||||
approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", ""
|
||||
if err := r.Retired(ctx, approved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open := openKeys(t, k); len(open) != 0 {
|
||||
t.Fatalf("an approval left conditions open: %+v", open)
|
||||
}
|
||||
if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" ||
|
||||
!strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") {
|
||||
t.Fatalf("an approval outside the controller was not recorded: %+v", recorded)
|
||||
}
|
||||
|
||||
// Waiting again, then the set settles (asked for again): cleared. And retired clears too.
|
||||
for _, change := range []string{link.RetireSettled, link.RetireRetired} {
|
||||
if err := r.Retired(ctx, waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := waiting
|
||||
done.Change = change
|
||||
if err := r.Retired(ctx, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open := openKeys(t, k); len(open) != 0 {
|
||||
t.Fatalf("%s left conditions open: %+v", change, open)
|
||||
}
|
||||
}
|
||||
|
||||
// A deletion asked of the provider directly is recorded too.
|
||||
deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x",
|
||||
Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}}
|
||||
if err := r.Retired(ctx, deleted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting {
|
||||
t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) {
|
||||
body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting",
|
||||
"held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"})
|
||||
r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body)
|
||||
if err != nil || r.Module != "idp" || len(r.Consumers) != 2 {
|
||||
t.Fatalf("%+v %v", r, err)
|
||||
}
|
||||
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement",
|
||||
[]byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil {
|
||||
t.Fatal("a change the mesh has no name for was read")
|
||||
}
|
||||
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil {
|
||||
t.Fatal("a failing word was read as a retirement")
|
||||
}
|
||||
k, _ := withConditionsInMemory(t)
|
||||
if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok {
|
||||
t.Fatal("not kept under the emitter the subject names")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "pg", Version: "1",
|
||||
Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}})
|
||||
if _, err := assign(ctx, open, "anchor", "pg"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||
for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} {
|
||||
if err := r.Retired(ctx, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{
|
||||
cleanupObservation(providerInstance{Node: "anchor", Module: "gone"},
|
||||
[]link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, _ := conditionsFrom.Open(ctx)
|
||||
if len(all) != 3 {
|
||||
t.Fatalf("%+v", all)
|
||||
}
|
||||
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left := openKeys(t, conditionsFrom)
|
||||
if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" {
|
||||
t.Fatalf("only the assigned provider's waiting should stay: %+v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked.
|
||||
type fakeProvider struct {
|
||||
mu sync.Mutex
|
||||
state link.RetirementState
|
||||
asked []map[string]any
|
||||
deleted []string
|
||||
approved []string
|
||||
rejected []string
|
||||
}
|
||||
|
||||
func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) {
|
||||
t.Helper()
|
||||
answer := func(m *nats.Msg, result any, refusal string) {
|
||||
body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node})
|
||||
_ = m.Respond(body)
|
||||
}
|
||||
names := func(args map[string]any) []string {
|
||||
var out []string
|
||||
for _, v := range args["consumers"].([]any) {
|
||||
out = append(out, v.(string))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
set := func(cs []link.RetiredConsumer) []string {
|
||||
var out []string
|
||||
for _, c := range cs {
|
||||
out = append(out, c.Consumer)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} {
|
||||
tool := tool
|
||||
sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var args map[string]any
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
f.asked = append(f.asked, map[string]any{"tool": tool, "args": args})
|
||||
switch tool {
|
||||
case link.ToolRetirement:
|
||||
answer(m, f.state, "")
|
||||
case link.ToolRetireApprove:
|
||||
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
|
||||
answer(m, nil, "not the set I wait with")
|
||||
return
|
||||
}
|
||||
f.approved = names(args)
|
||||
answer(m, map[string]any{"retired": f.approved}, "")
|
||||
case link.ToolRetireReject:
|
||||
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
|
||||
answer(m, nil, "not the set I wait with")
|
||||
return
|
||||
}
|
||||
f.rejected = names(args)
|
||||
answer(m, map[string]any{"kept": f.rejected}, "")
|
||||
case link.ToolRetiredDelete:
|
||||
if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController {
|
||||
answer(m, nil, "confirm, why and via")
|
||||
return
|
||||
}
|
||||
f.deleted = append(f.deleted, args["consumer"].(string))
|
||||
answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "")
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func onATestBus(t *testing.T) *nats.Conn {
|
||||
t.Helper()
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
return js.Conn()
|
||||
}
|
||||
|
||||
func whyFlags(t *testing.T, why string) handActFlags {
|
||||
t.Helper()
|
||||
set := flag.NewFlagSet("t", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
if err := set.Parse([]string{"--why", why}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct {
|
||||
t.Helper()
|
||||
acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []link.HandAct
|
||||
for _, a := range acts {
|
||||
if a.Verb == verb {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func retiredDaysAgo(name string, days int) link.RetiredConsumer {
|
||||
size := int64(4096)
|
||||
return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it",
|
||||
RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size}
|
||||
}
|
||||
|
||||
func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) {
|
||||
conn := onATestBus(t)
|
||||
fake := &fakeProvider{}
|
||||
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6}
|
||||
fake.serve(t, conn, "pg-approve", "anchor")
|
||||
p := providerInstance{Node: "anchor", Module: "pg-approve"}
|
||||
said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) })
|
||||
if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") {
|
||||
t.Fatalf("approved %v; said %s", fake.approved, said)
|
||||
}
|
||||
acts := handActsBy(t, conn, "retire approve")
|
||||
if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting ||
|
||||
acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" {
|
||||
t.Fatalf("the approval is not in the hand-act log: %+v", acts)
|
||||
}
|
||||
|
||||
// Reject, on another provider: the same set, and nothing approved.
|
||||
other := &fakeProvider{state: fake.state}
|
||||
other.serve(t, conn, "pg-reject", "anchor")
|
||||
printed(t, func() error {
|
||||
return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false,
|
||||
whyFlags(t, "still moving"))
|
||||
})
|
||||
if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil {
|
||||
t.Fatalf("rejected %v approved %v", other.rejected, other.approved)
|
||||
}
|
||||
|
||||
// Nothing waiting: refused, nothing asked but the question.
|
||||
idle := &fakeProvider{}
|
||||
idle.serve(t, conn, "pg-idle", "anchor")
|
||||
if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true,
|
||||
whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if len(idle.asked) != 1 {
|
||||
t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
|
||||
conn := onATestBus(t)
|
||||
fake := &fakeProvider{}
|
||||
fake.state.Held = []string{"active"}
|
||||
fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)}
|
||||
fake.serve(t, conn, "pg-clean", "anchor")
|
||||
p := providerInstance{Node: "anchor", Module: "pg-clean"}
|
||||
said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) })
|
||||
if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") {
|
||||
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
||||
}
|
||||
// An active consumer, or one it does not hold, is refused before the provider is asked to delete.
|
||||
for _, name := range []string{"active", "nobody"} {
|
||||
if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil ||
|
||||
!strings.Contains(err.Error(), "only a retired consumer is deleted") {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(fake.deleted, []string{"old"}) {
|
||||
t.Fatalf("more was deleted: %v", fake.deleted)
|
||||
}
|
||||
if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" {
|
||||
t.Fatalf("the deletion is not in the hand-act log: %+v", acts)
|
||||
}
|
||||
|
||||
// Older than: listed, and nothing deleted without confirm; with it, only the old one.
|
||||
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
|
||||
if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 {
|
||||
t.Fatalf("%+v", listing)
|
||||
}
|
||||
fake.deleted = nil
|
||||
said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
|
||||
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
|
||||
strings.Contains(said, "young") {
|
||||
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
||||
}
|
||||
printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
|
||||
if !slices.Equal(fake.deleted, []string{"old"}) {
|
||||
t.Fatalf("confirmed, deleted %v", fake.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) {
|
||||
conn := onATestBus(t)
|
||||
old := &fakeProvider{}
|
||||
old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)}
|
||||
old.serve(t, conn, "pg-d11-old", "anchor")
|
||||
young := &fakeProvider{}
|
||||
young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)}
|
||||
young.serve(t, conn, "pg-d11-young", "anchor")
|
||||
instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"},
|
||||
// Nothing serves this one: a provider older than the question is passed over, not a failure.
|
||||
{Node: "anchor", Module: "pg-d11-predates"}}
|
||||
found, err := retiredTooLong(t.Context(), conn, instances, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting ||
|
||||
found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") ||
|
||||
strings.Contains(found[0].Summary, "fresh") {
|
||||
t.Fatalf("%+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"retire", map[string]any{}, "retire --json"},
|
||||
{"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"},
|
||||
"retire approve anchor postgres --why moved"},
|
||||
{"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"},
|
||||
"retire reject anchor postgres --why no"},
|
||||
{"cleanup", map[string]any{}, "cleanup list --json"},
|
||||
{"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"},
|
||||
"cleanup delete anchor postgres x --why gone"},
|
||||
{"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"},
|
||||
{"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"},
|
||||
"cleanup delete --older-than 30 --why tidy --confirm"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{
|
||||
{"answer": "approve", "node": "anchor", "module": "postgres"}, // no why
|
||||
{"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"},
|
||||
} {
|
||||
if _, err := argvFor("retire", args); err == nil {
|
||||
t.Errorf("retire %v was composed", args)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{
|
||||
{"node": "anchor", "module": "postgres", "consumer": "x"}, // no why
|
||||
{"older-than": "30"},
|
||||
{"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"},
|
||||
} {
|
||||
if _, err := argvFor("cleanup", args); err == nil {
|
||||
t.Errorf("cleanup %v was composed", args)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" ||
|
||||
repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" {
|
||||
t.Error("the generic verb would let a retirement or a deletion through without a why")
|
||||
}
|
||||
}
|
||||
|
||||
// A mark-only provider's retired consumer keeps its access; the listing and the approval say so
|
||||
// rather than claiming it was disabled (ADR 0230).
|
||||
func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) {
|
||||
conn := onATestBus(t)
|
||||
fake := &fakeProvider{}
|
||||
kept := retiredDaysAgo("ledger", 3)
|
||||
kept.Access = "kept"
|
||||
fake.state.Retired = []link.RetiredConsumer{kept}
|
||||
fake.state.RetiresBy = "mark-only"
|
||||
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2}
|
||||
fake.serve(t, conn, "vault-mark", "anchor")
|
||||
p := providerInstance{Node: "anchor", Module: "vault-mark"}
|
||||
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
|
||||
said := printed(t, func() error { return printRetired(listing, false) })
|
||||
if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" {
|
||||
t.Fatalf("%s %+v", said, listing)
|
||||
}
|
||||
said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) })
|
||||
if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") {
|
||||
t.Fatal(said)
|
||||
}
|
||||
}
|
||||
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
|
||||
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||
// would bury the ones that matter under a list nobody can act on.
|
||||
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||
}
|
||||
|
||||
@@ -388,14 +388,7 @@ func rolloutMint(ctx context.Context, again bool) error {
|
||||
}
|
||||
|
||||
// providesBus is whether a manifest provides the mesh's bus.
|
||||
func providesBus(m catalogue.Manifest) bool {
|
||||
for _, o := range m.Provides {
|
||||
if o.Name == "mesh-bus" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func providesBus(m catalogue.Manifest) bool { return catalogue.ProvidesBus(m) }
|
||||
|
||||
// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
|
||||
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only
|
||||
// once that machine has reported it applied; a module whose policy says together goes everywhere at
|
||||
// once, as before.
|
||||
func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) {
|
||||
running := []string{"novox", "ace", "g14"}
|
||||
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
now := sentAt.Add(5 * time.Minute)
|
||||
bound := 30 * time.Minute
|
||||
after := sentAt.Add(time.Minute)
|
||||
next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep {
|
||||
return nextRollout(s, running, together, reports, now, bound)
|
||||
}
|
||||
|
||||
// Together: every machine at once.
|
||||
if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first {
|
||||
t.Fatalf("a together policy did not send every machine at once: %+v", step)
|
||||
}
|
||||
|
||||
// Otherwise the first by name, alone, when none has reported lately.
|
||||
step := next(inventory.PlanModule{}, running, false, nil)
|
||||
if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) {
|
||||
t.Fatalf("the first send was %+v, wanted ace alone", step)
|
||||
}
|
||||
|
||||
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||
report := func(outcome string, current bool) []inventory.Reported {
|
||||
return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current},
|
||||
{Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}}
|
||||
}
|
||||
|
||||
// No report yet, or one about an older declaration than it was last sent: wait.
|
||||
for what, reports := range map[string][]inventory.Reported{
|
||||
"no report": nil,
|
||||
"a report about older": report(inventory.OutcomeApplied, false),
|
||||
} {
|
||||
step := next(state, running, false, reports)
|
||||
if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" {
|
||||
t.Errorf("%s: %+v, wanted to wait for ace", what, step)
|
||||
}
|
||||
}
|
||||
|
||||
// Applied what it was last sent: the rest, and only the rest.
|
||||
step = next(state, running, false, report(inventory.OutcomeApplied, true))
|
||||
if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) {
|
||||
t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step)
|
||||
}
|
||||
|
||||
// Failed or refused: stop, the rest untouched.
|
||||
for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} {
|
||||
step := next(state, running, false, report(outcome, true))
|
||||
if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) ||
|
||||
!reflect.DeepEqual(step.rest, []string{"novox", "g14"}) {
|
||||
t.Errorf("a first machine that %s it: %+v", outcome, step)
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus went with the first send: the rest wait for it too, and it is
|
||||
// not sent again.
|
||||
both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt}
|
||||
half := report(inventory.OutcomeApplied, true)
|
||||
if step := next(both, running, false, half); step.waiting != "novox" {
|
||||
t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step)
|
||||
}
|
||||
all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true})
|
||||
if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||
t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step)
|
||||
}
|
||||
|
||||
// One machine, or none: nothing is waited for that cannot come.
|
||||
if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first {
|
||||
t.Fatalf("a module nothing runs was sent: %+v", step)
|
||||
}
|
||||
if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" {
|
||||
t.Fatalf("a module on one machine waited for more: %+v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there,
|
||||
// naming the machine and the bound; the rest are left alone.
|
||||
func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) {
|
||||
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||
step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute)
|
||||
if !strings.Contains(step.failed, "ace did not report it applied within 30m") ||
|
||||
!reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 {
|
||||
t.Fatalf("a silent first machine: %+v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The first machine is the first by name among those heard from lately: a laptop that is away is
|
||||
// not the one the rest wait on. When none has been heard from, the first by name.
|
||||
func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) {
|
||||
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour)
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately},
|
||||
}
|
||||
step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute)
|
||||
if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||
t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step)
|
||||
}
|
||||
}
|
||||
|
||||
// An announced move of a module an open plan is still rolling out is left to the plan: sending it
|
||||
// here as well put the bundle on every machine at once (novox/hq issue 249).
|
||||
func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) {
|
||||
sent := time.Now()
|
||||
plans := []inventory.Plan{
|
||||
{ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}},
|
||||
{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}},
|
||||
}
|
||||
if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" {
|
||||
t.Fatalf("a module the plan is rolling out was not left to it: %q", got)
|
||||
}
|
||||
for _, m := range []string{"gitea", "keycloak"} {
|
||||
if got := rolledOutByAPlan(plans, m); got != "" {
|
||||
t.Errorf("%s, which no plan will send, was left to %s", m, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A plan that ends without sending what it built says so, with the remedy; a module whose rollout
|
||||
// stopped at its first machine is not among them.
|
||||
func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) {
|
||||
at := time.Now()
|
||||
p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1",
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built"},
|
||||
"stopped": {State: "built", First: []string{"ace"}, FirstAt: &at},
|
||||
"sent": {State: "built", SentAt: &at},
|
||||
"notes": {State: "built"},
|
||||
"later": {},
|
||||
}}
|
||||
rollsOut := func(m string) bool { return m != "notes" }
|
||||
sayUnsent(&p, rollsOut)
|
||||
sayUnsent(&p, rollsOut)
|
||||
if p.Note != "closed by hand at tier 1; built and never sent: agent — `push --behind` sends them" {
|
||||
t.Fatalf("the note reads %q", p.Note)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user