Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac0f49fb06 | ||
|
|
a5a132ac15 | ||
|
|
7f25666cee | ||
|
|
4291fee68e | ||
|
|
b98fd0f396 | ||
|
|
863ebd4277 | ||
|
|
2799e95035 | ||
|
|
582f4a082a | ||
|
|
6c7af5c63f | ||
|
|
9d15f3a39e | ||
|
|
725fcd977e | ||
|
|
1cc6a2d759 | ||
|
|
5d3e52219b | ||
|
|
099c176fa9 | ||
|
|
ec3769a8a6 | ||
|
|
8b2abd08cd | ||
|
|
858b4672dd | ||
|
|
3d7ccc8aeb | ||
|
|
b39eaa485a | ||
|
|
0d2fd2c5bb | ||
|
|
a011743c69 | ||
|
|
72d7802415 | ||
|
|
b7d9f44936 | ||
|
|
75213b9091 | ||
|
|
487aa040de | ||
|
|
ba26ba2772 | ||
|
|
3d05d74400 | ||
|
|
58ebe590a5 | ||
|
|
b24bb030ec | ||
|
|
327654e57b | ||
|
|
14127d4878 | ||
|
|
d0580a17e5 | ||
|
|
d6e0a8250a | ||
|
|
9b6b0c5686 | ||
|
|
792352dfad | ||
|
|
b9e0cd34c3 | ||
|
|
1c26235bc1 | ||
|
|
bbd442cdf4 | ||
|
|
042874e0ce | ||
|
|
be92762969 | ||
|
|
9c714f00d6 | ||
|
|
cbce8f96ef | ||
|
|
068283137b | ||
|
|
0090bf6af7 | ||
|
|
58924dc920 | ||
|
|
b1016e5c66 | ||
|
|
b1e02aca1b | ||
|
|
4635341a9d | ||
|
|
dcec6a4db3 | ||
|
|
2bfa6ae4a0 | ||
|
|
41f7b2c152 | ||
|
|
37229b4db5 | ||
|
|
d7bf1bae83 | ||
|
|
c6f3d8cdfa | ||
|
|
d9289ef6d4 | ||
|
|
81f497e5dd | ||
|
|
8e8712e352 | ||
|
|
7aa98e64ce | ||
|
|
48581af35c | ||
|
|
4d05385819 | ||
|
|
dcee8cb5bf | ||
|
|
2b5060789f | ||
|
|
abf9125689 | ||
|
|
52af210e47 | ||
|
|
4b25af2aa3 | ||
|
|
fc65215c25 | ||
|
|
c988d6d7be | ||
|
|
8bfaf1523e | ||
|
|
b037c73fd5 | ||
|
|
18da8b37e9 | ||
|
|
4f4d365360 | ||
|
|
08e11ad761 | ||
|
|
3b2adda1c8 | ||
|
|
68009b16fe | ||
|
|
298daa6fbe | ||
|
|
751e39186c | ||
|
|
20c147ffdb | ||
|
|
2eb9a22c24 | ||
|
|
070ecafc07 | ||
|
|
e51c6a2cb9 | ||
|
|
722682f1c4 | ||
|
|
b853439792 | ||
|
|
9cf47f4429 | ||
|
|
8ddc019cd2 | ||
|
|
fab6b0059e | ||
|
|
e1f5d4fdf0 | ||
|
|
bb1607e424 | ||
|
|
cf4834a36c | ||
|
|
9d8cbe7b81 | ||
|
|
e74c32ed50 | ||
|
|
146c48fd96 | ||
|
|
1f3abd3e0e | ||
|
|
6d620f77c3 | ||
|
|
f8286c063d | ||
|
|
e096b4595a | ||
|
|
09c0c6b367 | ||
|
|
d1fc25f682 | ||
|
|
eda457f415 | ||
|
|
59f4d486b1 | ||
|
|
801552c0eb | ||
|
|
ede9bce6ef | ||
|
|
0f0028785c | ||
|
|
6fdcfad8d3 | ||
|
|
8d9d33ae85 | ||
|
|
cc25baa563 | ||
|
|
68a2ebdcc3 | ||
|
|
69b99eec68 | ||
|
|
09bd0eec4f | ||
|
|
222a38e050 | ||
|
|
ee99a24f77 | ||
|
|
f68521da28 | ||
|
|
296064c799 | ||
|
|
df9231c734 | ||
|
|
843b709b59 | ||
|
|
f506fb34ec | ||
|
|
c34b937dd3 | ||
|
|
d84c9699b3 | ||
|
|
002d5e578c | ||
|
|
2421b82ad2 | ||
|
|
0ebd48a6a8 | ||
|
|
67e291c02a | ||
|
|
8a400d165e | ||
|
|
53d3cd7ce9 | ||
|
|
6648e4a5c8 | ||
|
|
7fa2568ce7 | ||
|
|
4b382ceffd | ||
|
|
ce86d09d22 | ||
|
|
853be00ebe | ||
|
|
e0ce2236dd | ||
|
|
9873b3bf13 | ||
|
|
541603c15c | ||
|
|
106507b1d3 | ||
|
|
e610f2d92c | ||
|
|
f80b6cdbd1 | ||
|
|
5dcf33db45 | ||
|
|
6abce7e956 | ||
|
|
0d2b304f08 | ||
|
|
bdfbd0254f | ||
|
|
16c5e78fa8 | ||
|
|
ed90771382 | ||
|
|
672d1f4ca1 | ||
|
|
208901c6cc | ||
|
|
4ac5cfe3a7 | ||
|
|
22660dc274 | ||
|
|
d7f359c498 | ||
|
|
ed25fd68e2 | ||
|
|
01c5ab2aab | ||
|
|
bb3cd6437b | ||
|
|
0b07e68cb8 | ||
|
|
625d02862c | ||
|
|
e8478e208b | ||
|
|
5761737687 | ||
|
|
89ec48b9a9 | ||
|
|
869fb6d6bf | ||
|
|
d25b69178b | ||
|
|
a7bb1e0b1c | ||
|
|
5eaed84271 | ||
|
|
326b1aec14 | ||
|
|
134d039ff8 | ||
|
|
d90c6ab93a | ||
|
|
6b7d2ec49b | ||
|
|
4ed1057df3 | ||
|
|
1f4c67a01b | ||
|
|
5474ea2d41 | ||
|
|
b7912172af | ||
|
|
b36babcd7d | ||
|
|
49cf0aa562 | ||
|
|
5a4f73f761 | ||
|
|
6af891e358 | ||
|
|
568f55fd2e | ||
|
|
35314175f2 | ||
|
|
ec2e6255a9 | ||
|
|
f3f34a170e | ||
|
|
e2622fd031 | ||
|
|
3ee32970ef | ||
|
|
11b654499b | ||
|
|
d69e19103c | ||
|
|
10f948e970 | ||
|
|
f421d4588c | ||
|
|
74b0dab34c | ||
|
|
41b20b2782 | ||
|
|
912e9f4e85 | ||
|
|
babd7b2f47 | ||
|
|
892dfd1d08 | ||
|
|
cfac579392 | ||
|
|
fe0d295490 | ||
|
|
d8a0238e02 | ||
|
|
dcf710a8d5 | ||
|
|
a2003ab616 | ||
|
|
1363a2fe27 | ||
|
|
f19a2254ac | ||
|
|
7d46e48b26 | ||
|
|
78915f9f7a | ||
|
|
17b8f14fe1 | ||
|
|
42c394acc2 | ||
|
|
b9ad7a2948 | ||
|
|
cde22ff627 | ||
|
|
79993fb498 | ||
|
|
aec55b7072 | ||
|
|
c7884f5a72 | ||
|
|
580c4d66a7 | ||
|
|
63bfc5fda5 | ||
|
|
02e3482eb5 | ||
|
|
294e83dab1 | ||
|
|
b5438bb331 | ||
|
|
c23be73d4d | ||
|
|
d2d171f2d2 | ||
|
|
73fa64ea68 | ||
|
|
1a13dbeb17 | ||
|
|
e11caecdad | ||
|
|
7bb9e55d0b | ||
|
|
00037608ae | ||
|
|
cea59428b1 | ||
|
|
5c832f2d19 | ||
|
|
cdebb7d1a5 | ||
|
|
6a803ea5b3 | ||
|
|
9745c1ab31 | ||
|
|
c0c3c3fed4 | ||
|
|
c1449fffe9 | ||
|
|
f873c97db5 | ||
|
|
b0b3d87fe2 | ||
|
|
ba189e6943 | ||
|
|
cadf74a176 | ||
|
|
74efe8e2e7 | ||
|
|
68af9eff44 | ||
|
|
518eeb7941 | ||
|
|
bf2da878a0 | ||
|
|
50cf253a43 | ||
|
|
980a0dee93 | ||
|
|
ac9c2d57be | ||
|
|
8b016cc62b | ||
|
|
cf2bb3b87d | ||
|
|
473376259b | ||
|
|
e1293fb0ad | ||
|
|
6784efae75 | ||
|
|
d86baebe9a | ||
|
|
82481099b7 | ||
|
|
b127f005c3 | ||
|
|
58b4fcb8c8 | ||
|
|
796f6410c1 | ||
|
|
e67c58cd98 | ||
|
|
85873b19e1 | ||
|
|
2ebbb79937 | ||
|
|
9204190445 | ||
|
|
06ea2168d8 | ||
|
|
2b149dd43e | ||
|
|
17dba2a34c | ||
|
|
11e4bc0ba1 | ||
|
|
e56f3aa1cb | ||
|
|
f966693583 | ||
|
|
5acc763992 | ||
|
|
4f009fff83 | ||
|
|
f27e31954f | ||
|
|
62650cd48c | ||
|
|
408f6dbad9 | ||
|
|
eae0577567 | ||
|
|
de26918c52 | ||
|
|
e01d18e548 | ||
|
|
59166b1031 | ||
|
|
c294949f2a | ||
|
|
28853a251b | ||
|
|
76a8b8df9e | ||
|
|
f86f6a74f0 | ||
|
|
a3e8a4185b | ||
|
|
78de54381b | ||
|
|
ff5ef0ab60 | ||
|
|
a5d6a1187c | ||
|
|
06d0a4bfe8 | ||
|
|
d293a0deaf | ||
|
|
11a44e1ec4 | ||
|
|
28c7f05b39 | ||
|
|
93a0c6202e | ||
|
|
7d82751862 | ||
|
|
905f3363c9 | ||
|
|
9f9d9b3b25 | ||
|
|
bde4b61b3b | ||
|
|
d07018f3c5 | ||
|
|
729a5f9e6c | ||
|
|
773b561f5e | ||
|
|
ca7e81e964 | ||
|
|
08bb56f1d3 | ||
|
|
1a44d281c2 | ||
|
|
1c8fe65601 | ||
|
|
8eb6c3e94a | ||
|
|
9d4d847dc4 | ||
|
|
bea1a1c513 | ||
|
|
b1df688c62 | ||
|
|
21d38c9b0e | ||
|
|
1f86ed4135 | ||
|
|
689dd060b0 | ||
|
|
99c4c4ef04 | ||
|
|
e5e1666f91 | ||
|
|
bd58d1c3ef | ||
|
|
d134c89a7c | ||
|
|
a9307d9f33 | ||
|
|
5eb1c9c2b7 | ||
|
|
37b8edeec6 | ||
|
|
424406b2d6 | ||
|
|
90455c61f6 | ||
|
|
1f6793cf0c | ||
|
|
6ef522fbda | ||
|
|
46f65c12a0 | ||
|
|
8f7c02d77a | ||
|
|
a637df01ea | ||
|
|
252eb786a7 | ||
|
|
9d13b0593b | ||
|
|
30d548a762 | ||
|
|
550e4c6acb | ||
|
|
1587fd97f9 | ||
|
|
b5df244096 | ||
|
|
db47bb68e9 | ||
|
|
db84142d38 | ||
|
|
c9204591bf | ||
|
|
e8e707e013 | ||
|
|
0c003774ba | ||
|
|
fbc3d320ea | ||
|
|
cf495e315f | ||
|
|
86bc1fad2c | ||
|
|
c9eba5f3b8 | ||
|
|
24f024dd74 | ||
|
|
9acb5f1292 | ||
|
|
20516e3fcb | ||
|
|
0a40c046cf | ||
|
|
8f51c66eb1 | ||
|
|
477d3ac9a7 | ||
|
|
6ff449e363 | ||
|
|
3ca1f5d26a | ||
|
|
74a1382164 | ||
|
|
40f169229d | ||
|
|
9a99c422a1 | ||
|
|
73a34cc0a7 | ||
|
|
75932d6f3e | ||
|
|
c430ca76f0 | ||
|
|
ba6a3ea829 | ||
|
|
dcf6278523 | ||
|
|
d94f9e7f9f | ||
|
|
882687afd4 | ||
|
|
884c088949 | ||
|
|
76f27562a1 | ||
|
|
89d43e0dad | ||
|
|
09b636e628 | ||
|
|
d2ed6d50c9 | ||
|
|
fdf338dbd1 | ||
|
|
a69a832248 | ||
|
|
bdcbda801e | ||
|
|
5a7ed56f61 | ||
|
|
d54ecb3bf2 | ||
|
|
cf84117638 | ||
|
|
8d4e940866 | ||
|
|
11b20b10ff | ||
|
|
7e701c0db2 | ||
|
|
853c63b181 | ||
|
|
84ac840ff4 | ||
|
|
e8e502343f | ||
|
|
1edc44b25f | ||
|
|
5a1b37e477 | ||
|
|
4a6a4eadeb | ||
|
|
69f559ab4c | ||
|
|
05b90f966a | ||
|
|
184913b620 | ||
|
|
de7aed5016 | ||
|
|
18958154f0 | ||
|
|
a2b1f9e936 | ||
|
|
c9a0b1f9f4 | ||
|
|
f450303e8e | ||
|
|
603ad61142 | ||
|
|
e7cff3d38e | ||
|
|
55c5c061ab | ||
|
|
ccae1ec303 | ||
|
|
9fd5971212 | ||
|
|
05ccab2e4b | ||
|
|
05ae5e5040 | ||
|
|
988250f37a | ||
|
|
6ca4ba68c8 | ||
|
|
1469f5ff82 | ||
|
|
0e977399d4 | ||
|
|
c462db105e | ||
|
|
7273ca2f0f | ||
|
|
ad797d8742 | ||
|
|
5b39e95361 | ||
|
|
7e4a0ecf9a | ||
|
|
7661f57833 | ||
|
|
076e0ae259 | ||
|
|
a96e2f0d78 | ||
|
|
17f7cb0d9c | ||
|
|
f6685ed22d | ||
|
|
52c18f7a45 | ||
|
|
fa7415fcd0 | ||
|
|
f8947a806d | ||
|
|
b98e503a61 | ||
|
|
5b832918df | ||
|
|
17bbcc1596 | ||
|
|
29be985c23 | ||
|
|
05d977666a | ||
|
|
e871991495 | ||
|
|
f9e19814eb | ||
|
|
af31315a5f | ||
|
|
474f68b34c | ||
|
|
1a724f20fe | ||
|
|
0da0bb2157 | ||
|
|
118e333ff8 | ||
|
|
c1334f3f85 | ||
|
|
70d379816c | ||
|
|
d8e7c13f6d | ||
|
|
1851a15e57 | ||
|
|
d9dbc9a59a | ||
|
|
d5e1332dda | ||
|
|
5ea0e87059 | ||
|
|
8e81266cdc | ||
|
|
70705ffe45 | ||
|
|
91c4da8a82 | ||
|
|
e9df5dccab | ||
|
|
990ef27cd2 | ||
|
|
0a17a9a2eb | ||
|
|
23907984a3 | ||
|
|
f0634e11f4 | ||
|
|
542ce76c0a | ||
|
|
2882b5fcb1 | ||
|
|
481b1a7b05 | ||
|
|
b58578f88d | ||
|
|
6cb285dd5c | ||
|
|
46f324b10b | ||
|
|
d188eec318 | ||
|
|
c978aa7d64 | ||
|
|
0c7f42a18a | ||
|
|
9a5584b1b6 | ||
|
|
1bc099a2db | ||
|
|
3fc1feff38 | ||
|
|
ffe176f6d1 | ||
|
|
8057cc4888 | ||
|
|
9d6dad37c5 | ||
|
|
7f84ddecc5 | ||
|
|
94ab9f665e | ||
|
|
3600f2cf16 | ||
|
|
005bc16c24 | ||
|
|
985e2008ba | ||
|
|
bd7ee12938 | ||
|
|
0983b00284 | ||
|
|
8ee2e4d441 | ||
|
|
1d9c102889 | ||
|
|
2542aa67b0 | ||
|
|
1da96e8803 | ||
|
|
cf2f62cf14 | ||
|
|
7683ba8b5b | ||
|
|
a0d7d72a26 | ||
|
|
bfd983e3f8 | ||
|
|
e7da39de57 | ||
|
|
e6ddc59cde | ||
|
|
6c5dfd0c25 | ||
|
|
775df79893 | ||
|
|
3fbf658c16 | ||
|
|
bc31745607 | ||
|
|
e5c2eb20f2 | ||
|
|
05fb7fb5eb | ||
|
|
2c1733de8d | ||
|
|
e51c94dcb5 | ||
|
|
07c07902ff | ||
|
|
864cdea4c6 | ||
|
|
6e810907b2 | ||
|
|
2b20a12c4a | ||
|
|
9c83dacfce | ||
|
|
64ba053f3b | ||
|
|
96416bd8a7 | ||
|
|
4d2003d77b | ||
|
|
aaad02fd38 | ||
|
|
c68d3a7432 | ||
|
|
a5209bd849 | ||
|
|
bdf965dab6 | ||
|
|
b4da20ecc0 | ||
|
|
4b33b72160 | ||
|
|
d5505fe3d4 | ||
|
|
264c9e41e9 | ||
|
|
76ac3c99bd | ||
|
|
fe5988c536 | ||
|
|
ed5d467d90 | ||
|
|
228d0226dd | ||
|
|
6215ff0760 | ||
|
|
54812306be | ||
|
|
ce9e20fbbc | ||
|
|
878690697e | ||
|
|
ad97297576 | ||
|
|
683b1ed693 | ||
|
|
04f9f378b0 | ||
|
|
1c3f44a526 | ||
|
|
89e152dfe2 | ||
|
|
1ebad3786c | ||
|
|
f2f526a60a | ||
|
|
4b4c7e0e0d | ||
|
|
cec792ce9d | ||
|
|
338d033632 | ||
|
|
1be926cec4 | ||
|
|
2134768dfe | ||
|
|
ef825688ee | ||
|
|
77a14360df | ||
|
|
9be2fb4750 | ||
|
|
5a963aec10 | ||
|
|
45d1c28a28 | ||
|
|
a3e7683c63 | ||
|
|
da394b45e6 | ||
|
|
2f3bfda8c0 | ||
|
|
208388978a | ||
|
|
aa771616bb | ||
|
|
1513bbaac9 | ||
|
|
0014984116 | ||
|
|
d6e49dbd68 | ||
|
|
3756bb3460 | ||
|
|
60be9c5360 | ||
|
|
da31bcb11e | ||
|
|
f03e7b33c9 | ||
|
|
0baf727f36 | ||
|
|
94dd49a968 | ||
|
|
83a298e7e0 | ||
|
|
220b79f5cd | ||
|
|
e62201e227 | ||
|
|
0ab9b86f0a | ||
|
|
c7aabd3037 | ||
|
|
c74d990cee | ||
|
|
35252af665 | ||
|
|
aab6ded41b | ||
|
|
aecac5bda2 | ||
|
|
30362118a1 | ||
|
|
c585158836 |
+13
-5
@@ -1,5 +1,11 @@
|
||||
ARG GO_BASE=golang:1.25-alpine
|
||||
# The control plane's image.
|
||||
# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq
|
||||
# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how
|
||||
# `make image` broke once before (issue 146).
|
||||
ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||
# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go
|
||||
# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it.
|
||||
# Genesis builds this file and raises it as the container the process replaces on the first push
|
||||
# (mesh-host internal/bootstrap, novox/hq issue 223).
|
||||
#
|
||||
# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
|
||||
# machine where no mesh exists yet, and run before there is anything to check it against. So it
|
||||
@@ -15,13 +21,15 @@ ARG GO_BASE=golang:1.25-alpine
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
|
||||
# Dependencies first, so a change to the source does not refetch them.
|
||||
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
|
||||
# the image builds from this repository alone — the host's validator among them, whose module no
|
||||
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY vendor/ vendor/
|
||||
|
||||
COPY . .
|
||||
ARG VERSION=development
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o /mesh-controller ./cmd/mesh-controller
|
||||
|
||||
|
||||
@@ -27,8 +27,22 @@ build:
|
||||
IMAGE ?= mesh-controller:$(VERSION)
|
||||
DEV_TAG ?= mesh-controller:development
|
||||
|
||||
# The Go base the image is built on.
|
||||
#
|
||||
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
|
||||
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
|
||||
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
|
||||
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
|
||||
# what it cost).
|
||||
#
|
||||
# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds
|
||||
# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab —
|
||||
# still needs a Go base. The digest is the one the manifest declared until then.
|
||||
GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||
|
||||
image:
|
||||
docker build --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
@@ -38,7 +52,8 @@ BUILDER_IMAGE ?= mesh-builder:$(VERSION)
|
||||
BUILDER_DEV_TAG ?= mesh-builder:development
|
||||
|
||||
builder-image:
|
||||
docker build -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
@@ -48,7 +63,7 @@ PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
|
||||
PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development
|
||||
|
||||
provisioner-image:
|
||||
docker build -f examples/postgres-provisioner/Dockerfile \
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/postgres-provisioner/Dockerfile \
|
||||
-t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -59,7 +74,7 @@ OBJECTSTORE_IMAGE ?= mesh-provision-objectstore:$(VERSION)
|
||||
OBJECTSTORE_DEV_TAG ?= mesh-provision-objectstore:development
|
||||
|
||||
objectstore-image:
|
||||
docker build -f examples/objectstore-provisioner/Dockerfile \
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/objectstore-provisioner/Dockerfile \
|
||||
-t $(OBJECTSTORE_IMAGE) -t $(OBJECTSTORE_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(OBJECTSTORE_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -70,7 +85,7 @@ REDIS_PROVISIONER_IMAGE ?= mesh-provision-redis:$(VERSION)
|
||||
REDIS_PROVISIONER_DEV_TAG ?= mesh-provision-redis:development
|
||||
|
||||
redis-provisioner-image:
|
||||
docker build -f examples/redis-provisioner/Dockerfile \
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/redis-provisioner/Dockerfile \
|
||||
-t $(REDIS_PROVISIONER_IMAGE) -t $(REDIS_PROVISIONER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -80,25 +95,25 @@ PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||
|
||||
proxy-image:
|
||||
docker build -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
#
|
||||
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
|
||||
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
|
||||
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
|
||||
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
|
||||
# was reading, as "no response from stream". That reads as a bug in the code under test.
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: fmt vet postgres
|
||||
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
|
||||
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
test:
|
||||
go test -p 1 ./...
|
||||
go test -timeout 15m ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
@@ -193,6 +193,13 @@ passes every check that only looks at the message.
|
||||
|
||||
## The image
|
||||
|
||||
**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go
|
||||
bundle, `controller`, which the host on the controller's machine unpacks and runs as the process
|
||||
`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what
|
||||
still runs a container of the controller: genesis, which raises the first controller from it and
|
||||
installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the
|
||||
mesh's own build any more — `make image` builds it.
|
||||
|
||||
`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
|
||||
manager, no libc, no CA certificates.
|
||||
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
|
||||
//
|
||||
// **The queue is the controller's; the build running here is this machine's.** The controller can
|
||||
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
|
||||
// on this machine and containers in this machine's runtime, and only this machine can end them. So
|
||||
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
|
||||
// it, pause, resume.
|
||||
//
|
||||
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
|
||||
// set per build — so "the build running here" is one or none, and kill names it by id so a call
|
||||
// that arrives as one build ends and the next begins cannot end the wrong one.
|
||||
|
||||
// holder is this machine's state as a holder of the build seat.
|
||||
type holder struct {
|
||||
on, seat string
|
||||
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
|
||||
// rather than silently taking work again.
|
||||
workspace string
|
||||
// say publishes this machine's state: whether it takes work (link.HolderState).
|
||||
say func(link.HolderState) error
|
||||
// remove runs what a kill needs outside the build: the containers left behind.
|
||||
remove builder.Runner
|
||||
|
||||
mu sync.Mutex
|
||||
paused bool
|
||||
running *running
|
||||
}
|
||||
|
||||
// running is the build this machine is doing.
|
||||
type running struct {
|
||||
request link.BuildRequest
|
||||
step string
|
||||
started time.Time
|
||||
cancel context.CancelFunc
|
||||
killed bool
|
||||
// returned is the build's own work having ended, before a kill or not; outcome is what was then
|
||||
// announced, and announced whether it went out.
|
||||
returned bool
|
||||
outcome string
|
||||
announced bool
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// pausedFile is where the flag lives in the workspace.
|
||||
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
|
||||
|
||||
// newHolder reads the paused flag the workspace keeps.
|
||||
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
|
||||
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
|
||||
if _, err := os.Stat(pausedFile(workspace)); err == nil {
|
||||
h.paused = true
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Paused is asked by the taking loop before every fetch.
|
||||
func (h *holder) Paused() bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.paused
|
||||
}
|
||||
|
||||
// setPaused records the flag in the workspace first and then in memory, so what this holder says
|
||||
// it is and what it would be after a restart never differ.
|
||||
func (h *holder) setPaused(paused bool) error {
|
||||
path := pausedFile(h.workspace)
|
||||
if paused {
|
||||
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
|
||||
}
|
||||
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.paused = paused
|
||||
h.mu.Unlock()
|
||||
h.announce()
|
||||
return nil
|
||||
}
|
||||
|
||||
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
|
||||
// controller reading an older state is a plan read as late rather than a build lost.
|
||||
func (h *holder) announce() {
|
||||
if h.say == nil {
|
||||
return
|
||||
}
|
||||
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
|
||||
// a pause outlives the events stream's retention.
|
||||
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||
h.announce()
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if h.Paused() {
|
||||
h.announce()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// begin records the build this machine took, with the cancel that ends it.
|
||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||
h.mu.Lock()
|
||||
h.running = r
|
||||
h.mu.Unlock()
|
||||
return r
|
||||
}
|
||||
|
||||
// end clears it, and lets a kill waiting on it know it is over.
|
||||
func (h *holder) end(r *running) {
|
||||
h.mu.Lock()
|
||||
if h.running == r {
|
||||
h.running = nil
|
||||
}
|
||||
h.mu.Unlock()
|
||||
close(r.done)
|
||||
}
|
||||
|
||||
// stepped records the step a build is at, for `current`.
|
||||
func (h *holder) stepped(r *running, step string) {
|
||||
h.mu.Lock()
|
||||
r.step = step
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// currentBuild is what `current` answers.
|
||||
type currentBuild struct {
|
||||
On string `json:"on"`
|
||||
Paused bool `json:"paused"`
|
||||
Running *struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
} `json:"running,omitempty"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
func (h *holder) current() currentBuild {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out := currentBuild{On: h.on, Paused: h.paused}
|
||||
taking := "taking builds"
|
||||
if h.paused {
|
||||
taking = "paused, taking no new build"
|
||||
}
|
||||
if h.running == nil {
|
||||
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
|
||||
return out
|
||||
}
|
||||
r := h.running
|
||||
elapsed := time.Since(r.started).Round(time.Second)
|
||||
out.Running = &struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
|
||||
r.started.UTC().Format(time.RFC3339), elapsed.String()}
|
||||
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
|
||||
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
|
||||
return out
|
||||
}
|
||||
|
||||
// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
|
||||
// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
|
||||
// (killAnswer in the controller's queue.go, 75s).
|
||||
var (
|
||||
killRemoves = 15 * time.Second
|
||||
killWaits = 20 * time.Second
|
||||
)
|
||||
|
||||
// kill ends the build with this id, if it is the one running here and still working: its context
|
||||
// cancelled — which kills each command's process group — and the containers it started removed by
|
||||
// their label, once at once and again after the build has ended, so one created while it was being
|
||||
// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
|
||||
// ask so it is not redelivered; the answer says whether that happened.
|
||||
func (h *holder) kill(id string) (string, error) {
|
||||
h.mu.Lock()
|
||||
r := h.running
|
||||
if r == nil || r.request.ID != id {
|
||||
doing := "nothing"
|
||||
if r != nil {
|
||||
doing = r.request.ID
|
||||
}
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
|
||||
}
|
||||
if r.returned {
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
|
||||
}
|
||||
r.killed = true
|
||||
cancel, done := r.cancel, r.done
|
||||
h.mu.Unlock()
|
||||
|
||||
cancel()
|
||||
removed, removeErr := h.removeContainers(id)
|
||||
ended := false
|
||||
select {
|
||||
case <-done:
|
||||
ended = true
|
||||
case <-time.After(killWaits):
|
||||
}
|
||||
again, againErr := h.removeContainers(id)
|
||||
removed += again
|
||||
if removeErr == nil {
|
||||
removeErr = againErr
|
||||
}
|
||||
containers := fmt.Sprintf("%d container(s) it started removed", removed)
|
||||
if removeErr != nil {
|
||||
containers = "its containers could not all be listed or removed: " + removeErr.Error()
|
||||
}
|
||||
if !ended {
|
||||
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
|
||||
"its outcome is in", id, h.on, containers), nil
|
||||
}
|
||||
h.mu.Lock()
|
||||
outcome, announced := r.outcome, r.announced
|
||||
h.mu.Unlock()
|
||||
if !announced {
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
|
||||
"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
|
||||
containers), nil
|
||||
}
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
|
||||
"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
|
||||
}
|
||||
|
||||
// removeContainers is one pass of removing what the build left, bounded.
|
||||
func (h *holder) removeContainers(id string) (int, error) {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
||||
defer stop()
|
||||
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||
}
|
||||
|
||||
// returned records that the build's work ended, and says whether a kill came first — only then is
|
||||
// the build killed; an error it ended with on its own is its own outcome.
|
||||
func (h *holder) returned(r *running) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
r.returned = true
|
||||
return r.killed
|
||||
}
|
||||
|
||||
// said records the outcome announced, and whether it went out, for a kill to answer with.
|
||||
func (h *holder) said(r *running, outcome string, announced bool) {
|
||||
h.mu.Lock()
|
||||
r.outcome, r.announced = outcome, announced
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// handlers are the seat's verbs, as this machine answers them.
|
||||
func (h *holder) handlers() map[string]link.ToolHandler {
|
||||
return map[string]link.ToolHandler{
|
||||
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
|
||||
return nil, errors.New("kill needs the build's id")
|
||||
}
|
||||
said, err := h.kill(strings.TrimSpace(args.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": said}, nil
|
||||
},
|
||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
said := h.on + " is paused: it takes no new build until resumed"
|
||||
if c := h.current(); c.Running != nil {
|
||||
said += "; " + c.Running.ID + " runs on and finishes"
|
||||
}
|
||||
return map[string]any{"said": said, "paused": true}, nil
|
||||
},
|
||||
"resume": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func orNothing(s string) string {
|
||||
if s == "" {
|
||||
return "its start"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
|
||||
// log it would be is the one being ended.
|
||||
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
|
||||
// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
|
||||
// the module answering, the seat, scope node, the machine, its description and argument schema — so
|
||||
// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
|
||||
//
|
||||
// One service per machine, named for the seat and identified by the machine, so the answer is this
|
||||
// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
|
||||
// store, and what it serves is what this binary was built to serve.
|
||||
func announcement(seat, module, node string) micro.Info {
|
||||
s, _ := catalogue.SeatNamed(seat)
|
||||
var endpoints []micro.EndpointInfo
|
||||
for _, v := range s.Serves {
|
||||
schema, _ := json.Marshal(v.Input)
|
||||
endpoints = append(endpoints, micro.EndpointInfo{
|
||||
Name: seat + "__" + v.Name,
|
||||
Subject: link.NodeSeatToolSubject(seat, v.Name, node),
|
||||
// The queue group the verbs are served in, as every runtime announces its own.
|
||||
QueueGroup: "seat." + seat,
|
||||
Metadata: map[string]string{
|
||||
"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
|
||||
"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
|
||||
},
|
||||
})
|
||||
}
|
||||
return micro.Info{
|
||||
ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
|
||||
Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
|
||||
Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
}
|
||||
|
||||
// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
|
||||
func moduleOf(user string) string {
|
||||
if _, module, ok := strings.Cut(user, "."); ok && module != "" {
|
||||
return module
|
||||
}
|
||||
return "build-agent"
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
||||
// 0219), and what it says about itself follows.
|
||||
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
var said []link.HolderState
|
||||
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
||||
said = append(said, s)
|
||||
return nil
|
||||
})
|
||||
if h.Paused() {
|
||||
t.Fatal("a new holder starts paused")
|
||||
}
|
||||
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
||||
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
||||
}
|
||||
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
||||
if !again.Paused() {
|
||||
t.Fatal("restarted, the holder forgot it was paused")
|
||||
}
|
||||
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
||||
t.Fatal("resumed, the holder came back paused")
|
||||
}
|
||||
}
|
||||
|
||||
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
||||
// label, and refuses an id it is not building.
|
||||
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
var removed []string
|
||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "ps" {
|
||||
return "c1\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
kill := h.handlers()["kill"]
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
||||
t.Fatal("killed a build while none ran")
|
||||
}
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
||||
h.stepped(r, "image")
|
||||
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
||||
t.Fatalf("current says %+v", c)
|
||||
}
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "build-1") {
|
||||
t.Fatalf("killed another id: %v", err)
|
||||
}
|
||||
// The build's own goroutine: it ends when its context does, as a build's commands do, and
|
||||
// announces what came of it.
|
||||
go func() {
|
||||
<-building.Done()
|
||||
if h.returned(r) {
|
||||
h.said(r, link.KilledByHand, true)
|
||||
}
|
||||
h.end(r)
|
||||
}()
|
||||
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if building.Err() == nil {
|
||||
t.Fatal("the build's context was not cancelled")
|
||||
}
|
||||
if !r.killed {
|
||||
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
||||
}
|
||||
said := answer.(map[string]any)["said"].(string)
|
||||
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
|
||||
t.Errorf("kill said %q", said)
|
||||
}
|
||||
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
||||
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
||||
t.Errorf("removed %v", removed)
|
||||
}
|
||||
if c := h.current(); c.Running != nil {
|
||||
t.Errorf("after the kill current says %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
|
||||
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
|
||||
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
|
||||
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
|
||||
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
|
||||
}
|
||||
kill := info.Endpoints[1]
|
||||
md := kill.Metadata
|
||||
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
|
||||
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
|
||||
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
|
||||
t.Fatalf("kill is announced as %+v", kill)
|
||||
}
|
||||
body, err := json.Marshal(info)
|
||||
if err != nil || len(body) > 8*1024 {
|
||||
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
|
||||
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
|
||||
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
|
||||
_, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
|
||||
if killed := h.returned(r); killed {
|
||||
t.Fatal("a build nobody killed reads as killed")
|
||||
}
|
||||
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
|
||||
t.Fatalf("killed a build that had ended: %v", err)
|
||||
}
|
||||
h.end(r)
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
|
||||
go func() {
|
||||
<-building.Done()
|
||||
if h.returned(r) {
|
||||
h.said(r, link.KilledByHand, false)
|
||||
}
|
||||
h.end(r)
|
||||
}()
|
||||
said, err := h.kill("build-2")
|
||||
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
|
||||
t.Fatalf("kill said %q (%v)", said, err)
|
||||
}
|
||||
}
|
||||
+188
-109
@@ -17,20 +17,15 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
@@ -52,7 +47,6 @@ const usage = `mesh-builder — builds modules for the mesh
|
||||
It consumes build requests and answers with what it made. Nothing is listened on and nothing
|
||||
is dialled except the broker.
|
||||
|
||||
MESH_BROKER_AMQP where the broker is, with this builder's own credential
|
||||
MESH_BROKER_FILE a file the mesh sealed to this machine holding the same
|
||||
MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said
|
||||
MESH_BINDING a file the mesh wrote saying where the artifact store is
|
||||
@@ -115,105 +109,176 @@ func run() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
machine, err := takeWorkFrom(credential, on)
|
||||
js, seat, err := dialFor(credential)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
|
||||
// paused flag is read from the workspace before anything is taken, so a holder restarted while
|
||||
// paused takes nothing.
|
||||
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
|
||||
body, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
|
||||
return err
|
||||
})
|
||||
if h.Paused() {
|
||||
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||
}
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
// one serves none, and a subscription its holder has no grant for would be refused for ever.
|
||||
if seat == link.TheBuildMachine {
|
||||
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
|
||||
log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopServing()
|
||||
// And says so, for the console to find (novox/hq ADR 0197).
|
||||
stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
|
||||
announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopAnnouncing()
|
||||
go h.stateWhilePaused(ctx, time.Hour)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
answer(ctx, publisher, on, workspace, work)
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
|
||||
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
|
||||
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
|
||||
// holds.
|
||||
//
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||
// machine told about both would take work from one and answer on the other, and every log line would
|
||||
// say it was fine.
|
||||
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
|
||||
// **The credential decides, before any variable does.** A machine moved to the new bus was
|
||||
// handed a credential for it and nothing else changed in its environment; that credential
|
||||
// names the bus by scheme, so it is enough to know which bus to take work from.
|
||||
if credential.onTheNewBus() {
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.MachineOverNATS(js, on), nil
|
||||
func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
||||
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
||||
// and that is enough to dial it, pinned.
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
address, onNATS, err := broker.OnNATS()
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
if err := broker.MustBeOneBus(credential.URL, address); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if onNATS {
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus at %s: %w", address, err)
|
||||
}
|
||||
return link.MachineOverNATS(js, on), nil
|
||||
}
|
||||
|
||||
conn, err := dial(credential)
|
||||
if err != nil {
|
||||
// Not quoted back: the URL carries this builder's broker password.
|
||||
return nil, fmt.Errorf("cannot reach the broker: %w", err)
|
||||
}
|
||||
channel, err := conn.Channel()
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
return link.MachineOverCurrent(conn, channel, on), nil
|
||||
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
||||
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
||||
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
||||
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
||||
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
||||
return js, seat, nil
|
||||
}
|
||||
|
||||
// answer does one build and says what happened, whichever way it went.
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
|
||||
request := work.Request()
|
||||
|
||||
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
|
||||
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
|
||||
// reaches it through the parent, and that keeps today's meaning below.
|
||||
building, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
var mine *running
|
||||
if h != nil {
|
||||
mine = h.begin(request, cancel)
|
||||
defer h.end(mine)
|
||||
}
|
||||
|
||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||
// the handler said nothing until the end.
|
||||
fmt.Fprintf(os.Stderr, "a build request arrived for %s\n", request.Repository)
|
||||
fmt.Fprintf(os.Stderr, "a build request arrived for %s (%s)\n", request.Repository, request.ID)
|
||||
|
||||
// **Everything a build says goes two ways**: to stderr, as always, and onto the bus as the
|
||||
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
||||
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
||||
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
||||
say := func(step, message string) {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
h.stepped(mine, step)
|
||||
}
|
||||
}
|
||||
builder.Said = say
|
||||
defer func() { builder.Said = nil }()
|
||||
if err := work.Began(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
||||
}
|
||||
|
||||
result := link.BuildResult{
|
||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||
Ref: request.Ref, On: on,
|
||||
Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "building %s", request.Repository)
|
||||
what := "building " + request.Repository
|
||||
if request.Path != "" {
|
||||
fmt.Fprintf(os.Stderr, " at %s", request.Path)
|
||||
what += " at " + request.Path
|
||||
}
|
||||
if request.Ref != "" {
|
||||
fmt.Fprintf(os.Stderr, " at %s", request.Ref)
|
||||
what += " on " + request.Ref
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
say("build", what)
|
||||
|
||||
npmrc, err := packagesFrom()
|
||||
var built builder.Result
|
||||
if err == nil {
|
||||
if request.Check != nil {
|
||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
||||
// published or registered; the verdict is the outcome.
|
||||
result.Checked = request.Check
|
||||
registry := ""
|
||||
if r, ok := publisher.(builder.Registry); ok {
|
||||
registry = r.Address
|
||||
}
|
||||
var v builder.CheckVerdict
|
||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
||||
if err == nil {
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||
// after a clone that then fails at npm ci.
|
||||
built, err = builder.Build(ctx, builder.Command, publisher,
|
||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||
// docker client (ADR 0219).
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(),
|
||||
func(step, message string) {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
})
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
if err != nil {
|
||||
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
||||
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
||||
killed := false
|
||||
if mine != nil {
|
||||
killed = h.returned(mine) && err != nil
|
||||
}
|
||||
if killed {
|
||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||
// error it ended with is the kill's consequence, not a fault of the source.
|
||||
result.Failed = link.KilledByHand
|
||||
say("failed", link.KilledByHand)
|
||||
} else if err != nil {
|
||||
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
||||
// builder that is not running, and those want completely different responses.
|
||||
result.Failed = err.Error()
|
||||
fmt.Fprintf(os.Stderr, " failed: %v\n", err)
|
||||
say("failed", err.Error())
|
||||
} else if request.Check != nil {
|
||||
say("checked", result.Check.Verdict+": "+result.Check.Summary)
|
||||
} else {
|
||||
manifest, marshalErr := json.Marshal(built.Manifest)
|
||||
if marshalErr != nil {
|
||||
@@ -227,11 +292,30 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
})
|
||||
}
|
||||
result.Against = built.Against
|
||||
fmt.Fprintf(os.Stderr, " built %s from %s\n", built.Manifest.Module, short(built.Commit))
|
||||
result.SourceFingerprint = built.Source
|
||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
||||
for _, r := range built.Read {
|
||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||
}
|
||||
}
|
||||
|
||||
if err := work.Announce(ctx, result); err != nil {
|
||||
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
|
||||
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
|
||||
// to be built again. A machine being stopped is the other case and keeps its meaning: the
|
||||
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
|
||||
announcing := ctx
|
||||
if killed {
|
||||
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
announcing = fresh
|
||||
}
|
||||
announceErr := work.Announce(announcing, result)
|
||||
if mine != nil {
|
||||
h.said(mine, result.Failed, announceErr == nil)
|
||||
}
|
||||
if err := announceErr; err != nil {
|
||||
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||
// nothing was said at all, so another machine can try.
|
||||
@@ -246,6 +330,31 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
}
|
||||
|
||||
// checkSpecOf is a check request as the builder runs it.
|
||||
func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
c := request.Check
|
||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
}
|
||||
for _, m := range c.Members {
|
||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||
Ref: m.Ref, Paths: m.Paths})
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
// layerOf is one layer of a check as the outcome carries it.
|
||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules}
|
||||
}
|
||||
|
||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||
// (novox/hq ADR 0076, issue 053).
|
||||
//
|
||||
@@ -452,13 +561,8 @@ func brokerFrom() (Credential, error) {
|
||||
// broker is then verified against whatever this machine already trusts.
|
||||
return Credential{URL: said}, nil
|
||||
}
|
||||
url := strings.TrimSpace(os.Getenv("MESH_BROKER_AMQP"))
|
||||
if url == "" {
|
||||
return Credential{}, fmt.Errorf(
|
||||
"neither MESH_BROKER_FILE nor MESH_BROKER_AMQP: a builder with no broker has " +
|
||||
"nothing to build")
|
||||
}
|
||||
return Credential{URL: url}, nil
|
||||
return Credential{}, fmt.Errorf(
|
||||
"no MESH_BROKER_FILE: a build machine with no credential for the bus has nothing to build")
|
||||
}
|
||||
|
||||
// Credential is what a build machine is given so it can reach the broker.
|
||||
@@ -477,6 +581,21 @@ type Credential struct {
|
||||
// as two fields and this machine joins them once, here, to dial.
|
||||
User string `json:"user,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
// Claims are the seats the module this credential was issued for claims, as the mesh writes
|
||||
// them beside the credential (novox/hq ADR 0159). The first is the build role this machine
|
||||
// serves; a credential naming none is from before claims travelled in it.
|
||||
Claims []struct {
|
||||
Seat string `json:"seat"`
|
||||
} `json:"claims,omitempty"`
|
||||
}
|
||||
|
||||
// seatsClaimed is the seats the credential names, in order.
|
||||
func (c Credential) seatsClaimed() []string {
|
||||
out := make([]string, 0, len(c.Claims))
|
||||
for _, claim := range c.Claims {
|
||||
out = append(out, claim.Seat)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
|
||||
@@ -491,43 +610,3 @@ func (c Credential) natsURL() string {
|
||||
}
|
||||
return "nats://" + c.User + ":" + c.Password + "@" + rest
|
||||
}
|
||||
|
||||
// dial opens the connection, pinning the broker's certificate when there is one to pin.
|
||||
func dial(held Credential) (*amqp.Connection, error) {
|
||||
if held.Fingerprint == "" {
|
||||
return amqp.Dial(held.URL)
|
||||
}
|
||||
return amqp.DialTLS(held.URL, pinning(held.Fingerprint))
|
||||
}
|
||||
|
||||
// pinning is a TLS configuration that trusts exactly one certificate.
|
||||
//
|
||||
// InsecureSkipVerify with a VerifyPeerCertificate is **pinning, not skipping**: the standard chain
|
||||
// check is replaced, not removed, and what replaces it is stricter — one certificate is accepted
|
||||
// rather than every certificate a public authority would sign.
|
||||
//
|
||||
// Its own function so a test can drive it against a real handshake. A pin check that is only ever
|
||||
// exercised through a broker is a pin check nothing tests.
|
||||
func pinning(fingerprint string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(raw) == 0 {
|
||||
return errors.New("the broker presented no certificate")
|
||||
}
|
||||
// The leaf, and in the same spelling the mesh writes it — `sha256:` and 64 hex
|
||||
// characters. Comparing a bare digest against a written fingerprint never matches,
|
||||
// and the failure is indistinguishable from being pointed at the wrong broker.
|
||||
sum := sha256.Sum256(raw[0])
|
||||
got := "sha256:" + hex.EncodeToString(sum[:])
|
||||
if got != fingerprint {
|
||||
return fmt.Errorf(
|
||||
"this is not the broker this builder was told about\n expected %s\n "+
|
||||
"got %s\nEither this mesh's broker was replaced, or this builder is "+
|
||||
"being pointed at something else. Retrying will not help",
|
||||
fingerprint, got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,6 +105,10 @@ func buildOnce(ctx context.Context, args []string) error {
|
||||
Ref: *ref,
|
||||
Manifest: built.Manifest,
|
||||
Against: built.Against,
|
||||
Source: built.Source,
|
||||
}
|
||||
for _, r := range built.Read {
|
||||
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
for _, made := range built.Built {
|
||||
out.Made = append(out.Made, madeArtifact{Name: made.Name, Kind: made.Kind, Reference: made.Reference})
|
||||
@@ -123,14 +127,22 @@ func buildOnce(ctx context.Context, args []string) error {
|
||||
// The same fields the mesh records for a build, so a reader comparing a genesis build against an
|
||||
// ordinary one is comparing the same thing said the same way.
|
||||
type onceResult struct {
|
||||
Module string `json:"module"`
|
||||
Commit string `json:"commit"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Manifest any `json:"manifest"`
|
||||
Made []madeArtifact `json:"made"`
|
||||
Against []string `json:"against,omitempty"`
|
||||
Module string `json:"module"`
|
||||
Commit string `json:"commit"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Manifest any `json:"manifest"`
|
||||
Made []madeArtifact `json:"made"`
|
||||
Against []string `json:"against,omitempty"`
|
||||
Read []readRepository `json:"read,omitempty"`
|
||||
Source string `json:"source-fingerprint,omitempty"`
|
||||
}
|
||||
|
||||
// readRepository is a repository this build read source from besides the module's own.
|
||||
type readRepository struct {
|
||||
Repository string `json:"repository"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
}
|
||||
|
||||
type madeArtifact struct {
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover).
|
||||
func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) {
|
||||
var held Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x",
|
||||
"claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" {
|
||||
t.Errorf("the old builder's credential serves %q", got)
|
||||
}
|
||||
var bare Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine {
|
||||
t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Where a builder publishes.
|
||||
@@ -193,9 +195,9 @@ func TestThePinIsComparedInTheSpellingTheMeshWritesIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// handshakeWith runs the builder's own pin check against an address.
|
||||
// handshakeWith runs the pin check the builder dials with against an address.
|
||||
func handshakeWith(address, pin string) error {
|
||||
conn, err := tls.Dial("tcp", address, pinning(pin))
|
||||
conn, err := tls.Dial("tcp", address, broker.PinnedToFingerprint(pin))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
|
||||
//
|
||||
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
|
||||
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
|
||||
// not happen. Three ways a process stands to the lease:
|
||||
//
|
||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
||||
// its service manager restarts it as a candidate (serve, in push.go).
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
|
||||
// between makes it stale and refused like any other. **When nobody holds the lease, the command
|
||||
// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
|
||||
// it, as it would for another controller.
|
||||
// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
|
||||
// refused nothing: there is nothing to order against, and nothing it does reaches a machine.
|
||||
//
|
||||
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
|
||||
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
|
||||
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
|
||||
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
|
||||
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
|
||||
// instead would be a controller that can never send the user list that lets it act.
|
||||
|
||||
// actor is this process's standing to the lease.
|
||||
type actor struct {
|
||||
mu sync.Mutex
|
||||
// held is the lease this process holds: the serving controller's, or a command's own.
|
||||
held *lease.Lease
|
||||
// unleased is why a serving controller acts without the lease; empty while it holds it or is not
|
||||
// serving.
|
||||
unleased string
|
||||
// serving is a serving controller, which never borrows another's epoch.
|
||||
serving bool
|
||||
// kv is the lease bucket, for a command to read the holder's epoch from.
|
||||
kv jetstream.KeyValue
|
||||
close func()
|
||||
// noBus is a process with no bus configured.
|
||||
noBus bool
|
||||
// reset is when the lease bucket was found raised again from nothing and its revisions moved past
|
||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
var theLease = &actor{}
|
||||
|
||||
// instance names this process among controller instances: its machine, its process and when it
|
||||
// started. The lease's holder and every call this process keeps carry it.
|
||||
var instance = func() string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||
}()
|
||||
|
||||
// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
|
||||
func (a *actor) epoch(ctx context.Context) (uint64, error) {
|
||||
a.mu.Lock()
|
||||
held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
|
||||
a.mu.Unlock()
|
||||
switch {
|
||||
case held != nil:
|
||||
return held.Epoch()
|
||||
case serving && unleased != "":
|
||||
return 0, nil
|
||||
case serving:
|
||||
return 0, lease.ErrNotHeld
|
||||
case noBus:
|
||||
return 0, nil
|
||||
}
|
||||
return a.forACommand(ctx)
|
||||
}
|
||||
|
||||
// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
|
||||
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if a.held != nil {
|
||||
return a.held.Epoch()
|
||||
}
|
||||
if a.kv == nil {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
// No bus: this process reaches no machine, and has nothing to order against.
|
||||
a.noBus = true
|
||||
return 0, nil
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
|
||||
"read and nothing is done: %w", err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := broker.EnsureLeaseBucket(reading, api); err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
kv, err := api.KeyValue(reading, broker.LeaseBucket)
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
a.kv, a.close = kv, js.Close
|
||||
if _, found, err := lease.Current(reading, kv); err == nil && !found {
|
||||
// Nobody: this command takes it for as long as it runs.
|
||||
l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
epoch, err := l.TryTake(reading)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
|
||||
}
|
||||
keeping, stop := context.WithCancel(context.Background())
|
||||
go l.Keep(keeping)
|
||||
a.held = l
|
||||
closeBus := a.close
|
||||
a.close = func() {
|
||||
stop()
|
||||
l.Release(context.Background())
|
||||
closeBus()
|
||||
}
|
||||
return epoch, nil
|
||||
}
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
holder, found, err := lease.Current(reading, a.kv)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
|
||||
"more is done under an epoch nobody holds — run the command again")
|
||||
}
|
||||
return holder.Epoch, nil
|
||||
}
|
||||
|
||||
// release gives back what this process holds, at its end.
|
||||
func (a *actor) release() {
|
||||
a.mu.Lock()
|
||||
closing := a.close
|
||||
a.close = nil
|
||||
a.mu.Unlock()
|
||||
if closing != nil {
|
||||
closing()
|
||||
}
|
||||
}
|
||||
|
||||
// holderOf is this process as the lease's holder.
|
||||
func holderOf(instance string) lease.Holder {
|
||||
host, _ := os.Hostname()
|
||||
return lease.Holder{Instance: instance, Host: host, Build: version}
|
||||
}
|
||||
|
||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
||||
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
|
||||
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
|
||||
a.mu.Lock()
|
||||
a.serving = true
|
||||
a.mu.Unlock()
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.EnsureLeaseBucket(asserting, api)
|
||||
cancel()
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
|
||||
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
|
||||
"no machine refuses the next epoch", was, floor, floor)
|
||||
}})
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
gone := make(chan struct{})
|
||||
epoch, err := l.Take(ctx)
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
js.Close()
|
||||
return nil, ctx.Err()
|
||||
case err != nil && !errors.Is(err, lease.ErrUnwritable):
|
||||
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
|
||||
js.Close()
|
||||
return nil, err
|
||||
case err != nil:
|
||||
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
|
||||
a.mu.Lock()
|
||||
a.unleased = err.Error()
|
||||
a.mu.Unlock()
|
||||
say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
|
||||
"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
|
||||
go a.takeWhenGranted(ctx, l, inv, gone)
|
||||
default:
|
||||
a.took(ctx, l, inv, epoch, gone)
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.close = func() {
|
||||
if held, err := l.Epoch(); err == nil {
|
||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
|
||||
say("how epoch %d ended could not be recorded: %v", held, err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
l.Release(context.Background())
|
||||
js.Close()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
return gone, nil
|
||||
}
|
||||
|
||||
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
|
||||
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
|
||||
a.mu.Lock()
|
||||
a.held, a.unleased = l, ""
|
||||
a.mu.Unlock()
|
||||
h := holderOf(instance)
|
||||
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
|
||||
Build: h.Build, Taken: time.Now()})
|
||||
cancel()
|
||||
if err != nil {
|
||||
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
|
||||
}
|
||||
for _, e := range expired {
|
||||
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
|
||||
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
|
||||
}
|
||||
go l.Keep(ctx)
|
||||
go func() {
|
||||
<-l.Lost()
|
||||
if ctx.Err() == nil {
|
||||
why := l.LostWhy()
|
||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
|
||||
cancel()
|
||||
fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
|
||||
"be started again as a candidate\n", epoch, why)
|
||||
}
|
||||
close(gone)
|
||||
}()
|
||||
}
|
||||
|
||||
// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
|
||||
// controller if another took it meanwhile: two serving at once is what the lease is for.
|
||||
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
|
||||
tick := time.NewTicker(lease.RenewEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
epoch, err := l.TryTake(ctx)
|
||||
if errors.Is(err, lease.ErrTaken) {
|
||||
fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
|
||||
"stops and exits\n", err)
|
||||
close(gone)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
// Still not written, or not readable this time: unleased, said by S12, tried again.
|
||||
a.mu.Lock()
|
||||
a.unleased = err.Error()
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
a.took(ctx, l, inv, epoch, gone)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// standing is what `status` and the self-check say of this process and the lease.
|
||||
type standing struct {
|
||||
Epoch uint64
|
||||
Held bool
|
||||
Renewed time.Time
|
||||
Unleased string
|
||||
// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
|
||||
Reset time.Time
|
||||
ResetSaid string
|
||||
}
|
||||
|
||||
func (a *actor) standing() standing {
|
||||
a.mu.Lock()
|
||||
held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
|
||||
a.mu.Unlock()
|
||||
st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
|
||||
if held == nil {
|
||||
return st
|
||||
}
|
||||
epoch, err := held.Epoch()
|
||||
st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
|
||||
return st
|
||||
}
|
||||
|
||||
// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
|
||||
func init() {
|
||||
link.ActingGate = func(ctx context.Context) error {
|
||||
_, err := theLease.epoch(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("this controller may not send: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
+194
-17
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -38,7 +40,10 @@ import (
|
||||
//
|
||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||
// machines this just blocked is worth more than the milliseconds.
|
||||
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("assign %s names no module", node)
|
||||
}
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
@@ -46,55 +51,197 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
||||
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
||||
// resources are applied through a seat nothing on the node holds is refused, because that order
|
||||
// — the service manager, the package manager and the runtime before anything that installs,
|
||||
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
|
||||
// holders that depend on each other go on in one command.
|
||||
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !fresh {
|
||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||
return fmt.Sprintf("%s already runs %s — one node runs one of each (ADR 0115); nothing changed",
|
||||
node, module), nil
|
||||
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
||||
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
||||
// assignment resolves against a record rather than against a coincidence.
|
||||
settled, err := recordDerivedHolders(ctx, open)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var lines []string
|
||||
var added []string
|
||||
for _, module := range modules {
|
||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||
if err != nil {
|
||||
return strings.Join(lines, "\n"), err
|
||||
}
|
||||
if !fresh {
|
||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||
lines = append(lines, fmt.Sprintf(
|
||||
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
|
||||
continue
|
||||
}
|
||||
added = append(added, module)
|
||||
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
|
||||
}
|
||||
if len(added) == 0 {
|
||||
return strings.Join(lines, "\n"), nil
|
||||
}
|
||||
answer := strings.Join(lines, "\n")
|
||||
for _, line := range settled {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||
// node's list, and every other node's, is `status`'s.
|
||||
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||
for _, module := range added {
|
||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||
answer += "\n " + line
|
||||
}
|
||||
}
|
||||
said := fmt.Sprintf("%s is assigned %s", node, module)
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||
return said + blockedElsewhere(ctx, open, node), err
|
||||
return answer + blockedElsewhere(ctx, open, node), err
|
||||
}
|
||||
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
||||
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
||||
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
||||
isAdded := map[string]bool{}
|
||||
for _, m := range added {
|
||||
isAdded[m] = true
|
||||
}
|
||||
for _, u := range plan.Unhostable {
|
||||
if u.Module != module {
|
||||
if !isAdded[u.Module] {
|
||||
continue
|
||||
}
|
||||
for _, c := range u.Missing {
|
||||
said += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||
}
|
||||
}
|
||||
return said + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||
blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// unassign takes a module off a node. What it leaves behind is the host's business: a directory
|
||||
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
||||
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
|
||||
// and are not judged again.
|
||||
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
|
||||
map[string]catalogue.Manifest, []string, error) {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
already := map[string]bool{}
|
||||
for _, a := range assigned {
|
||||
already[a] = true
|
||||
}
|
||||
var adding []string
|
||||
for _, m := range modules {
|
||||
if !already[m] {
|
||||
adding = append(adding, m)
|
||||
}
|
||||
}
|
||||
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
|
||||
// with everything else this act changed, so they are not said twice.
|
||||
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Two modules declaring one package, path or unit is refused before anything is recorded
|
||||
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
||||
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return shelf, assigned, nil
|
||||
}
|
||||
|
||||
// unassign takes modules off a node. What they leave behind is the host's business: a directory
|
||||
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
||||
//
|
||||
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
||||
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
||||
// about the command's own output would ever have said so.
|
||||
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
//
|
||||
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
|
||||
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, module, node) + blockedElsewhere(ctx, open, node), nil
|
||||
assigned, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Every one checked before any is taken off, so a refusal leaves the node as it was.
|
||||
runs := map[string]bool{}
|
||||
for _, a := range assigned {
|
||||
runs[a] = true
|
||||
}
|
||||
for _, module := range modules {
|
||||
if !runs[module] {
|
||||
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
||||
}
|
||||
}
|
||||
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, module := range modules {
|
||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, strings.Join(modules, ", "), node)
|
||||
var left []string
|
||||
for _, a := range assigned {
|
||||
if !slices.Contains(modules, a) {
|
||||
left = append(left, a)
|
||||
}
|
||||
}
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||
// command API and the controller seat's verbs as they do from the command line.
|
||||
func splitModules(field string) []string {
|
||||
var out []string
|
||||
for _, m := range strings.Split(field, ",") {
|
||||
if m = strings.TrimSpace(m); m != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
||||
@@ -142,3 +289,33 @@ func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
||||
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
||||
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
|
||||
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
|
||||
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
|
||||
// module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
m, known := shelf[module]
|
||||
if !known || mayIssue(m) != nil {
|
||||
return ""
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||
return ""
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err == nil {
|
||||
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
|
||||
"`push %s` refuses to send %s until it is", err, module, node, node, module)
|
||||
}
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
@@ -17,8 +17,8 @@ import (
|
||||
|
||||
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
||||
|
||||
// **A build is recorded by digest and path**, whatever address the builder pushed to — and only
|
||||
// what the build made is rewritten: an image the module runs from elsewhere is left where it says.
|
||||
// **A build is recorded by digest and path**, whatever address the builder pushed to — what it
|
||||
// made and what it stood on both; an image the module runs from elsewhere is left where it says.
|
||||
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
manifest, _ := json.Marshal(map[string]any{
|
||||
"module": "gitea", "version": "1",
|
||||
@@ -65,8 +65,11 @@ func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
|
||||
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
|
||||
}
|
||||
if kept.Against[0] != "anchor.internal:5100/mesh-tools/runtime@"+aDigest {
|
||||
t.Errorf("what the build stood on was rewritten: %v", kept.Against)
|
||||
// What the build stood on is an edge to another module's artifact, and it is recorded the way
|
||||
// that artifact is: by path in the store, so the edge still names the same thing when the
|
||||
// store answers at another address.
|
||||
if kept.Against[0] != catalogue.ArtifactStoreScheme+"mesh-tools/runtime@"+aDigest {
|
||||
t.Errorf("what the build stood on was recorded by address: %v", kept.Against)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +111,14 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
@@ -142,7 +153,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// container is told so beside the sealed connection genesis wrote.
|
||||
// process is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -154,8 +165,8 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
|
||||
Reference: "registry.example/control@" + aDigest}})
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -172,6 +183,12 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -191,12 +208,12 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.server" {
|
||||
if r["id"] == "mesh-controller.controller" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's container is not in its own node's declaration")
|
||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
@@ -229,7 +246,7 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
out := m
|
||||
out.Resources = nil
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
if r["type"] != "container" && r["type"] != "process" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -85,17 +85,32 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
||||
Firewall: "ufw", Held: held, Reachable: reachable,
|
||||
// A machine says which of its links face outside on every apply (novox/hq ADR 0140), and a
|
||||
// filter is not sent to one that has not. The anchor reports one, as a real host does; this
|
||||
// fixture lacked it from 2026-09-28 and nothing ran the test (issue 177).
|
||||
Outward: []string{"eth0"},
|
||||
// And what filters it (ADR 0168): its front end, the runtime's own, and a chain a
|
||||
// predecessor left in the runtime's user chain.
|
||||
Filters: anchorFilters,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// anchorFilters is what the adopted anchor says filters it: ufw's chains, the runtime's, and a
|
||||
// predecessor's chain the mesh did not write.
|
||||
var anchorFilters = []link.Filter{
|
||||
{Where: "table ip filter, chain ufw-reject-input", Owner: "found-firewall", Refuses: "reject"},
|
||||
{Where: "table ip filter, chain DOCKER", Owner: "runtime", Refuses: `iifname != "docker0" oifname "docker0" drop`},
|
||||
{Where: "table ip filter, chain DOCKER-USER", Owner: "other", Refuses: `iifname "eth0" tcp dport 6000 drop`},
|
||||
}
|
||||
|
||||
var (
|
||||
heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container",
|
||||
Target: "hello-web", Since: time.Now()}
|
||||
@@ -105,10 +120,10 @@ var (
|
||||
|
||||
func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
if _, err := take(t.Context(), open, "anchor", "nftables"); !errors.Is(err, inventory.ErrNotAssigned) {
|
||||
if _, err := take(t.Context(), open, "anchor", "nftables", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAssigned) {
|
||||
t.Fatalf("taking an unassigned module gave %v", err)
|
||||
}
|
||||
if _, err := take(t.Context(), open, "laptop", "network"); !errors.Is(err, inventory.ErrNotAdopted) {
|
||||
if _, err := take(t.Context(), open, "laptop", "network", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAdopted) {
|
||||
t.Fatalf("taking on a converged node gave %v", err)
|
||||
}
|
||||
}
|
||||
@@ -139,7 +154,24 @@ func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) {
|
||||
func TestTakingNamesWhatItReplaces(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
said, err := take(t.Context(), open, "anchor", "hello-web")
|
||||
ctx := t.Context()
|
||||
// The machine holds something for the module, so the take acts on the preview the operator
|
||||
// saw and names its digest (novox/hq ADR 0163).
|
||||
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saw := takeDigestIn(t, preview)
|
||||
if !strings.Contains(preview, "nothing taken; `take anchor hello-web --yes "+saw+"`") {
|
||||
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
||||
}
|
||||
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
|
||||
t.Fatal("the preview took something")
|
||||
}
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil || !strings.Contains(err.Error(), "name its digest") {
|
||||
t.Fatalf("--yes without the digest was not refused: %v", err)
|
||||
}
|
||||
said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -149,10 +181,71 @@ func TestTakingNamesWhatItReplaces(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// takeDigestIn is the digest a take's preview printed.
|
||||
func takeDigestIn(t *testing.T, preview string) string {
|
||||
t.Helper()
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
||||
return fields[1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("the preview printed no digest:\n%s", preview)
|
||||
return ""
|
||||
}
|
||||
|
||||
// A take acts on the preview the operator saw, and on an account of the machine that is still the
|
||||
// machine: a changed preview and a stale account refuse (novox/hq ADR 0163, rule 1).
|
||||
func TestATakeIsRefusedOnAChangedPreviewOrAStaleAccount(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saw := takeDigestIn(t, preview)
|
||||
|
||||
// The machine reports again, and what it holds has changed: the found container now carries
|
||||
// facts the preview never showed.
|
||||
changed := heldContainer
|
||||
changed.Facts = map[string]any{"image": "hello:2", "declared_image": "registry.example/hello"}
|
||||
reportsHolding(t, open, changed, heldFile)
|
||||
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
||||
t.Fatalf("a changed preview was acted on: %v", err)
|
||||
}
|
||||
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
|
||||
t.Fatal("a refused take took something")
|
||||
}
|
||||
|
||||
// And an account older than the flip allows.
|
||||
preview, err = take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saw = takeDigestIn(t, preview)
|
||||
saved := reportFreshFor
|
||||
reportFreshFor = -time.Second
|
||||
defer func() { reportFreshFor = saved }()
|
||||
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||
if err == nil || !strings.Contains(err.Error(), "a take acts only on an account newer than") {
|
||||
t.Fatalf("a stale account was acted on: %v", err)
|
||||
}
|
||||
reportFreshFor = saved
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the machine holds nothing for has nothing to compare: --yes alone suffices.
|
||||
if _, err := take(ctx, open, "anchor", "notes", takeOptions{Yes: true}); err == nil {
|
||||
// notes holds a file, so this one needs the digest too.
|
||||
t.Fatal("notes holds a found file and was taken without a digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
@@ -182,6 +275,20 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T)
|
||||
if strings.Contains(preview, "15672") {
|
||||
t.Errorf("a loopback listener is in the preview:\n%s", preview)
|
||||
}
|
||||
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the predecessor's
|
||||
// chain is named as not the mesh's and left, so the reader knows before the flip.
|
||||
for _, want := range []string{
|
||||
"table ip filter, chain DOCKER-USER",
|
||||
"NOT THE MESH'S; left in force",
|
||||
`iifname "eth0" tcp dport 6000 drop`,
|
||||
"table ip filter, chain ufw-reject-input",
|
||||
"the found firewall's; retired with it",
|
||||
"the container runtime's own; left",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") {
|
||||
t.Errorf("an undeclared published port is not said to close: %s", line)
|
||||
@@ -326,7 +433,7 @@ func digestIn(t *testing.T, preview string) string {
|
||||
func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
@@ -392,7 +499,7 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
@@ -437,7 +544,7 @@ func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since := time.Now()
|
||||
@@ -480,7 +587,7 @@ func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||
func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Only a loopback listener: nothing off the machine, which is the same silence.
|
||||
@@ -508,7 +615,7 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
|
||||
+507
-20
@@ -24,6 +24,15 @@ import (
|
||||
func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node) error {
|
||||
if !node.Adopted {
|
||||
fmt.Printf(" mode converged\n")
|
||||
// A converged machine holds nothing, and can still run what nobody asked for
|
||||
// (novox/hq ADR 0163): what it reports as strays is said whatever its mode.
|
||||
if said, err := inv.AdoptionOf(ctx, node.Name); err == nil && len(said.Strays) > 0 {
|
||||
showStrays(said.Strays)
|
||||
}
|
||||
// And what filters it, truthfully (novox/hq ADR 0168): the mesh alone, or not.
|
||||
if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil {
|
||||
showFiltering(filtering, false)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Printf(" mode adopted since %s\n",
|
||||
@@ -62,11 +71,81 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
|
||||
if h.Kept != "" {
|
||||
fmt.Printf(" %-17s original kept at %s\n", "", h.Kept)
|
||||
}
|
||||
for _, f := range comparisonLines(h) {
|
||||
fmt.Printf(" %-17s %s\n", "", f)
|
||||
}
|
||||
}
|
||||
showStrays(said.Strays)
|
||||
if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil {
|
||||
showFiltering(filtering, true)
|
||||
}
|
||||
fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime))
|
||||
return nil
|
||||
}
|
||||
|
||||
// showFiltering says what filters a machine, with owners (novox/hq ADR 0168), and for a converged
|
||||
// machine the state of the firewall it was found with. A machine that has not said is not said to
|
||||
// be filtered by anything.
|
||||
func showFiltering(f inventory.Filtering, adopted bool) {
|
||||
if len(f.Filters) == 0 && f.FoundFirewall == nil {
|
||||
return
|
||||
}
|
||||
if fw := f.FoundFirewall; fw != nil && !adopted {
|
||||
switch {
|
||||
case fw.Active:
|
||||
fmt.Printf(" found firewall %s is ACTIVE on this converged machine; the next apply retires it again\n", fw.Kind)
|
||||
case fw.RetiredBy == "removed":
|
||||
fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0180)\n", fw.Kind)
|
||||
case fw.RetiredBy == inventory.FilterMesh || fw.RetiredBy == "mesh":
|
||||
fmt.Printf(" found firewall %s, retired by the mesh; its configuration stays on disk\n", fw.Kind)
|
||||
case fw.RetiredBy != "":
|
||||
fmt.Printf(" found firewall %s, found inactive — not by the mesh\n", fw.Kind)
|
||||
default:
|
||||
fmt.Printf(" found firewall %s, inactive\n", fw.Kind)
|
||||
}
|
||||
}
|
||||
if len(f.Filters) == 0 {
|
||||
return
|
||||
}
|
||||
if f.Alone() {
|
||||
fmt.Printf(" filtered by the mesh alone (%s)\n", filterSummary(f.Filters))
|
||||
return
|
||||
}
|
||||
fmt.Printf(" filtered by NOT the mesh alone: %d rule set(s) the mesh did not write refuse traffic here\n", len(f.Others()))
|
||||
for _, x := range f.Filters {
|
||||
if x.Owner == inventory.FilterOther || x.Owner == inventory.FilterFoundFirewall {
|
||||
fmt.Printf(" %-17s %s — %s: %s\n", "", x.Where, x.Owner, x.Refuses)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" %-17s and its own: %s\n", "", filterSummary(f.Filters))
|
||||
}
|
||||
|
||||
// filterSummary counts a machine's filters by owner: "mesh 2, runtime 3, ban 1".
|
||||
func filterSummary(filters []inventory.Filter) string {
|
||||
counts := map[string]int{}
|
||||
for _, x := range filters {
|
||||
counts[x.Owner]++
|
||||
}
|
||||
var parts []string
|
||||
for _, owner := range []string{inventory.FilterMesh, inventory.FilterRuntime, inventory.FilterBan, inventory.FilterFoundFirewall, inventory.FilterOther} {
|
||||
if n := counts[owner]; n > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%s %d", owner, n))
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// showStrays says what a machine runs that the mesh neither wrote nor holds (ADR 0163).
|
||||
func showStrays(strays []inventory.Stray) {
|
||||
if len(strays) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Printf(" strays %d container(s) the mesh neither wrote nor holds:\n", len(strays))
|
||||
for _, s := range strays {
|
||||
fmt.Printf(" %-17s %s (%s)\n", "", s.Name, s.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq
|
||||
// ADR 0105): what it presented at enrolment, and what it last said about taking it over.
|
||||
func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||
@@ -136,7 +215,28 @@ const DefaultFilter = "nftables"
|
||||
// take is a module's cutover on an adopted node: the operator's act, done when that module's data
|
||||
// has moved. From the next push its resources converge there like any other, replacing what the
|
||||
// node found and holds for it.
|
||||
func take(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
//
|
||||
// **Previewed, and the preview is a comparison** (novox/hq ADR 0163): for every held thing the
|
||||
// module would replace, what runs beside what the module declares, and the difference; the
|
||||
// module's secrets on the machine and where each came from; its settings on the machine. Without
|
||||
// --yes the comparison is printed and nothing changes. `--yes <digest>` cuts over exactly what was
|
||||
// previewed, the way the flip is confirmed: the preview ends with a digest of what it said, and a
|
||||
// take naming an older one, or acting on an account of the machine older than the flip allows, is
|
||||
// refused. A module the machine holds nothing for has nothing to compare, and `--yes` suffices.
|
||||
// takeOptions is what a take was told about the differences it may pass (novox/hq ADR 0163).
|
||||
type takeOptions struct {
|
||||
Yes bool
|
||||
// Digest is the preview's, named with --yes; required whenever the machine holds something
|
||||
// for the module.
|
||||
Digest string
|
||||
Downgrade bool
|
||||
Replace map[string]bool
|
||||
// Mint names the secrets the service shall take a new value for, although the mesh minted
|
||||
// one and the service already has its own (rule 2).
|
||||
Mint map[string]bool
|
||||
}
|
||||
|
||||
func take(ctx context.Context, open *stores, node, module string, opts takeOptions) (string, error) {
|
||||
inv := open.inventory
|
||||
assigned, err := inv.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
@@ -150,27 +250,337 @@ func take(ctx context.Context, open *stores, node, module string) (string, error
|
||||
}
|
||||
}
|
||||
}
|
||||
// The comparison first (novox/hq ADR 0163): every held thing the module would replace, beside
|
||||
// what the module declares, and the differences that refuse unless named.
|
||||
c, err := comparisonFor(ctx, open, node, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
preview, refusals, saw := comparisonOf(module, c, opts)
|
||||
if len(refusals) > 0 {
|
||||
return "", fmt.Errorf("taking %s on %s is refused:\n %s\n%s", module, node,
|
||||
strings.Join(refusals, "\n "), preview)
|
||||
}
|
||||
holds := len(heldOf(c.reported, module)) > 0
|
||||
if holds {
|
||||
preview += "\n preview " + saw
|
||||
}
|
||||
if !opts.Yes {
|
||||
if !holds {
|
||||
return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes` declares it as the mesh's own", node, module), nil
|
||||
}
|
||||
return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes %s` cuts it over as previewed", node, module, saw), nil
|
||||
}
|
||||
if holds {
|
||||
// The take acts on the preview the operator saw, and on an account of the machine that
|
||||
// is still the machine: the same two refusals the flip makes.
|
||||
if age := time.Since(c.reported.At); age > reportFreshFor {
|
||||
return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+
|
||||
"an account newer than %s: run `push %s --wait 2m`, then preview again",
|
||||
node, age.Round(time.Second), reportFreshFor, node)
|
||||
}
|
||||
if opts.Digest == "" {
|
||||
return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+
|
||||
"`take %s %s --yes %s`, once you have read it", module, node, node, module, saw)
|
||||
}
|
||||
if opts.Digest != saw {
|
||||
return preview, fmt.Errorf("what taking %s on %s would replace has changed since preview %s "+
|
||||
"(it is now %s): read the preview above, and run `take %s %s --yes %s` if it is "+
|
||||
"what you want", module, node, opts.Digest, saw, node, module, saw)
|
||||
}
|
||||
}
|
||||
if err := inv.Take(ctx, node, module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
said := fmt.Sprintf("%s is taken on %s", module, node)
|
||||
reported, err := inv.AdoptionOf(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var replaces []string
|
||||
for _, h := range reported.Held {
|
||||
if h.Module == module {
|
||||
replaces = append(replaces, " "+heldLine(h))
|
||||
}
|
||||
}
|
||||
if len(replaces) > 0 {
|
||||
said += "; the next push replaces what the node found and holds for it:\n" +
|
||||
strings.Join(replaces, "\n")
|
||||
if holds {
|
||||
said += "; the next push replaces what the node found and holds for it:\n" + preview
|
||||
}
|
||||
return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil
|
||||
}
|
||||
|
||||
// comparison is everything a take puts beside what the module declares: the machine's account of
|
||||
// what it holds and what is reachable on it, the module's secrets on the machine, its settings
|
||||
// there, and which found networks a setting keeps for each of its containers (by held id).
|
||||
type comparison struct {
|
||||
reported inventory.Adoption
|
||||
secrets []inventory.SecretState
|
||||
layers []catalogue.Layer
|
||||
keeps map[string][]string
|
||||
// settingsRefused is why the module's settings cannot compose with its definition, when
|
||||
// they cannot — the module would be left out of the declaration (rule 6).
|
||||
settingsRefused string
|
||||
}
|
||||
|
||||
func comparisonFor(ctx context.Context, open *stores, node, module string) (comparison, error) {
|
||||
inv := open.inventory
|
||||
var c comparison
|
||||
var err error
|
||||
if c.reported, err = inv.AdoptionOf(ctx, node); err != nil {
|
||||
return c, err
|
||||
}
|
||||
if c.secrets, err = inv.SecretsOf(ctx, node, module); err != nil {
|
||||
return c, err
|
||||
}
|
||||
if c.layers, err = inv.SettingsFor(ctx, node, module); err != nil {
|
||||
return c, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if m, known := shelf[module]; known && len(c.layers) > 0 {
|
||||
if err := catalogue.JudgeSettings(m, c.layers, true); err != nil {
|
||||
c.settingsRefused = err.Error()
|
||||
}
|
||||
if kept, err := catalogue.KeptNetworks(m, c.layers, true); err == nil && len(kept) > 0 {
|
||||
c.keeps = map[string][]string{}
|
||||
for id, networks := range kept {
|
||||
c.keeps[module+"."+id] = networks
|
||||
}
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// heldOf is what a node holds for one module.
|
||||
func heldOf(reported inventory.Adoption, module string) []inventory.Held {
|
||||
var out []inventory.Held
|
||||
for _, h := range reported.Held {
|
||||
if h.Module == module {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// comparisonOf is a take's preview: for every held thing of the module, what runs beside what the
|
||||
// module declares; its secrets and its settings on the machine; and the refusals the differences
|
||||
// earn unless the take named them (novox/hq ADR 0163): an image older than the one running, a
|
||||
// declared file that differs from the found one, a secret the mesh minted for a service whose data
|
||||
// was found. A narrowed port and a shared network are said and not refused. The digest is of what
|
||||
// the preview says, so anything in it changing changes the digest.
|
||||
func comparisonOf(module string, c comparison, opts takeOptions) (preview string, refusals []string, digest string) {
|
||||
var b strings.Builder
|
||||
held := heldOf(c.reported, module)
|
||||
foundData := false
|
||||
for _, h := range held {
|
||||
if h.Kind == "container" || h.Kind == "directory" {
|
||||
foundData = true
|
||||
}
|
||||
fmt.Fprintf(&b, " %s", heldLine(h))
|
||||
if h.Kept != "" {
|
||||
fmt.Fprintf(&b, ", original kept at %s", h.Kept)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
for _, line := range comparisonLinesWith(h, c.keeps[h.ID], c.reported) {
|
||||
fmt.Fprintf(&b, " %s\n", line)
|
||||
}
|
||||
f := factsOf(h)
|
||||
if f.downgrade && !opts.Downgrade {
|
||||
refusals = append(refusals, fmt.Sprintf("%s: the module's image (%s, made %s) is older than the one running (%s, made %s) — "+
|
||||
"a service that migrated its data forward may not start on it; `--downgrade` to take it anyway",
|
||||
h.Target, f.declaredImage, day(f.declaredCreated), f.image, day(f.imageCreated)))
|
||||
}
|
||||
if f.differs && !opts.Replace[h.Target] && !opts.Replace["*"] {
|
||||
refusals = append(refusals, fmt.Sprintf("%s: the module's content differs from the file found; the lines above "+
|
||||
"marked - are lost by taking it; `--replace %s` to replace it anyway, or declare the file partially",
|
||||
h.Target, h.Target))
|
||||
}
|
||||
}
|
||||
// The module's secrets on the machine (rule 2 and 3): a service whose data was found already
|
||||
// has a value for each, so one the mesh minted and nobody accepted refuses unless --mint says
|
||||
// the service shall take a new one.
|
||||
for _, sec := range c.secrets {
|
||||
name := sec.Name
|
||||
if sec.Local != "" {
|
||||
name += " (" + sec.Local + ")"
|
||||
}
|
||||
what := "own secret"
|
||||
accept := fmt.Sprintf("`secret accept <node> %s %s`", module, sec.Name)
|
||||
if !sec.Own() {
|
||||
what = "secret from " + sec.Provider
|
||||
accept = fmt.Sprintf("`secret accept <node> %s %s --provider %s`", module, sec.Name, sec.Provider)
|
||||
if sec.Local != "" {
|
||||
accept = strings.TrimSuffix(accept, "`") + " --local " + sec.Local + "`"
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case sec.Origin == inventory.OriginAccepted:
|
||||
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
||||
case opts.Mint[sec.Name]:
|
||||
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
||||
case foundData:
|
||||
fmt.Fprintf(&b, " %s %s: MINTED by the mesh and not accepted — the running service already has one\n", what, name)
|
||||
refusals = append(refusals, fmt.Sprintf("%s: the mesh minted a value and the service whose data was found "+
|
||||
"already uses its own; %s carries the existing value in, or `--mint %s` says the service shall take "+
|
||||
"the new one", name, accept, sec.Name))
|
||||
default:
|
||||
fmt.Fprintf(&b, " %s %s: minted by the mesh\n", what, name)
|
||||
}
|
||||
}
|
||||
// And its settings on this machine, composed against its definition (rule 1, rule 6).
|
||||
for _, layer := range c.layers {
|
||||
keys := make([]string, 0, len(layer.Values))
|
||||
for k := range layer.Values {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", "))
|
||||
}
|
||||
if c.settingsRefused != "" {
|
||||
fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: %s\n", c.settingsRefused)
|
||||
}
|
||||
preview = strings.TrimRight(b.String(), "\n")
|
||||
sum := sha256.Sum256([]byte(preview))
|
||||
return preview, refusals, hex.EncodeToString(sum[:])[:12]
|
||||
}
|
||||
|
||||
// facts is a held thing's facts as the preview reads them.
|
||||
type facts struct {
|
||||
image, imageCreated, declaredImage, declaredCreated string
|
||||
downgrade, differs bool
|
||||
networks map[string][]string
|
||||
mounts, ports, declaredPorts, declaredVolumes []string
|
||||
difference []string
|
||||
}
|
||||
|
||||
func factsOf(h inventory.Held) facts {
|
||||
var f facts
|
||||
if h.Facts == nil {
|
||||
return f
|
||||
}
|
||||
str := func(k string) string { s, _ := h.Facts[k].(string); return s }
|
||||
list := func(k string) []string {
|
||||
var out []string
|
||||
if raw, ok := h.Facts[k].([]any); ok {
|
||||
for _, x := range raw {
|
||||
if s, ok := x.(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
f.image, f.imageCreated = str("image"), str("image_created")
|
||||
f.declaredImage, f.declaredCreated = str("declared_image"), str("declared_image_created")
|
||||
f.downgrade, _ = h.Facts["downgrade"].(bool)
|
||||
f.differs, _ = h.Facts["differs"].(bool)
|
||||
f.mounts, f.ports = list("mounts"), list("ports")
|
||||
f.declaredPorts, f.declaredVolumes, f.difference = list("declared_ports"), list("declared_volumes"), list("difference")
|
||||
if raw, ok := h.Facts["networks"].(map[string]any); ok {
|
||||
f.networks = map[string][]string{}
|
||||
for name, members := range raw {
|
||||
var out []string
|
||||
if ms, ok := members.([]any); ok {
|
||||
for _, m := range ms {
|
||||
if s, ok := m.(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
f.networks[name] = out
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// comparisonLines says a held thing's facts the way a person weighs them.
|
||||
func comparisonLines(h inventory.Held) []string {
|
||||
return comparisonLinesWith(h, nil, inventory.Adoption{})
|
||||
}
|
||||
|
||||
// comparisonLinesWith is comparisonLines knowing which found networks this machine's setting keeps
|
||||
// for the container (rule 4) and what the machine reports reachable, so a published port's reach
|
||||
// is said beside the port (rule 1).
|
||||
func comparisonLinesWith(h inventory.Held, keeps []string, reported inventory.Adoption) []string {
|
||||
f := factsOf(h)
|
||||
var out []string
|
||||
if f.image != "" || f.declaredImage != "" {
|
||||
line := fmt.Sprintf("runs %s", orNone(f.image))
|
||||
if f.imageCreated != "" {
|
||||
line += " (made " + day(f.imageCreated) + ")"
|
||||
}
|
||||
line += "; the module declares " + orNone(f.declaredImage)
|
||||
switch {
|
||||
case f.declaredCreated != "":
|
||||
line += " (made " + day(f.declaredCreated) + ")"
|
||||
case f.declaredImage != "":
|
||||
line += " (not on the machine yet, so its age is unknown)"
|
||||
}
|
||||
if f.downgrade {
|
||||
line += " — DOWNGRADE"
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
names := make([]string, 0, len(f.networks))
|
||||
for n := range f.networks {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
members := f.networks[n]
|
||||
if len(members) == 0 {
|
||||
continue
|
||||
}
|
||||
if slices.Contains(keeps, n) {
|
||||
out = append(out, fmt.Sprintf("on the network %s with %s — kept by this machine's setting, so they still reach it by name once taken",
|
||||
n, strings.Join(members, ", ")))
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("on the network %s with %s, which may reach it by name and will not once it moves to the module's own network"+
|
||||
" (`settings set %s --node <node>` with {%q: {<container>: [%q]}} keeps it)",
|
||||
n, strings.Join(members, ", "), h.Module, catalogue.NetworksSetting, n))
|
||||
}
|
||||
for _, n := range keeps {
|
||||
if _, found := f.networks[n]; !found {
|
||||
out = append(out, fmt.Sprintf("keeps the network %s by this machine's setting, which the found container is not on", n))
|
||||
}
|
||||
}
|
||||
if len(f.ports) > 0 || len(f.declaredPorts) > 0 {
|
||||
out = append(out, fmt.Sprintf("publishes %s; the module declares %s",
|
||||
orNone(strings.Join(f.ports, " ")), orNone(strings.Join(f.declaredPorts, " "))))
|
||||
// How far each published port reaches now, as the machine reported it: the listener the
|
||||
// runtime publishes for this container. The found firewall's and the guard's rules are
|
||||
// not read; what they let through is said as what was reported reachable.
|
||||
var reach []string
|
||||
for _, r := range reported.Reachable {
|
||||
if r.By == h.Target && r.Published {
|
||||
reach = append(reach, fmt.Sprintf("%s:%d (%s, container port %d)", r.Address, r.Port, r.Protocol, r.ContainerPort))
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(reach) > 0:
|
||||
line := "reachable now at " + strings.Join(reach, ", ")
|
||||
if reported.Firewall != "" && reported.Firewall != "none" {
|
||||
line += ", behind the found firewall (" + reported.Firewall + "), whose rules are not read"
|
||||
}
|
||||
out = append(out, line)
|
||||
case len(f.ports) > 0 && len(reported.Reachable) > 0:
|
||||
out = append(out, "not reported reachable on the machine")
|
||||
}
|
||||
}
|
||||
if len(f.mounts) > 0 || len(f.declaredVolumes) > 0 {
|
||||
out = append(out, fmt.Sprintf("mounts %s; the module declares %s",
|
||||
orNone(strings.Join(f.mounts, " ")), orNone(strings.Join(f.declaredVolumes, " "))))
|
||||
}
|
||||
if f.differs {
|
||||
out = append(out, "the declared content differs from the file found (- lost, + new):")
|
||||
for _, d := range f.difference {
|
||||
out = append(out, " "+d)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// day is a timestamp as a person reads it in a preview: its date.
|
||||
func day(stamp string) string {
|
||||
if len(stamp) >= 10 {
|
||||
return stamp[:10]
|
||||
}
|
||||
return stamp
|
||||
}
|
||||
|
||||
// reportFreshFor is how old a node's account of itself may be for the flip to act on it. A
|
||||
// variable so a test can age a report without waiting.
|
||||
var reportFreshFor = 15 * time.Minute
|
||||
@@ -309,8 +719,12 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
return "", err
|
||||
}
|
||||
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||
outward: plan.PublicDomain != ""}
|
||||
preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
|
||||
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks}
|
||||
filtering, err := inv.FilteringOf(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
preview, saw := previewOf(node, reported, filtering, derived, plan, taken, filter, runs[filter])
|
||||
preview += "\n\n preview " + saw
|
||||
if !yes {
|
||||
return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+
|
||||
@@ -380,7 +794,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
// previewOf is what converging a node will change, before it changes it, and a short digest of
|
||||
// what it said: every reachable thing and its fate, the modules the flip takes and the filter. The
|
||||
// digest is what the flip is asked to act on, so it changes whenever any of those would.
|
||||
func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||
func previewOf(node string, reported inventory.Adoption, filtering inventory.Filtering, derived derivedFilter,
|
||||
plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) {
|
||||
var said []string
|
||||
var b strings.Builder
|
||||
@@ -414,6 +828,18 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||
b.WriteString(" not previewed: traffic the machine routes that is not a published port " +
|
||||
"(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " +
|
||||
"declares it\n")
|
||||
// Which links the filter constrains, said rather than left to the sentence above (novox/hq ADR
|
||||
// 0140). Everything arriving anywhere else is this machine's own guest and keeps working — which
|
||||
// is what a reader most wants to know, because the previous shape of this filter cut a machine's
|
||||
// guests off at the flip without saying so, and that is how this was found.
|
||||
if len(derived.outwardLinks) > 0 {
|
||||
b.WriteString(fmt.Sprintf(" it filters what arrives on: %s, and on the private network "+
|
||||
"— everything its own guests send keeps working\n",
|
||||
strings.Join(derived.outwardLinks, ", ")))
|
||||
} else {
|
||||
b.WriteString(" it has reported no link facing outside, so no filter can be composed " +
|
||||
"for it — the flip is refused until it reports one\n")
|
||||
}
|
||||
|
||||
isTaken := map[string]bool{}
|
||||
for _, m := range taken {
|
||||
@@ -460,6 +886,30 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||
fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw)
|
||||
}
|
||||
said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw))
|
||||
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the found firewall
|
||||
// retired, the runtime's own and bans left, and what the mesh did not write left and named —
|
||||
// so the reader knows before the flip that the machine will not be filtered by the mesh alone.
|
||||
if len(filtering.Filters) > 0 {
|
||||
b.WriteString("\n what filters the machine now, and what the flip does to each:\n")
|
||||
for _, x := range filtering.Filters {
|
||||
fate := "left: " + x.Owner + "'s"
|
||||
switch x.Owner {
|
||||
case inventory.FilterMesh:
|
||||
fate = "the mesh's guard; replaced by its filter"
|
||||
case inventory.FilterFoundFirewall:
|
||||
fate = "the found firewall's; retired with it"
|
||||
case inventory.FilterRuntime:
|
||||
fate = "the container runtime's own; left"
|
||||
case inventory.FilterBan:
|
||||
fate = "a ban list; left"
|
||||
case inventory.FilterOther:
|
||||
fate = "NOT THE MESH'S; left in force — the machine is not filtered by the mesh alone until you remove it"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-50s %s\n", x.Where, fate)
|
||||
fmt.Fprintf(&b, " %-50s %s\n", "", x.Refuses)
|
||||
said = append(said, "filter "+x.Owner+" "+x.Where)
|
||||
}
|
||||
}
|
||||
// Sorted: the same account, reported in another order, is the same preview.
|
||||
sort.Strings(said)
|
||||
sum := sha256.Sum256([]byte(strings.Join(said, "\n")))
|
||||
@@ -473,6 +923,10 @@ type derivedFilter struct {
|
||||
// mesh is every address on the private network; outward says the machine faces outside.
|
||||
mesh []string
|
||||
outward bool
|
||||
// outwardLinks is the links this machine reported as facing outside it (novox/hq ADR 0140).
|
||||
// The filter constrains what arrives on them; everything arriving elsewhere is this machine's
|
||||
// own guest and is not filtered.
|
||||
outwardLinks []string
|
||||
}
|
||||
|
||||
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
||||
@@ -498,6 +952,12 @@ func (d derivedFilter) fate(r inventory.Reach) string {
|
||||
return "stays open — the mesh's own, from anywhere"
|
||||
}
|
||||
}
|
||||
// This machine's own guests ask it for an address and for names, and those two arrive here
|
||||
// (novox/hq ADR 0140). Admitted by the link they arrive on, so a listener bound anywhere but an
|
||||
// outward link keeps answering them.
|
||||
if (r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53) {
|
||||
return "stays open — this machine's own guests asking it for an address and for names"
|
||||
}
|
||||
for _, rule := range d.rules {
|
||||
if rule.Port != r.Port || rule.Protocol != r.Protocol {
|
||||
continue
|
||||
@@ -574,12 +1034,39 @@ func adopt(ctx context.Context, open *stores, node string) (string, error) {
|
||||
|
||||
// takeCommand, convergeCommand and adoptCommand are the command line's adapters to the acts above.
|
||||
func takeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("take <node> <module>")
|
||||
set := flag.NewFlagSet("take", flag.ContinueOnError)
|
||||
yes := set.Bool("yes", false, "cut over as previewed, naming the digest the preview printed after it; "+
|
||||
"without it the comparison is printed and nothing is taken")
|
||||
downgrade := set.Bool("downgrade", false, "take it although the module's image is older than the one running")
|
||||
var replace, mint stringList
|
||||
set.Var(&replace, "replace", "a found file's path whose content the module may replace although it differs (repeatable; * for every one)")
|
||||
set.Var(&mint, "mint", "a secret the service shall take the mesh's minted value for, although it already has its own (repeatable)")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, args[0], args[1]) })
|
||||
if len(positionals) < 2 || len(positionals) > 3 || (len(positionals) == 3 && !*yes) {
|
||||
return errors.New("take <node> <module> [--yes <digest>] [--downgrade] [--replace <path>]... [--mint <secret>]...")
|
||||
}
|
||||
opts := takeOptions{Yes: *yes, Downgrade: *downgrade, Replace: map[string]bool{}, Mint: map[string]bool{}}
|
||||
if len(positionals) == 3 {
|
||||
opts.Digest = positionals[2]
|
||||
}
|
||||
for _, r := range replace {
|
||||
opts.Replace[r] = true
|
||||
}
|
||||
for _, m := range mint {
|
||||
opts.Mint[m] = true
|
||||
}
|
||||
return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, positionals[0], positionals[1], opts) })
|
||||
}
|
||||
|
||||
// stringList is a repeatable flag.
|
||||
type stringList []string
|
||||
|
||||
func (l *stringList) String() string { return strings.Join(*l, ",") }
|
||||
func (l *stringList) Set(v string) error { *l = append(*l, v); return nil }
|
||||
|
||||
func convergeCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("converge", flag.ContinueOnError)
|
||||
yes := set.String("yes", "", "do it, naming the digest the preview printed; without it, only "+
|
||||
|
||||
@@ -95,14 +95,14 @@ func commands(who Authenticator) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return assign(ctx, open, in.Node, in.Module)
|
||||
return assign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||
}))
|
||||
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return unassign(ctx, open, in.Node, in.Module)
|
||||
return unassign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||
}))
|
||||
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
||||
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return take(ctx, open, in.Node, in.Module)
|
||||
return take(ctx, open, in.Node, in.Module, takeOptions{Yes: in.Yes, Digest: in.Digest})
|
||||
}))
|
||||
mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter)
|
||||
@@ -124,8 +124,8 @@ func commands(who Authenticator) http.Handler {
|
||||
type request struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
// Yes, Digest and Filter are converge's: do it rather than preview it, the digest of the
|
||||
// preview it acts on, and which module loads the mesh's filter.
|
||||
// Yes, Digest and Filter are converge's and take's: do it rather than preview it, the digest
|
||||
// of the preview it acts on, and (converge) which module loads the mesh's filter.
|
||||
Yes bool `json:"yes,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Filter string `json:"filter,omitempty"`
|
||||
|
||||
@@ -43,7 +43,7 @@ func askCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
answer, err := link.Ask(ctx, server.Channel(), module, tool, arguments, *wait)
|
||||
answer, err := link.Ask(ctx, server.Bus(), module, tool, arguments, *wait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
||||
|
||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
||||
func aKeeper() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
||||
// are made — without a bus to carry it.
|
||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
||||
t.Helper()
|
||||
digest := pushedBy(t, open, node)
|
||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// d1 runs D1 once.
|
||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
t.Helper()
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-time.Hour)
|
||||
cases := []struct {
|
||||
name string
|
||||
send lastSend
|
||||
want bool
|
||||
}{
|
||||
{"never sent", lastSend{}, false},
|
||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.send.settled("keeper", now); got != c.want {
|
||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A binding to data moves only by a person (novox/hq ADR 0232, issue 273).
|
||||
//
|
||||
// The resolver keeps each consumer of a provision that keeps its data at the provider it was last
|
||||
// sent (catalogue/bound.go) and says what it would have moved. This is where that is said: on the
|
||||
// push that composed it, and by the self-check's D12 every run, as an urgent condition naming the
|
||||
// consumer, both providers and the pin that confirms the move.
|
||||
|
||||
// The condition kinds of D12.
|
||||
const (
|
||||
// kindBindingKept is a move the resolver refused: the consumer is still where its data is.
|
||||
kindBindingKept = "binding-kept"
|
||||
// kindBindingMoved is a consumer about to be sent another provider than the one on record with
|
||||
// no pin naming it — what the resolver exists to make impossible, said if it ever is not.
|
||||
kindBindingMoved = "binding-moved"
|
||||
// kindBindingMoving is a move a pin asked for, not yet sent: a person's act, said so that the
|
||||
// data is moved before the push that carries it.
|
||||
kindBindingMoving = "binding-moving"
|
||||
)
|
||||
|
||||
// probeBindingsID is the self-check's id for this probe, and the source of what it raises.
|
||||
const probeBindingsID = "D12"
|
||||
|
||||
// keptObservation is the urgent condition for one refused move.
|
||||
func keptObservation(k catalogue.KeptBinding) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: bindingID(k.Machine, k.Consumer, k.Provision),
|
||||
Token: kindBindingKept, Kind: kindBindingKept, Machine: k.Machine, Also: otherMachines(k.Machine, k.Bound.Node, k.Would.Node),
|
||||
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("on %s, the mesh %s", k.Machine, k.String())}
|
||||
}
|
||||
|
||||
func bindingID(machine, consumer, provision string) string {
|
||||
return machine + "." + consumer + "." + provision
|
||||
}
|
||||
|
||||
func otherMachines(machine string, nodes ...string) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{machine: true}
|
||||
for _, n := range nodes {
|
||||
if n != "" && !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
|
||||
// condition at once where this process keeps the conditions: a push is when a person is looking.
|
||||
func reportKept(ctx context.Context, plan catalogue.Resolution) {
|
||||
for _, k := range plan.Kept {
|
||||
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
|
||||
if conditionsFrom != nil {
|
||||
o := keptObservation(k)
|
||||
o.Source = probeBindingsID
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
|
||||
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
|
||||
// moves it (said, so the data goes first), and never anything else.
|
||||
func probeBindings(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, d.open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
// D1 says it, with the binding that refused it when that is why.
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
bound, err := inv.BindingsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pins, err := inv.PinsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, bindingFindings(plan, bound, pins)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// bindingFindings is what one machine's resolution says against its record.
|
||||
func bindingFindings(plan catalogue.Resolution, bound map[string]map[string]catalogue.Chosen,
|
||||
pins map[string]catalogue.Chosen) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, k := range plan.Kept {
|
||||
out = append(out, keptObservation(k))
|
||||
}
|
||||
said := map[string]bool{}
|
||||
for _, need := range plan.Needs {
|
||||
if !need.KeepsData || need.ByRecord {
|
||||
continue
|
||||
}
|
||||
was, recorded := bound[need.For][need.Name]
|
||||
now := catalogue.Chosen{Node: need.From, Module: need.Module}
|
||||
if !recorded || was == now || (was.Module == "" && was.Node == now.Node) {
|
||||
continue
|
||||
}
|
||||
id := bindingID(plan.Node, need.For, need.Name)
|
||||
if said[id] {
|
||||
continue
|
||||
}
|
||||
said[id] = true
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Machine: plan.Node,
|
||||
Also: otherMachines(plan.Node, was.Node, now.Node), Resolver: conditions.ResolverOperator}
|
||||
if pin, pinned := pins[need.Name]; pinned && pin.Node == now.Node && (pin.Module == "" || pin.Module == now.Module) {
|
||||
o.Token, o.Kind, o.Severity = kindBindingMoving, kindBindingMoving, conditions.Warning
|
||||
o.Summary = fmt.Sprintf("on %s, %s's %s moves from %s to %s at the next push, by the pin — its data "+
|
||||
"is on %s: move it first", plan.Node, need.For, need.Name, was, now, was)
|
||||
} else {
|
||||
o.Token, o.Kind, o.Severity = kindBindingMoved, kindBindingMoved, conditions.Urgent
|
||||
o.Summary = fmt.Sprintf("on %s, %s's %s would be sent %s, and it is bound to %s, where its data is, "+
|
||||
"with no pin naming %s — a move nothing asked for", plan.Node, need.For, need.Name, now, was, now)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// novox/hq issue 273, ADR 0232: a consumer of a provision that keeps its data moves only by a pin.
|
||||
|
||||
func storeManifests() []catalogue.Manifest {
|
||||
return []catalogue.Manifest{
|
||||
{Module: "store", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
|
||||
{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
|
||||
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
|
||||
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
|
||||
}
|
||||
}
|
||||
|
||||
func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) catalogue.Needed {
|
||||
t.Helper()
|
||||
for _, n := range plan.Needs {
|
||||
if n.For == consumer && n.Name == provision {
|
||||
return n
|
||||
}
|
||||
}
|
||||
t.Fatalf("no %s for %s: %+v", provision, consumer, plan.Needs)
|
||||
return catalogue.Needed{}
|
||||
}
|
||||
|
||||
// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
|
||||
// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
|
||||
// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
|
||||
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
assignAll := func(pairs ...[2]string) {
|
||||
for _, a := range pairs {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
|
||||
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
||||
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
||||
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
||||
[2]string{"laptop", "board"})
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
|
||||
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
|
||||
}
|
||||
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
|
||||
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
|
||||
}
|
||||
|
||||
// Sent, and recorded: only the binding to data.
|
||||
bindings := boundToData(plan, nil)
|
||||
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
|
||||
t.Fatalf("recorded %+v", bindings)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
|
||||
}
|
||||
|
||||
// The laptop's store taken away: refused, not moved to the anchor's empty one.
|
||||
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
|
||||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
|
||||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
|
||||
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
|
||||
}
|
||||
|
||||
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
found, err := probeBindings(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
|
||||
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := inv.Bindings(ctx)
|
||||
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
|
||||
t.Fatalf("%+v, %v", all, err)
|
||||
}
|
||||
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a move sent is still said: %+v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What one machine's resolution says against its record.
|
||||
func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
|
||||
home, anchor := catalogue.Chosen{Node: "home", Module: "store"}, catalogue.Chosen{Node: "anchor", Module: "store"}
|
||||
plan := catalogue.Resolution{Node: "laptop",
|
||||
Kept: []catalogue.KeptBinding{{Machine: "laptop", Consumer: "board", Provision: "postgres-database",
|
||||
Bound: home, Would: anchor}},
|
||||
Needs: []catalogue.Needed{
|
||||
{Name: "postgres-database", For: "board", From: "home", Module: "store", KeepsData: true},
|
||||
{Name: "postgres-database", For: "game", From: "anchor", Module: "store", KeepsData: true},
|
||||
{Name: "wildcard-resolution", For: "network", From: "anchor", Module: "resolver"},
|
||||
}}
|
||||
bound := map[string]map[string]catalogue.Chosen{
|
||||
"board": {"postgres-database": home},
|
||||
"game": {"postgres-database": home},
|
||||
"network": {"wildcard-resolution": {Node: "home", Module: "resolver"}},
|
||||
}
|
||||
found := linted(bindingFindings(plan, bound, nil))
|
||||
if len(found) != 2 {
|
||||
t.Fatalf("found %+v", found)
|
||||
}
|
||||
kept, moved := found[0], found[1]
|
||||
if kept.Kind != kindBindingKept || kept.Severity != conditions.Urgent || kept.Machine != "laptop" ||
|
||||
!strings.Contains(kept.Summary, "would move board's postgres-database from home/store to anchor/store") ||
|
||||
!strings.Contains(kept.Summary, "its data is on home/store") ||
|
||||
!strings.Contains(kept.Summary, "`pin laptop postgres-database anchor store` to confirm a move (and move the data first)") {
|
||||
t.Errorf("kept: %+v", kept)
|
||||
}
|
||||
if moved.Kind != kindBindingMoved || moved.Severity != conditions.Urgent ||
|
||||
!strings.Contains(moved.Summary, "game's postgres-database would be sent anchor/store") {
|
||||
t.Errorf("moved: %+v", moved)
|
||||
}
|
||||
if kept.Key() == moved.Key() {
|
||||
t.Error("two consumers, one condition")
|
||||
}
|
||||
}
|
||||
|
||||
// A push says the move it refused, and raises its condition at once.
|
||||
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
|
||||
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
|
||||
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
|
||||
open[0].Source != probeBindingsID {
|
||||
t.Fatalf("raised %+v", open)
|
||||
}
|
||||
}
|
||||
+411
-137
@@ -2,16 +2,18 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
@@ -44,23 +46,21 @@ func buildOn(ctx context.Context, base string, wait time.Duration) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
against, err := open.inventory.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var on []inventory.Entry
|
||||
for _, e := range held {
|
||||
if e.Manifest.Build == nil {
|
||||
continue
|
||||
}
|
||||
for _, b := range e.Manifest.Build.On {
|
||||
if b.Module == base {
|
||||
on = append(on, e)
|
||||
break
|
||||
}
|
||||
if standsOnModule(e, base, against) {
|
||||
on = append(on, e)
|
||||
}
|
||||
}
|
||||
if len(on) == 0 {
|
||||
fmt.Printf("nothing the mesh holds stands on %s\n", base)
|
||||
return nil
|
||||
}
|
||||
on = orderByBases(on)
|
||||
on = orderByBases(on, against)
|
||||
fmt.Printf("%d module(s) stand on %s:\n", len(on), base)
|
||||
var failed []string
|
||||
for _, e := range on {
|
||||
@@ -136,8 +136,9 @@ func buildCommand(ctx context.Context, args []string) error {
|
||||
//
|
||||
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
|
||||
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
|
||||
// reference and composes the store's address back in where a reference is used. `against` is kept
|
||||
// as announced: it is what the build stood on as the builder saw it, and the catalogue's edge.
|
||||
// reference and composes the store's address back in where a reference is used. `against` — what
|
||||
// the build stood on, the catalogue's edge — is recorded the same way, so an edge names a module's
|
||||
// artifact and not the machine it was pulled from.
|
||||
func buildFrom(result link.BuildResult) inventory.Build {
|
||||
kept := inventory.Build{
|
||||
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
|
||||
@@ -147,7 +148,20 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
|
||||
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
||||
// rebuild the graph rather than a list of names.
|
||||
Path: result.Path, Against: result.Against,
|
||||
Path: result.Path,
|
||||
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
||||
SourceFingerprint: result.SourceFingerprint,
|
||||
}
|
||||
// When it was asked, which is what orders it against another build of the same module
|
||||
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
||||
if asked, ok := link.BuildAskedAt(result.ID); ok {
|
||||
kept.Asked = asked
|
||||
}
|
||||
for _, ref := range result.Against {
|
||||
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
||||
}
|
||||
for _, r := range result.Read {
|
||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
var announced []inventory.Artifact
|
||||
for _, made := range result.Made {
|
||||
@@ -172,10 +186,14 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
func buildsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("builds", flag.ContinueOnError)
|
||||
limit := set.Int("n", 20, "how many to show")
|
||||
logOf := set.String("log", "", "a build's id: print what the build machine said, line by line")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *logOf != "" {
|
||||
return buildLog(ctx, *logOf)
|
||||
}
|
||||
module := ""
|
||||
if len(positionals) == 1 {
|
||||
module = positionals[0]
|
||||
@@ -216,8 +234,8 @@ func buildsCommand(ctx context.Context, args []string) error {
|
||||
if !b.Worked() {
|
||||
outcome = "failed"
|
||||
}
|
||||
fmt.Printf("%-18s %-14s %-10s %s\n",
|
||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"))
|
||||
fmt.Printf("%-18s %-14s %-10s %s %s\n",
|
||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
||||
fmt.Printf(" %s", b.Repository)
|
||||
if b.Ref != "" {
|
||||
fmt.Printf(" at %s", b.Ref)
|
||||
@@ -261,85 +279,45 @@ func builderCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
name := positionals[1]
|
||||
|
||||
management, err := broker.ManagementFromEnvironment()
|
||||
// **The build machine's credential is a module's credential** (novox/hq ADR 0131, design 28
|
||||
// task 5.5): minted into the mesh's records and sealed to the machine as the builder module's
|
||||
// broker secret, usable at the next push — the same act `module issue` performs, and the same
|
||||
// account the composed user list carries. Nothing is created on a server; the bus reads the list.
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The same shape of secret a token carries: enough entropy that guessing is not a strategy,
|
||||
// and safe to put in a URL because that is where it goes.
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
password := base64.RawURLEncoding.EncodeToString(raw)
|
||||
if err := management.CreateBuilderAccount(ctx, name, password); err != nil {
|
||||
m, known := shelf[*module]
|
||||
if !known {
|
||||
return fmt.Errorf("%s is not in the catalogue; `module add` it first", *module)
|
||||
}
|
||||
fmt.Printf("build machine %s: ", name)
|
||||
node := *forNode
|
||||
if node == "" {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == *module && len(e.On) > 0 {
|
||||
node = e.On[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if node == "" {
|
||||
return fmt.Errorf("%s is assigned nowhere; `assign <machine> %s` first, or say --node", *module, *module)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("broker account %s created, scoped to the %s queue and the %s exchange\n\n",
|
||||
name, link.BuildQueue, link.Exchange)
|
||||
|
||||
if *forNode != "" {
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
|
||||
}
|
||||
inv, err := openInventory(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer inv.Close()
|
||||
|
||||
brokerAddr, err := brokerReachableAt(ctx, inv, known, *forNode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The URL and what verifies the broker, together. A mesh's broker presents a certificate
|
||||
// of the mesh's own, which is in no public trust store — so a URL on its own reaches only
|
||||
// a broker somebody else vouches for, and the connection fails at TLS with an error about
|
||||
// an unknown authority rather than about a missing pin.
|
||||
//
|
||||
// **The same two facts a node's token carries** (novox/hq ADR 0004), delivered the same
|
||||
// way: out of band relative to the broker, so what is trusted does not come from the thing
|
||||
// being trusted.
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
}{
|
||||
URL: fmt.Sprintf("amqps://%s:%s@%s/", name, password, brokerAddr),
|
||||
Fingerprint: known.Fingerprint,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, *forNode, *module, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Not printed. It is sealed to that machine and the mesh cannot read it back, which is
|
||||
// the whole point — printing it here would put the one copy that matters on a terminal.
|
||||
fmt.Printf(" sealed to %s, for the %s module. It arrives with the next push.\n",
|
||||
*forNode, *module)
|
||||
fmt.Printf(" run `push %s` to send it\n", *forNode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// The whole line only when the address is known. A URL with a placeholder where the host
|
||||
// should be is a URL somebody pastes and then debugs, and the placeholder is the last thing
|
||||
// they look at.
|
||||
if known, err := broker.FromEnvironment(); err == nil {
|
||||
fmt.Printf(" MESH_BROKER_AMQP=amqps://%s:%s@%s/\n\n", name, password, known.Address)
|
||||
} else {
|
||||
fmt.Printf(" the password is %s\n\n", password)
|
||||
fmt.Printf(" This control plane has no %s, so it cannot say where the broker is.\n"+
|
||||
" Put the password in MESH_BROKER_AMQP on the build machine.\n\n",
|
||||
broker.AddressVar)
|
||||
}
|
||||
// Shown once, like a token, and for the same reason: what is stored is the broker's own hash
|
||||
// of it, and a control plane that could show it back would be a control plane that holds it.
|
||||
fmt.Println("This is the only time it is shown.")
|
||||
return nil
|
||||
return issueOnTheNewBus(ctx, inv, m, node, address)
|
||||
}
|
||||
|
||||
// buildBehind builds every module the mesh holds older than its source has.
|
||||
@@ -386,7 +364,11 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
|
||||
// Bases first: a module built before the module it stands on is built against the old one
|
||||
// and reports success (novox/hq 04-ISSUES/131).
|
||||
stale = orderByBases(stale)
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stale = orderByBases(stale, against)
|
||||
|
||||
var failed []string
|
||||
for _, e := range stale {
|
||||
@@ -414,38 +396,54 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
//
|
||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
||||
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
||||
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
||||
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
||||
dryRun bool) (string, error) {
|
||||
if dryRun && wait != 0 {
|
||||
return "", errors.New("a dry run waited for is `build --dry-run`")
|
||||
}
|
||||
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
||||
// the reason, rather than sent to a machine to fail at `git clone`.
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
// Correlated by something the control plane makes, not by the module's name: two builds of one
|
||||
// module can be in flight, and the second answer is not the first one's.
|
||||
request := link.BuildRequest{
|
||||
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Path: path,
|
||||
Ref: ref,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
DryRun: dryRun,
|
||||
}
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
fmt.Printf(" (%s)", repository)
|
||||
}
|
||||
// The id is how a person follows this build while it runs: `builds --log <id>`.
|
||||
fmt.Printf(" as %s", request.ID)
|
||||
if path != "" {
|
||||
fmt.Printf(" at %s", path)
|
||||
}
|
||||
@@ -454,69 +452,206 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
ask, err := askOver(server)
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
fmt.Printf(" of %s\n", seat)
|
||||
|
||||
if wait == 0 {
|
||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||
// controller takes it in — records the build, registers the module — whether or not anybody
|
||||
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
|
||||
// follows the build by its id instead.
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
||||
"its outcome is not taken in\n", request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
|
||||
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
result, err := ask.Submit(ctx, request, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
return request.ID, err
|
||||
}
|
||||
|
||||
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
|
||||
// nobody asked for, and the difference is the whole of whether somebody should be looking at
|
||||
// something.
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return request.ID, err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return err
|
||||
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
||||
if err != nil {
|
||||
return request.ID, err
|
||||
}
|
||||
|
||||
if result.Failed != "" {
|
||||
// The builder's own words. Wrapping them in something about the control plane would put
|
||||
// two explanations between a person and a build log.
|
||||
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
|
||||
}
|
||||
|
||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||
for _, made := range kept.Made {
|
||||
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
|
||||
}
|
||||
fmt.Printf("\n%s %s, built on %s from %s\n",
|
||||
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
|
||||
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// Parsed with the same parser a hand-written manifest goes through. A second path would be a
|
||||
// second thing to disagree about what a manifest is. The manifest as recorded, so the catalogue
|
||||
// holds references by digest and path and every declaration composes the store's address in.
|
||||
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
|
||||
// result on at once (novox/hq issue 126, ADR 0163): a person choreographing a data move must
|
||||
// know which module will not wait for them.
|
||||
func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module string) {
|
||||
if u, err := inv.UpgradeOf(ctx, module); err == nil && u.RollOut {
|
||||
how := "one machine at a time"
|
||||
if u.Together {
|
||||
how = "every machine at once"
|
||||
}
|
||||
fmt.Printf(" %s rolls out on build: the machines running it are sent this now, %s — "+
|
||||
"`upgrade %s record` first if something must move before it does\n", module, how, module)
|
||||
}
|
||||
}
|
||||
|
||||
// takeIn is what the mesh does with a build's outcome, whoever hears it: the waiting command and
|
||||
// the daemon that follows the role's events both come here (novox/hq issue 176), so a build's
|
||||
// result reaches the catalogue whether or not the asker was still listening.
|
||||
//
|
||||
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
|
||||
// nobody asked for, and the difference is the whole of whether somebody should be looking at
|
||||
// something. Then parsed with the same parser a hand-written manifest goes through — a second path
|
||||
// would be a second thing to disagree about what a manifest is — and registered with where it came
|
||||
// from: **for a source on a seat, as the path and the seat, never the URL just cloned** (ADR 0111),
|
||||
// which the request carried and the outcome echoes. A definition naming an installation is refused
|
||||
// here, where it would enter the catalogue; the build stays recorded and the refusal says which.
|
||||
//
|
||||
// Idempotent: the same outcome taken in twice registers the same module twice, which is one row
|
||||
// written with the same values.
|
||||
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
|
||||
catalogue.Manifest, inventory.Build, error) {
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
}
|
||||
if result.Failed != "" {
|
||||
// The builder's own words. Wrapping them in something about the control plane would put
|
||||
// two explanations between a person and a build log.
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
||||
result.On, result.Repository, result.Failed)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
}
|
||||
|
||||
// Recorded with where it came from, so "is this current?" is answerable without building it
|
||||
// again (novox/hq ADR 0009). **For a source on a seat, as the path and the seat, never the URL
|
||||
// just cloned** (ADR 0111): the URL is where the forge runs today, and recording it would put
|
||||
// the forge's address back into every module built from it. The build log above keeps the URL,
|
||||
// because that is what was cloned.
|
||||
recorded := inventory.Source{
|
||||
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
|
||||
BuiltFrom: result.Commit, Head: result.Commit,
|
||||
// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
|
||||
Against: kept.Against,
|
||||
// When it was asked, so an older request heard later does not replace a newer one
|
||||
// (novox/hq 04-ISSUES/219).
|
||||
Asked: kept.Asked,
|
||||
}
|
||||
if source.Seat != "" {
|
||||
recorded.Repository, recorded.Seat = source.Repository, source.Seat
|
||||
if result.Source != nil && result.Source.Seat != "" {
|
||||
recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
|
||||
}
|
||||
// **A build at a commit does not change the branch a module follows** (novox/hq 04-ISSUES/215):
|
||||
// the commit is built and recorded as what it was built from, and the module keeps following
|
||||
// what it followed before — the repository's default branch for one new to the catalogue.
|
||||
if followedBranch(result.Ref) == "" && result.Ref != "" {
|
||||
recorded.Ref = ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
recorded.Ref = followedBranch(was.Ref)
|
||||
}
|
||||
}
|
||||
if err := namesNoInstallation(manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
result.On, result.Repository, short(result.Commit), err)
|
||||
}
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
|
||||
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
|
||||
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
|
||||
// The branch the module already follows is its trunk — never the branch a build was asked at, or a
|
||||
// build of a feature branch by name would make that branch its trunk.
|
||||
follows := ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
follows = followedBranch(was.Ref)
|
||||
}
|
||||
if err := publishable(result, follows); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
||||
return manifest, kept, err
|
||||
} else if failed {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
||||
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
|
||||
short(result.Commit))
|
||||
}
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
||||
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
||||
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
||||
// byte-reproducible, and the bus rebuilt for another module's merge demanded a planned upgrade.
|
||||
if kept.SourceFingerprint != "" {
|
||||
stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID)
|
||||
if err != nil {
|
||||
return manifest, kept, err
|
||||
}
|
||||
if stands != "" && stands != kept.ID && len(raw) > 0 {
|
||||
if same, err := catalogue.ParseManifest(raw); err == nil && same.Module == manifest.Module {
|
||||
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
|
||||
manifest.Module, short(result.Commit), stands)
|
||||
manifest = same
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||
return err
|
||||
if errors.Is(err, inventory.ErrSuperseded) {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
|
||||
result.On, manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||
// worked and the module is registered.
|
||||
collect(ctx, inv)
|
||||
return manifest, kept, nil
|
||||
}
|
||||
|
||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
||||
|
||||
// publishable says whether a build's outcome may become a module's version: its commit on the module's
|
||||
// trunk, as the build seat read the forge at the build — the branch the module follows when its source
|
||||
// names one, else its repository's default branch. A build seat that could not say — one older than the
|
||||
// rule, building its own successor — is let through and said, so the rule can reach the mesh.
|
||||
func publishable(result link.BuildResult, follows string) error {
|
||||
if result.Check != nil || result.Checked != nil || result.DryRun {
|
||||
return errors.New("a check or a dry run is never published")
|
||||
}
|
||||
if result.Trunk == "" {
|
||||
fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+
|
||||
"registered as before\n", result.ID, short(result.Commit))
|
||||
return nil
|
||||
}
|
||||
trunk := result.Trunk
|
||||
if follows != "" {
|
||||
trunk = follows
|
||||
}
|
||||
on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk)
|
||||
if !on {
|
||||
return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk)
|
||||
}
|
||||
fmt.Printf("\n%s %s, built on %s from %s\n",
|
||||
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -537,16 +672,17 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
ask, err := askOver(server)
|
||||
ask, err := askOverOn(buildSeatHeld(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
|
||||
result, err := ask.Submit(ctx, link.BuildRequest{
|
||||
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository, Path: path, Ref: ref,
|
||||
Held: heldBy(ctx),
|
||||
Held: heldBy(ctx), Seats: seatBases(ctx),
|
||||
DryRun: true,
|
||||
}, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -580,6 +716,10 @@ type answers struct {
|
||||
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
||||
// recorded at send, not at apply).
|
||||
reported []inventory.Reported
|
||||
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
||||
plans []inventory.Plan
|
||||
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||
paused pauseView
|
||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||
// other answer here: those are about a machine that was told something, and this is about one
|
||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||
@@ -589,6 +729,43 @@ type answers struct {
|
||||
// a consequence of the refusals above: a node that does not resolve is not on the network, and
|
||||
// a mesh whose hub is that node has no hub.
|
||||
network string
|
||||
// filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||
// 0168): what filters it beyond the mesh's own, the runtime's plumbing and bans, by name — a
|
||||
// predecessor's chain, a found firewall in force again. Such a machine is not "all well".
|
||||
filtered map[string]inventory.Filtering
|
||||
// untaken is, per machine, each assigned module whose resources the machine is holding as it
|
||||
// found them, and how many — a module that was assigned, sent, and is running none of what it
|
||||
// declares because nothing has taken it (novox/hq ADR 0100, 04-ISSUES/125).
|
||||
//
|
||||
// **Its absence cost an outage.** The module was assigned, the push reported success, this
|
||||
// command said the machine was doing everything it was told, and the module's three containers
|
||||
// did not exist. On the strength of those reports the predecessor's proxy was stopped and every
|
||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||
untaken map[string]map[string]int
|
||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||
unheld []catalogue.Unheld
|
||||
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
|
||||
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
|
||||
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
|
||||
// not be read when they could not — never read as none.
|
||||
conditions []conditions.Condition
|
||||
conditionsUnread string
|
||||
// overflowing is every module whose identity overflows the bound of a provision it requires
|
||||
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||
// this is the one place it is said across the mesh. Not well while there is any.
|
||||
overflowing []catalogue.Overflow
|
||||
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||
handActs *int
|
||||
handActsUnread string
|
||||
// heals is what the healers did in the last seven days (novox/hq to-be 45 §7): acts, and conditions
|
||||
// handed to the operator; healsUnread why it could not be read.
|
||||
heals *healsCount
|
||||
healsUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
@@ -605,12 +782,14 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
|
||||
"base will be told that base is missing: %v\n", err)
|
||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.inventory.Held(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
@@ -634,16 +813,111 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOver(server *link.Server) (link.Builders, error) {
|
||||
address, onNATS, err := broker.OnNATS()
|
||||
func askOverOn(seat string) (link.Builders, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), address); err != nil {
|
||||
return nil, err
|
||||
return link.BuildsOverNATSOn(address, seat)
|
||||
}
|
||||
|
||||
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
|
||||
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
|
||||
// moment the first build-agent is assigned — and a controller that asked the new role before then
|
||||
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
|
||||
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
|
||||
func buildSeatHeld(ctx context.Context) string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
if onNATS {
|
||||
return link.BuildsOverNATS(address)
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
return link.BuildsOverCurrent(server.Channel()), nil
|
||||
return buildSeatAmong(entries)
|
||||
}
|
||||
|
||||
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
|
||||
// assigned module claims it; else the retired role while an assigned module still claims that; else
|
||||
// the current role, which is where every ask goes once the handover is done.
|
||||
func buildSeatAmong(entries []inventory.Entry) string {
|
||||
heldBefore := false
|
||||
for _, e := range entries {
|
||||
if len(e.On) == 0 {
|
||||
continue
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
|
||||
heldBefore = true
|
||||
}
|
||||
}
|
||||
if heldBefore {
|
||||
return link.TheBuildMachineBefore
|
||||
}
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
|
||||
// buildLog prints everything a build machine said about one build, read back from the bus.
|
||||
//
|
||||
// **From the stream, not from a record** (novox/hq ADR 0157). A build's lines are the role's own
|
||||
// events under the build's id, retained with every other event; the mesh keeps no second copy. Read
|
||||
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
|
||||
// for the reading, and filtered by subject, so one build's lines are all that travel.
|
||||
func buildLog(ctx context.Context, id string) error {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// Under whichever build role did it: a build asked of the retired role during the handover
|
||||
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
|
||||
lines := link.BuildLogOf("*", id)
|
||||
sub, err := js.Context().PullSubscribe(lines, "",
|
||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
printed := 0
|
||||
for {
|
||||
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return fmt.Errorf("reading a build's log: %w", err)
|
||||
}
|
||||
for _, msg := range batch {
|
||||
var line link.BuildLine
|
||||
if err := json.Unmarshal(msg.Data, &line); err != nil {
|
||||
fmt.Printf(" ? %s\n", string(msg.Data))
|
||||
continue
|
||||
}
|
||||
at := line.At
|
||||
if t, err := time.Parse(time.RFC3339Nano, line.At); err == nil {
|
||||
at = t.Local().Format("15:04:05")
|
||||
}
|
||||
fmt.Printf("%s %4d [%s] %s\n", at, line.Seq, line.Step, line.Message)
|
||||
printed++
|
||||
}
|
||||
if len(batch) < 200 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if printed == 0 {
|
||||
fmt.Printf("nothing on the bus for build %s: no build by that id in the last week, or a build "+
|
||||
"machine older than this that said nothing while building\n", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func claiming(module, seat string, on ...string) inventory.Entry {
|
||||
return inventory.Entry{
|
||||
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
|
||||
On: on,
|
||||
}
|
||||
}
|
||||
|
||||
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
|
||||
// one while only the builder is assigned, the current one from the first build-agent on, and the
|
||||
// current one when nothing holds either — where every ask goes once the handover is done.
|
||||
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
|
||||
onlyTheBuilder := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
|
||||
}
|
||||
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
|
||||
t.Errorf("with only the builder assigned, asked %q", got)
|
||||
}
|
||||
bothHeld := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine, "home-server"),
|
||||
}
|
||||
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
|
||||
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
|
||||
}
|
||||
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
|
||||
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
|
||||
t.Errorf("with no holder of either, asked %q, want the current role", got)
|
||||
}
|
||||
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
|
||||
t.Errorf("an empty catalogue asks %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A build's outcome is taken in the same way whoever hears it (novox/hq issue 176): recorded, and
|
||||
// the module registered with its source as the seat and path when the request said so — never the
|
||||
// URL. A definition naming an installation is recorded and not registered; a failure is recorded
|
||||
// and said.
|
||||
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||
m, _, err := takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != "shop" {
|
||||
t.Fatalf("registered %q", m.Module)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, held := shelf["shop"]; !held {
|
||||
t.Fatal("the module a heard build produced is not in the catalogue")
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "shop")
|
||||
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
|
||||
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
|
||||
}
|
||||
builds, err := open.inventory.Builds(ctx, "shop", 5)
|
||||
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
|
||||
t.Fatalf("the build is not recorded once: %v %v", builds, err)
|
||||
}
|
||||
|
||||
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
|
||||
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
||||
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
||||
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
||||
t.Fatal("the refused module was registered anyway")
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
|
||||
t.Fatalf("the refused build was not recorded: %v", builds)
|
||||
}
|
||||
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
|
||||
if err == nil || !strings.Contains(err.Error(), "no compiler") {
|
||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the
|
||||
// module keeps following the branch it followed — a new one, the default branch.
|
||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||
result := func(id, ref, commit string) link.BuildResult {
|
||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-1", "main", "1111111aaaa")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||
}
|
||||
|
||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||
r.Manifest, r.Path = other, "modules/letta"
|
||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered,
|
||||
// so a push sends what the newer request built.
|
||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
result := func(asked time.Time, image string) link.BuildResult {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result(newer, "4bcd5f73")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||
t.Errorf("the late build was not recorded: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||
at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC)
|
||||
if got, ok := link.BuildAskedAt(link.NewBuildID(at)); !ok || !got.Equal(at) {
|
||||
t.Errorf("read back %v %v; want %v", got, ok, at)
|
||||
}
|
||||
if got, ok := link.BuildAskedAt("build-1791064317392539762"); !ok || got.Format(time.TimeOnly) != "21:51:57" {
|
||||
t.Errorf("the incident's id reads as %v %v", got, ok)
|
||||
}
|
||||
for _, id := range []string{"b-1", "build-2", "build-", "build-x", ""} {
|
||||
if _, ok := link.BuildAskedAt(id); ok {
|
||||
t.Errorf("%q read as a request time", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236).
|
||||
//
|
||||
// **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the
|
||||
// controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything,
|
||||
// and a version whose data format moved (2.10 → 2.11) cannot be undone by putting the old one back.
|
||||
// So nothing sends a new bus build on its own: its policy is `record` whatever anyone says (the
|
||||
// catalogue's DerivedUpgrade, and `upgrade` refuses a roll-out), a plan builds it and sends nothing, a
|
||||
// cascade holds the machine (ADR 0221), and a push naming that machine is refused while a new bus build
|
||||
// waits for it (busHeld). The one way is this verb: a person, with why, the streams snapshotted first,
|
||||
// whether it can be reverted said before it starts, the step said as `bus-maintenance` while it runs,
|
||||
// and every stream, durable consumer and a round trip checked after (H-bus) — or the step said failed,
|
||||
// with its snapshot as the way back.
|
||||
|
||||
// busStepProbe is the registry's row for the step; its kinds.
|
||||
const (
|
||||
busStepProbe = "DB"
|
||||
kindBusMaintenance = "bus-maintenance"
|
||||
kindBusUpgradeFailed = "bus-upgrade-failed"
|
||||
)
|
||||
|
||||
// busStepBound is how long after its start a bus upgrade must be followed by a healthy bus.
|
||||
var busStepBound = 15 * time.Minute
|
||||
|
||||
// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is: the bus
|
||||
// machine's backup holder backs the bus module up now, whose dump is the streams' snapshot (novox/hq ADR
|
||||
// 0235). A variable so a test takes none. A person who took one by hand says where with --snapshot-taken,
|
||||
// and then none is taken — for a bus whose module does not yet carry the snapshot program.
|
||||
var takeBusSnapshot = snapshotTheBusNow
|
||||
|
||||
// busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per
|
||||
// machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds
|
||||
// no bus module.
|
||||
type busPending struct {
|
||||
module string
|
||||
machines []string
|
||||
from map[string]string
|
||||
to string
|
||||
// same are the commits whose build was made from the same source as the build the mesh holds, or
|
||||
// made the same artifacts and manifest (novox/hq issue 280).
|
||||
same map[string]bool
|
||||
}
|
||||
|
||||
// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the
|
||||
// two builds were made from the same source, or made the same artifacts from the same manifest — a
|
||||
// rebuild of the same source for another module's merge changes nothing the machine runs, whatever
|
||||
// image digest it made (novox/hq issue 280).
|
||||
func (b busPending) moves(machine string) bool {
|
||||
from, known := b.from[machine]
|
||||
if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) {
|
||||
return false
|
||||
}
|
||||
return !known || !b.same[from]
|
||||
}
|
||||
|
||||
// pendingBus reads what a bus upgrade would do.
|
||||
func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, error) {
|
||||
var b busPending
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for name, m := range shelf {
|
||||
if catalogue.ProvidesBus(m) {
|
||||
b.module = name
|
||||
}
|
||||
}
|
||||
if b.module == "" {
|
||||
return b, nil
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.to = current[b.module].Commit
|
||||
if b.machines, err = inv.Running(ctx, b.module); err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.from, b.same = map[string]string{}, map[string]bool{}
|
||||
made, err := inv.BuildFingerprints(ctx, b.module)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for commit, refs := range made {
|
||||
b.same[commit] = refs != "" && refs == made[b.to]
|
||||
}
|
||||
sources, err := inv.BuildSourceFingerprints(ctx, b.module)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for commit, src := range sources {
|
||||
if src != "" && src == sources[b.to] {
|
||||
b.same[commit] = true
|
||||
}
|
||||
}
|
||||
for _, n := range b.machines {
|
||||
sent, known, err := inv.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
if known {
|
||||
if c, carried := sent[b.module]; carried {
|
||||
b.from[n] = c
|
||||
}
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// busHeld names the machines a push may not send because sending them would replace the bus: the
|
||||
// planned step's, not a push's (ADR 0236). Said with the remedy.
|
||||
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
|
||||
b, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, n := range machines {
|
||||
for _, holder := range b.machines {
|
||||
if n == holder && b.moves(n) {
|
||||
out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+
|
||||
"`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)",
|
||||
n, b.module, short(orNotKnown(b.from[n])), short(b.to))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busCommand is `bus` — what a bus upgrade would do and how the last went — and `bus upgrade`.
|
||||
func busCommand(ctx context.Context, args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
||||
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
||||
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
||||
irreversible := set.Bool("irreversible", false, "the new version cannot be undone by putting the old one back, "+
|
||||
"and this is the person's explicit word that it runs anyway")
|
||||
why := addHandActFlags(set)
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>]]")
|
||||
}
|
||||
switch sub {
|
||||
case "":
|
||||
return busStatus(ctx)
|
||||
case "upgrade":
|
||||
default:
|
||||
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub)
|
||||
}
|
||||
// Everything refused before anything is done.
|
||||
if err := why.require("bus upgrade"); err != nil {
|
||||
return err
|
||||
}
|
||||
if *reversible == *irreversible {
|
||||
return errors.New("bus upgrade says, before it starts, whether the new version can be undone by putting the " +
|
||||
"old one back: --reversible, or --irreversible as your explicit word that it runs anyway (to-be 45 §8). " +
|
||||
"Nothing was done")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
b, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.module == "" {
|
||||
return errors.New("the mesh holds no module that provides its bus: there is nothing to upgrade")
|
||||
}
|
||||
var moving []string
|
||||
for _, n := range b.machines {
|
||||
if b.moves(n) {
|
||||
moving = append(moving, n)
|
||||
}
|
||||
}
|
||||
if len(moving) == 0 {
|
||||
fmt.Printf("every machine running %s runs the build the mesh holds (%s): nothing to upgrade\n", b.module, short(b.to))
|
||||
return nil
|
||||
}
|
||||
where := strings.TrimSpace(*snapshot)
|
||||
if where == "" {
|
||||
for _, n := range moving {
|
||||
fmt.Printf("snapshotting the bus's streams on %s first (its backup holder, ADR 0235)…\n", n)
|
||||
if where, err = takeBusSnapshot(ctx, b.module, n); err != nil {
|
||||
return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w — a snapshot "+
|
||||
"taken by hand is said with --snapshot-taken <where>", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
from := map[string]bool{}
|
||||
for _, n := range moving {
|
||||
from[orNotKnown(b.from[n])] = true
|
||||
}
|
||||
step, err := inv.StartBusStep(ctx, inventory.BusStep{Module: b.module, Machines: moving,
|
||||
From: strings.Join(sortedKeys(from), ", "), To: b.to, Snapshot: where, Reversible: *reversible,
|
||||
By: link.Caller(), Why: strings.TrimSpace(*why.why)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cause := "bus-upgrade"
|
||||
if strings.TrimSpace(*why.cause) == "" {
|
||||
why.cause = &cause
|
||||
}
|
||||
why.record(ctx, "bus upgrade", append([]string{b.module}, moving...))
|
||||
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
|
||||
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
|
||||
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
|
||||
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
|
||||
if err != nil {
|
||||
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
|
||||
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
|
||||
}
|
||||
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
||||
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
|
||||
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busStatus is `bus`: what an upgrade would do, and the last step.
|
||||
func busStatus(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
b, err := pendingBus(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.module == "" {
|
||||
fmt.Println("the mesh holds no module that provides its bus")
|
||||
} else {
|
||||
fmt.Printf("the bus is %s, built %s, on %s; never rolled out — a planned step (`bus upgrade`)\n", b.module,
|
||||
short(b.to), orNone(strings.Join(b.machines, ", ")))
|
||||
for _, n := range b.machines {
|
||||
state := "runs it"
|
||||
if b.moves(n) {
|
||||
state = "runs " + short(orNotKnown(b.from[n])) + ": `bus upgrade` replaces it"
|
||||
}
|
||||
fmt.Printf(" %-10s %s\n", n, state)
|
||||
}
|
||||
}
|
||||
fmt.Println(" `bus upgrade` has the bus machine's backup holder snapshot the streams first (ADR 0235)")
|
||||
s, found, err := open.inventory.LatestBusStep(ctx)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
state := "running since " + s.Started.Local().Format("2006-01-02 15:04")
|
||||
if s.Ended != nil {
|
||||
state = s.Outcome + " at " + s.Ended.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
fmt.Printf("last step %d: %s → %s on %s by %s (%s): %s; snapshot %s\n", s.ID, s.From, short(s.To),
|
||||
strings.Join(s.Machines, ", "), orNone(s.By), s.Why, state, s.Snapshot)
|
||||
if s.Found != "" {
|
||||
fmt.Printf(" %s\n", s.Found)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeBusStep is DB: a bus upgrade running is said as `bus-maintenance`; the bus healthy again after
|
||||
// the machines reported the new build ends it done; past its bound, unhealthy, it ends failed and is
|
||||
// said — urgent, with its snapshot — while the bus is still not healthy.
|
||||
func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found {
|
||||
return nil, err
|
||||
}
|
||||
if s.Ended != nil && s.Outcome != "failed" {
|
||||
return nil, nil
|
||||
}
|
||||
problems, err := busHealth(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applied := true
|
||||
for _, r := range reports {
|
||||
for _, n := range s.Machines {
|
||||
if r.Node == n && (!r.Current || r.Outcome != inventory.OutcomeApplied || r.At == nil || r.At.Before(s.Started)) {
|
||||
applied = false
|
||||
problems = append(problems, n+" has not reported the new bus applied")
|
||||
}
|
||||
}
|
||||
}
|
||||
id := s.Module
|
||||
if s.Ended == nil {
|
||||
switch {
|
||||
case applied && len(problems) == 0:
|
||||
return nil, inv.EndBusStep(ctx, s.ID, "done", "the bus answered healthy after the upgrade")
|
||||
case time.Since(s.Started) > busStepBound:
|
||||
found := strings.Join(problems, "; ")
|
||||
if err := inv.EndBusStep(ctx, s.ID, "failed", found); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Found = found
|
||||
default:
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "maintenance",
|
||||
Kind: kindBusMaintenance, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus is being upgraded (step %d, %s → %s on %s, by %s: %s); its snapshot is %s",
|
||||
s.ID, s.From, short(s.To), strings.Join(s.Machines, ", "), orNone(s.By), s.Why, s.Snapshot),
|
||||
Said: orNone(strings.Join(problems, "; "))}}, nil
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil // failed, and healthy since: nothing wrong now
|
||||
}
|
||||
way := "put the old build back"
|
||||
if !s.Reversible {
|
||||
way = "restore the snapshot"
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "upgrade-failed",
|
||||
Kind: kindBusUpgradeFailed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the bus upgrade (step %d, %s → %s) did not end healthy within %s: %s — the way back is to %s (%s)",
|
||||
s.ID, s.From, short(s.To), busStepBound, strings.Join(problems, "; "), way, s.Snapshot)}}, nil
|
||||
}
|
||||
|
||||
func sortedKeys(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// busSnapshotWithin is how long the bus machine's backup holder is given to take the bus's snapshot.
|
||||
var busSnapshotWithin = 15 * time.Minute
|
||||
|
||||
// snapshotTheBusNow asks the bus machine's backup holder to back the bus module up now — its dump is
|
||||
// the streams' snapshot (novox/hq ADR 0235) — and waits until it says a backup newer than the ask:
|
||||
// where the snapshot is, as a person reads it. The serving controller's connection, or one of its own.
|
||||
func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) {
|
||||
var where string
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
asked := time.Now()
|
||||
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", node,
|
||||
map[string]any{"module": module}, 30*time.Second)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s's backup holder would not take the bus's snapshot: %s", node, answer.Error)
|
||||
}
|
||||
deadline := time.Now().Add(busSnapshotWithin)
|
||||
for {
|
||||
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", node, map[string]any{}, 10*time.Second)
|
||||
if err == nil && answer.Error == "" {
|
||||
if measured, err := readHolder(answer.Result); err == nil {
|
||||
for item, m := range measured[module] {
|
||||
if m.LastBackup != nil && m.LastBackup.After(asked) && m.Error == "" {
|
||||
where = fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, item,
|
||||
m.LastBackup.UTC().Format(time.RFC3339))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s; the bus is "+
|
||||
"not replaced", node, busSnapshotWithin)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
}
|
||||
})
|
||||
return where, err
|
||||
}
|
||||
|
||||
// errBusWaits is a send refused because it would replace the bus outside its planned step.
|
||||
var errBusWaits = errors.New("a new bus build waits for its planned step")
|
||||
|
||||
type busStepKey struct{}
|
||||
|
||||
// withBusStep marks a send as the bus's planned step: the one send that may replace the bus.
|
||||
func withBusStep(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, busStepKey{}, true)
|
||||
}
|
||||
|
||||
func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on }
|
||||
@@ -0,0 +1,258 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The bus's own certificate, made by the mesh rather than borrowed from an image.
|
||||
//
|
||||
// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq
|
||||
// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's
|
||||
// image, which worked while the broker was one that happened to carry it and stopped the day the
|
||||
// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be
|
||||
// raised. Substituting another image the bundle names does not help — none of them carry it
|
||||
// either.
|
||||
//
|
||||
// So the program that needs a certificate makes one. It is the mesh's own binary, already on the
|
||||
// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the
|
||||
// image it writes into but a mounted directory.
|
||||
//
|
||||
// **Self-signed, and that is the design** — a host pins this server's exact certificate and
|
||||
// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at
|
||||
// this moment in a bootstrap there is no mesh to ask one of.
|
||||
//
|
||||
// Idempotent, because the step is applied again on every reconcile and a second certificate would
|
||||
// be one the hosts that pinned the first no longer believe.
|
||||
|
||||
// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the
|
||||
// loopback address the machine's own foundation dials.
|
||||
var busCertificateNames = []string{"mesh-broker"}
|
||||
|
||||
const busCertificateLife = 10 * 365 * 24 * time.Hour
|
||||
|
||||
// busCertificate makes the bus's certificate in a directory, or says whether one is there.
|
||||
//
|
||||
// broker certificate --into /tls make it if it is not there
|
||||
// broker certificate --check --into /tls exit non-zero unless a usable pair is
|
||||
func busCertificate(args []string) error {
|
||||
into, check := "", false
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--check":
|
||||
check = true
|
||||
case "--into":
|
||||
if i+1 >= len(args) {
|
||||
return errors.New("--into needs a directory")
|
||||
}
|
||||
into = args[i+1]
|
||||
i++
|
||||
default:
|
||||
return fmt.Errorf("broker certificate [--check] --into <directory>: %q", args[i])
|
||||
}
|
||||
}
|
||||
if into == "" {
|
||||
return errors.New("broker certificate [--check] --into <directory>")
|
||||
}
|
||||
crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")
|
||||
|
||||
if usable, err := busCertificateUsable(crt, key); err != nil {
|
||||
return err
|
||||
} else if usable {
|
||||
fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt)
|
||||
return nil
|
||||
}
|
||||
if check {
|
||||
// Said as a failure, because that is what the caller asked: a bootstrap's verify runs
|
||||
// this and a false answer is what makes the step run.
|
||||
return fmt.Errorf("no usable certificate and key at %s", into)
|
||||
}
|
||||
return writeBusCertificate(crt, key)
|
||||
}
|
||||
|
||||
// busCertificateUsable says whether a certificate and its key are both there and parse.
|
||||
//
|
||||
// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted
|
||||
// between the two files leaves behind, and a step that treated it as done would hand the server a
|
||||
// certificate with no key and report success.
|
||||
func busCertificateUsable(crt, key string) (bool, error) {
|
||||
certPEM, err := os.ReadFile(crt)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
keyPEM, err := os.ReadFile(key)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := tlsPairParses(certPEM, keyPEM); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) {
|
||||
block, _ := pem.Decode(certPEM)
|
||||
if block == nil || block.Type != "CERTIFICATE" {
|
||||
return nil, errors.New("not a certificate")
|
||||
}
|
||||
certificate, err := x509.ParseCertificate(block.Bytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyBlock, _ := pem.Decode(keyPEM)
|
||||
if keyBlock == nil {
|
||||
return nil, errors.New("not a key")
|
||||
}
|
||||
if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil {
|
||||
if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return certificate, nil
|
||||
}
|
||||
|
||||
func writeBusCertificate(crt, key string) error {
|
||||
private, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: busCertificateNames[0]},
|
||||
DNSNames: busCertificateNames,
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(busCertificateLife),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pkcs8, err := x509.MarshalPKCS8PrivateKey(private)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// **The key first, and only then the certificate**, so the pair a reader finds is never a
|
||||
// certificate whose key has not been written yet — the one order in which an interruption
|
||||
// leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable).
|
||||
if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n",
|
||||
busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busAccounts writes the mesh's composed user list to a file.
|
||||
//
|
||||
// **For genesis, where no declaration can deliver it** (novox/hq 04-ISSUES/146). Everywhere else
|
||||
// the list reaches the machine running the bus as a resource of the module that holds it — which
|
||||
// requires that machine to be an enrolled node, and at genesis it is not: the first node cannot
|
||||
// enrol because the account it would enrol with cannot be composed onto a bus it has no declaration
|
||||
// for. The installer breaks that circle by placing the file itself, once, and the module takes the
|
||||
// file over from its first push.
|
||||
//
|
||||
// The same composition, not a second one: this asks the store for the same records and renders them
|
||||
// with the same composer the declaration uses. A genesis that hand-wrote an account would be a
|
||||
// second statement of who may say what, able to disagree with the first.
|
||||
//
|
||||
// **It writes to standard output unless told a file**, and that is the point: the control plane
|
||||
// composes and says what it composed, and whoever is raising the machine puts it where that
|
||||
// machine's bus reads it. A control plane that wrote into the bus's own directory would have to
|
||||
// know where that is and how to make the server re-read it — which is the module's knowledge, and
|
||||
// the module is what takes this over on the first push.
|
||||
//
|
||||
// broker accounts > /var/lib/mesh-bus-conf/accounts.conf
|
||||
func busAccounts(ctx context.Context, args []string) error {
|
||||
into := ""
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--into":
|
||||
if i+1 >= len(args) {
|
||||
return errors.New("--into needs a file")
|
||||
}
|
||||
into = args[i+1]
|
||||
i++
|
||||
default:
|
||||
return fmt.Errorf("broker accounts --into <file>: %q", args[i])
|
||||
}
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kept, err := open.inventory.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashes := make(map[string]string, len(kept))
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
filled, missing := broker.WithPasswords(users, hashes)
|
||||
if len(missing) > 0 {
|
||||
// To standard error, always: the composed file may be going to standard output, and a
|
||||
// remark in the middle of it is a configuration the server refuses to parse.
|
||||
fmt.Fprintf(os.Stderr, "leaving out %d user(s) the mesh has minted no credential for: %s\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
if len(filled) == 0 {
|
||||
return errors.New("not one user has a credential, so this list would refuse every " +
|
||||
"connection in the mesh")
|
||||
}
|
||||
accounts, err := broker.ComposeAccounts(filled)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if into == "" {
|
||||
fmt.Print(accounts)
|
||||
return nil
|
||||
}
|
||||
if err := os.WriteFile(into, []byte(accounts), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("wrote %d user(s) to %s\n", len(filled), into)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for
|
||||
// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is
|
||||
// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once.
|
||||
|
||||
func TestTheBusCertificateLoadsAsAServersWould(t *testing.T) {
|
||||
into := t.TempDir()
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatalf("the bus could not be given a certificate: %v", err)
|
||||
}
|
||||
|
||||
// The check a cheaper test would not make. The key was present and valid and the server could
|
||||
// not start, once, because nothing loaded the pair the way a server loads it
|
||||
// (novox/hq 04-ISSUES/014).
|
||||
pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key"))
|
||||
if err != nil {
|
||||
t.Fatalf("a TLS server cannot load what was written: %v", err)
|
||||
}
|
||||
leaf := pair.Leaf
|
||||
if leaf == nil {
|
||||
if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := leaf.VerifyHostname("mesh-broker"); err != nil {
|
||||
t.Errorf("the certificate is not for the name the bus is reached by: %v", err)
|
||||
}
|
||||
if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" {
|
||||
t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses)
|
||||
}
|
||||
|
||||
// The key is not readable by anything else on the machine; the certificate is public and is.
|
||||
key, err := os.Stat(filepath.Join(into, "tls.key"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if key.Mode().Perm() != 0o600 {
|
||||
t.Errorf("the key is %v", key.Mode().Perm())
|
||||
}
|
||||
crt, err := os.Stat(filepath.Join(into, "tls.crt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if crt.Mode().Perm() != 0o644 {
|
||||
t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
// **Made once.** The step is applied again on every reconcile, and a second certificate is one the
|
||||
// hosts that pinned the first no longer believe (novox/hq ADR 0004).
|
||||
func TestTheBusCertificateIsMadeOnce(t *testing.T) {
|
||||
into := t.TempDir()
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := os.ReadFile(filepath.Join(into, "tls.crt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := os.ReadFile(filepath.Join(into, "tls.crt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(first) != string(again) {
|
||||
t.Fatal("running it twice replaced the certificate every host had pinned")
|
||||
}
|
||||
}
|
||||
|
||||
// The verify half: false before, true after, which is what makes the bootstrap run the step at all.
|
||||
func TestTheCheckIsFalseUntilThereIsAPair(t *testing.T) {
|
||||
into := t.TempDir()
|
||||
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
|
||||
t.Fatal("an empty directory reported a usable certificate")
|
||||
}
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busCertificate([]string{"--check", "--into", into}); err != nil {
|
||||
t.Fatalf("the certificate it just made does not satisfy its own check: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that
|
||||
// called it done would hand the server a certificate with no key and report success.
|
||||
func TestACertificateWithoutItsKeyIsNotUsable(t *testing.T) {
|
||||
into := t.TempDir()
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(into, "tls.key")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
|
||||
t.Fatal("a certificate with no key passed the check")
|
||||
}
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil {
|
||||
t.Fatalf("it did not replace the unusable pair: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhereToWriteIsRequired(t *testing.T) {
|
||||
if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") {
|
||||
t.Fatalf("it did not ask where to write: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
|
||||
// D6 and D7).
|
||||
//
|
||||
// **What the controller asserts at its start and what the self-check expects to find are one
|
||||
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
|
||||
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
|
||||
// the drift rather than the fault.
|
||||
|
||||
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
|
||||
// can now hear a declaration.
|
||||
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
if err := broker.Raise(r, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||
// consumer nothing had created (2026-09-28).
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return nil, errors.Join(failed...)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// What raises the condition a send says when the objects it implies could not be asserted, and its kind.
|
||||
const (
|
||||
sourceBusObjects = "bus-objects"
|
||||
kindBusObjectsUnasserted = "bus-objects-unasserted"
|
||||
)
|
||||
|
||||
// assertOnSend asserts, on the bus a send is about to use, every object assertBusObjects derives —
|
||||
// **whenever a declaration is sent, not only when the controller starts** (novox/hq issue 208).
|
||||
//
|
||||
// A module assigned after the controller started was sent its declaration and found no consumer to
|
||||
// bind (`consumer not found`, messenger on 2026-10-06), and a seat holder assigned after it found no
|
||||
// worker: the objects a declaration implies were asserted at start and nowhere else, so they existed
|
||||
// only for what was assigned before the last restart. The same derivation, not a second list of what
|
||||
// a send needs: what start asserts, the self-check expects and a send asserts are one answer. Every
|
||||
// part is idempotent, so asserting the whole of it again is the no-op a restart already relies on.
|
||||
//
|
||||
// **A failure is said and raised, and the send goes on.** The objects are the mesh's, not the
|
||||
// machines' being sent: holding every machine back for one consumer that none of them may use would
|
||||
// turn one fault into all of them, and the declarations are not what is wrong. It is never silent —
|
||||
// said in the send's own output and raised as a condition, which the next send that asserts them
|
||||
// clears — and the start-time raise still refuses to serve without them.
|
||||
func assertOnSend(ctx context.Context, inv *inventory.Inventory, r broker.Raiser, indent string) error {
|
||||
_, err := assertBusObjects(ctx, inv, r)
|
||||
var observed []conditions.Observation
|
||||
if err != nil {
|
||||
fmt.Printf("%sTHE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES: %v\n", indent, err)
|
||||
fmt.Printf("%s a module may find no consumer to bind, or a holder no worker; sent anyway, raised as "+
|
||||
"condition %s, and asserted again by the next send\n", indent, unassertedObservation(err).Key())
|
||||
observed = append(observed, unassertedObservation(err))
|
||||
}
|
||||
// Observed when it failed, cleared when it did not: a send that asserted everything is the
|
||||
// observation that the bus holds what it should.
|
||||
if kerr := withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
return k.Reconcile(ctx, sourceBusObjects, observed)
|
||||
}); kerr != nil {
|
||||
fmt.Printf("%sand whether the bus holds what this send implies could not be kept as a condition: %v\n",
|
||||
indent, kerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// unassertedObservation is a send's failure to assert the bus's objects, as a condition.
|
||||
func unassertedObservation(err error) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "objects", Token: "unasserted",
|
||||
Kind: kindBusObjectsUnasserted, Severity: conditions.Warning, Source: sourceBusObjects,
|
||||
Summary: "the bus's streams and consumers could not be asserted when a declaration was sent: " +
|
||||
"a module may find no consumer to bind, or a seat's holder no worker",
|
||||
Said: err.Error()}
|
||||
}
|
||||
|
||||
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return broker.ConsumersOf(users), nil
|
||||
}
|
||||
|
||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
return len(consumers), err
|
||||
}
|
||||
|
||||
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
|
||||
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||
var rec recordingRaiser
|
||||
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
|
||||
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
|
||||
}
|
||||
return rec.streams, rec.consumers, nil
|
||||
}
|
||||
|
||||
// recordingRaiser keeps what it was asked to assert and asserts nothing.
|
||||
type recordingRaiser struct {
|
||||
streams []broker.Stream
|
||||
consumers []broker.Consumer
|
||||
}
|
||||
|
||||
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
|
||||
r.streams = append(r.streams, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||
r.consumers = append(r.consumers, c)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
// only when the controller starts.
|
||||
|
||||
// aCarriedConsumer is the runtime on laptop and, carried by it, a module that consumes an event: the
|
||||
// shape of messenger on 2026-10-06, assigned after the controller started.
|
||||
func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}})
|
||||
register(t, open, catalogue.Manifest{Module: "messenger", Version: "1",
|
||||
Consumes: []string{"billing.order.placed"}})
|
||||
for _, m := range []string{catalogue.RuntimeModule, "messenger"} {
|
||||
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
consumers, err := moduleConsumers(t.Context(), open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if c.Module == "messenger" && c.Node == "laptop" {
|
||||
return c.Consumer
|
||||
}
|
||||
}
|
||||
t.Fatalf("messenger on laptop is derived no consumer: %+v", consumers)
|
||||
return broker.Consumer{}
|
||||
}
|
||||
|
||||
// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
|
||||
// started, and the worker its seat's queue should have for it.
|
||||
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
|
||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range inventory.MeshSeats() {
|
||||
if s.Name == "node-build-agent" {
|
||||
c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
|
||||
if !needed {
|
||||
t.Fatal("the build agent's seat needs no worker")
|
||||
}
|
||||
return c
|
||||
}
|
||||
}
|
||||
t.Fatal("the mesh declares no build agent's seat")
|
||||
return broker.Consumer{}
|
||||
}
|
||||
|
||||
// asserted says whether a recording holds a consumer by stream and name.
|
||||
func asserted(rec *recordingRaiser, want broker.Consumer) bool {
|
||||
for _, c := range rec.consumers {
|
||||
if c.Stream == want.Stream && c.Name == want.Name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// What a send asserts includes what was assigned after the start's assertion: a carried module's
|
||||
// consumer and a late holder's worker. The derivation is the start's own, so this holds without a bus.
|
||||
func TestASendAssertsWhatWasAssignedAfterStart(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
var atStart recordingRaiser
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, &atStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
if asserted(&atStart, consumer) || asserted(&atStart, worker) {
|
||||
t.Fatal("the start asserted what was not yet assigned; the test proves nothing")
|
||||
}
|
||||
var onSend recordingRaiser
|
||||
if err := assertOnSend(t.Context(), open.inventory, &onSend, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !asserted(&onSend, consumer) {
|
||||
t.Fatalf("messenger's consumer %s on %s is not asserted by the send: %+v", consumer.Name, consumer.Stream, onSend.consumers)
|
||||
}
|
||||
if !asserted(&onSend, worker) {
|
||||
t.Fatalf("the late holder's worker %s on %s is not asserted by the send: %+v", worker.Name, worker.Stream, onSend.consumers)
|
||||
}
|
||||
}
|
||||
|
||||
// failingRaiser refuses one consumer by name and records everything it was asked.
|
||||
type failingRaiser struct {
|
||||
recordingRaiser
|
||||
refuse string
|
||||
}
|
||||
|
||||
func (f *failingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||
f.consumers = append(f.consumers, c)
|
||||
if c.Name == f.refuse {
|
||||
return errors.New("nats: API error: code=503 description=insufficient resources")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A send whose objects cannot be asserted says so in its output and raises a condition — and the next
|
||||
// send that asserts them clears it. The consumers after the refused one are still asked for.
|
||||
func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
failing := &failingRaiser{refuse: consumer.Name}
|
||||
|
||||
var sendErr error
|
||||
out := stdoutOf(t, func() error {
|
||||
sendErr = assertOnSend(t.Context(), open.inventory, failing, " ")
|
||||
return nil
|
||||
})
|
||||
if sendErr == nil || !strings.Contains(sendErr.Error(), "how messenger on laptop hears what it consumes") {
|
||||
t.Fatalf("the failure is not answered: %v", sendErr)
|
||||
}
|
||||
if !strings.Contains(out, "THE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES") ||
|
||||
!strings.Contains(out, "insufficient resources") || !strings.Contains(out, "bus.objects.unasserted") {
|
||||
t.Fatalf("the failure is not said in the send's output:\n%s", out)
|
||||
}
|
||||
if !asserted(&failing.recordingRaiser, worker) {
|
||||
t.Fatal("the seat's worker was not asked for")
|
||||
}
|
||||
c, open1, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted")
|
||||
if err != nil || !open1 {
|
||||
t.Fatalf("no condition raised: %v", err)
|
||||
}
|
||||
if c.Kind != kindBusObjectsUnasserted || c.Source != sourceBusObjects ||
|
||||
len(c.Evidence) == 0 || !strings.Contains(c.Evidence[0].Said, "insufficient resources") {
|
||||
t.Fatalf("the condition does not say what failed: %+v", c)
|
||||
}
|
||||
|
||||
// The next send asserts them, and that observation clears it.
|
||||
if err := assertOnSend(t.Context(), open.inventory, &recordingRaiser{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, stillOpen, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); err != nil || stillOpen {
|
||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
// The controller starting, before either was assigned.
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, nil)
|
||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
||||
t.Fatal("a send that asserted everything raised a condition")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
|
||||
// RequestTimeout) — the shortest wait of a caller the mesh ships.
|
||||
const consoleWaits = 30 * time.Second
|
||||
|
||||
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
|
||||
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
|
||||
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
|
||||
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
||||
if link.AnswerWithin >= consoleWaits/2 {
|
||||
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
|
||||
}
|
||||
if link.AnswerWithin >= broker.ResponseTTL {
|
||||
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
|
||||
}
|
||||
}
|
||||
|
||||
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
||||
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
||||
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want bool
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
||||
}
|
||||
if got := answersFirst(argv); got != c.want {
|
||||
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
|
||||
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
|
||||
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
|
||||
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
|
||||
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
|
||||
// planner, so the two can be compared.
|
||||
|
||||
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
|
||||
type policyOf func(module string) (inventory.Upgrade, bool)
|
||||
|
||||
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
|
||||
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
|
||||
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
|
||||
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
machines := map[string]*link.MachinePlan{}
|
||||
machine := func(name string) *link.MachinePlan {
|
||||
if machines[name] == nil {
|
||||
machines[name] = &link.MachinePlan{Machine: name}
|
||||
}
|
||||
return machines[name]
|
||||
}
|
||||
for _, tier := range r.Plan.Tiers {
|
||||
for _, name := range tier {
|
||||
e, held := byName[name]
|
||||
if !held {
|
||||
continue
|
||||
}
|
||||
u, known := policy(name)
|
||||
waits := known && !u.RollOut
|
||||
for _, on := range e.On {
|
||||
if waits {
|
||||
machine(on).Waits = append(machine(on).Waits, name)
|
||||
} else {
|
||||
machine(on).Receives = append(machine(on).Receives, name)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case name == "nats":
|
||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
||||
case waits && len(e.On) > 0:
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
||||
orNone(u.From)))
|
||||
}
|
||||
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
|
||||
var offers []string
|
||||
for _, o := range e.Manifest.Provides {
|
||||
offers = append(offers, o.Name)
|
||||
}
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
|
||||
name, strings.Join(offers, ", ")))
|
||||
}
|
||||
if e.Manifest.Data != nil && len(e.On) > 0 {
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
|
||||
}
|
||||
}
|
||||
}
|
||||
var names []string
|
||||
for name := range machines {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
p.Machines = append(p.Machines, *machines[name])
|
||||
}
|
||||
p.Summary = summaryOf(p)
|
||||
return p
|
||||
}
|
||||
|
||||
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
|
||||
func summaryOf(p link.ChangePlan) string {
|
||||
if len(p.Moved) == 0 && len(p.New) == 0 {
|
||||
return "builds nothing: the change touches no module of the mesh's graph"
|
||||
}
|
||||
var parts []string
|
||||
what := strings.Join(p.Moved, ", ")
|
||||
if len(p.Dependents) > 0 {
|
||||
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
|
||||
}
|
||||
if len(p.New) > 0 {
|
||||
if what != "" {
|
||||
what += ", "
|
||||
}
|
||||
what += "new: " + strings.Join(p.New, ", ")
|
||||
}
|
||||
var to []string
|
||||
for _, m := range p.Machines {
|
||||
if len(m.Receives) > 0 {
|
||||
to = append(to, m.Machine)
|
||||
}
|
||||
}
|
||||
if len(to) > 0 {
|
||||
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
|
||||
} else {
|
||||
parts = append(parts, "builds "+what+", sent nowhere")
|
||||
}
|
||||
bus := "no bus step"
|
||||
for _, s := range p.Steps {
|
||||
if strings.HasPrefix(s, "a planned bus step") {
|
||||
bus = "a bus step"
|
||||
}
|
||||
}
|
||||
parts = append(parts, bus)
|
||||
waiting := 0
|
||||
for _, m := range p.Machines {
|
||||
waiting += len(m.Waits)
|
||||
}
|
||||
if waiting > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// planText is a change plan as a person reads it, for the pull request's comment.
|
||||
func planText(p link.ChangePlan) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "change plan of %s at %.8s into %s: %s\n", p.Repository, p.Head, p.Base, p.Summary)
|
||||
for i, tier := range p.Tiers {
|
||||
fmt.Fprintf(&b, " tier %d: %s\n", i, strings.Join(tier, ", "))
|
||||
}
|
||||
for _, m := range p.Machines {
|
||||
line := " " + m.Machine + ": "
|
||||
if len(m.Receives) > 0 {
|
||||
line += "receives " + strings.Join(m.Receives, ", ")
|
||||
}
|
||||
if len(m.Waits) > 0 {
|
||||
if len(m.Receives) > 0 {
|
||||
line += "; "
|
||||
}
|
||||
line += "waits for a person: " + strings.Join(m.Waits, ", ")
|
||||
}
|
||||
b.WriteString(line + "\n")
|
||||
}
|
||||
for _, s := range p.Steps {
|
||||
b.WriteString(" - " + s + "\n")
|
||||
}
|
||||
if len(p.Unread) > 0 {
|
||||
fmt.Fprintf(&b, " read by no module's build: %s\n", strings.Join(p.Unread, ", "))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
|
||||
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
|
||||
// person, a provider whose consumers follow it.
|
||||
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name, path string, on ...string) inventory.Entry {
|
||||
e := fromRepo(name, catalogue_, path)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = on
|
||||
return e
|
||||
}
|
||||
nats := entry("nats", "modules/nats", "anchor")
|
||||
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
gitea := entry("gitea", "modules/gitea", "anchor")
|
||||
held := entry("photos", "modules/photos", "anchor", "laptop")
|
||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
||||
tools.On = []string{"anchor", "laptop"}
|
||||
entries := []inventory.Entry{nats, gitea, held, tools}
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
||||
policy := func(module string) (inventory.Upgrade, bool) {
|
||||
if module == "photos" {
|
||||
return inventory.Upgrade{RollOut: false, From: "a person"}, true
|
||||
}
|
||||
return inventory.Upgrade{RollOut: true}, true
|
||||
}
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
||||
}
|
||||
|
||||
p := plan("modules/gitea/index.ts")
|
||||
if p.Summary != "builds gitea → anchor; no bus step" {
|
||||
t.Errorf("one module's change reads %q", p.Summary)
|
||||
}
|
||||
|
||||
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
|
||||
if !strings.Contains(p.Summary, "a bus step") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
||||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
|
||||
t.Errorf("the bus and a held module read %q", p.Summary)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, m := range p.Machines {
|
||||
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
|
||||
}
|
||||
// The bus first, what stands on it after: the build plan's order, per machine.
|
||||
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
|
||||
t.Errorf("the deploy plan reads %v", got)
|
||||
}
|
||||
text := strings.Join(p.Steps, "\n")
|
||||
for _, want := range []string{"a planned bus step", "photos waits for a person", "nats provides mesh-bus"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("the steps do not say %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
p = plan("merge-check.sh")
|
||||
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
|
||||
t.Errorf("a root file's plan reads %+v", p)
|
||||
}
|
||||
if !strings.Contains(planText(p), "read by no module's build: merge-check.sh") {
|
||||
t.Errorf("the plan's text does not say why nothing is built:\n%s", planText(p))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
|
||||
//
|
||||
// **The same functions registration runs, and nothing the command line adds** (ADR 0035): the strict
|
||||
// parse with every per-manifest problem, then the rules no single manifest can be judged against,
|
||||
// over exactly the manifests given. Somebody describing their own application in their own
|
||||
// repository runs this before pushing and finds out there, rather than when a running mesh refuses
|
||||
// the registration or, later, when a machine applies something that resolved and should not have.
|
||||
//
|
||||
// **What it cannot know without a store, it says.** The mesh's own seat set is the store's (ADR
|
||||
// 0122); this binary carries a compiled copy that the store overrides when loaded, so a claim on a
|
||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||
// refused what it could not see would teach people to ignore it.
|
||||
//
|
||||
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
||||
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||
// provider's machine when a real machine's name first meets the module's.
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||
}
|
||||
|
||||
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
if len(paths) == 0 {
|
||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||
"manifest of a repository together so the rules between them are checked too")
|
||||
}
|
||||
shelf := catalogue.Shelf{}
|
||||
faulted := map[string]bool{}
|
||||
failed := 0
|
||||
for _, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
if first, twice := shelf[m.Module]; twice {
|
||||
_ = first
|
||||
fmt.Fprintf(out, "%s: %s was already given; two manifests name one module\n", path, m.Module)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
|
||||
// catalogue-wide test, and at registration, which refuses in the same words.
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(named)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
|
||||
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
||||
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
||||
// has already been said and would be said again here in a worse form.
|
||||
problems := catalogue.CatalogueProblems(shelf)
|
||||
sort.Strings(problems)
|
||||
for _, p := range problems {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||
// only the provider's manifest states.
|
||||
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
||||
sort.Strings(identities)
|
||||
for _, p := range identities {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(identities)
|
||||
|
||||
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
|
||||
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
|
||||
data := catalogue.DataProblems(shelf)
|
||||
sort.Strings(data)
|
||||
for _, p := range data {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(data)
|
||||
|
||||
var names []string
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
// **Every long-running resource says how it is ready** (novox/hq ADR 0240 rule 8): warned until the
|
||||
// date, refused from it. The count is the catalogue's: its merge check keeps the number and lets a
|
||||
// change lower it, never raise it.
|
||||
undeclared := 0
|
||||
required := !checkNow().Before(catalogue.HealthRequiredFrom)
|
||||
for _, name := range names {
|
||||
missing := catalogue.Undeclared(shelf[name])
|
||||
undeclared += len(missing)
|
||||
if len(missing) == 0 {
|
||||
continue
|
||||
}
|
||||
if required {
|
||||
for _, id := range missing {
|
||||
fmt.Fprintf(out, "%s: %s stays up and does not say how it is ready: a long-running resource declares "+
|
||||
"health since %s (novox/hq ADR 0240 rule 8)\n", name, id, catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||
}
|
||||
failed += len(missing)
|
||||
faulted[name] = true
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
if faulted[name] {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(out, "%s: ok", name)
|
||||
if n := len(m.Tools); n > 0 {
|
||||
fmt.Fprintf(out, ", %d tool(s)", n)
|
||||
}
|
||||
if len(m.Invokes) > 0 {
|
||||
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
||||
}
|
||||
// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
|
||||
if len(m.State) > 0 {
|
||||
kept := make([]string, 0, len(m.State))
|
||||
for _, s := range m.State {
|
||||
kept = append(kept, s.Name)
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||
}
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
|
||||
if items := m.DataItems(); len(items) > 0 {
|
||||
kept := make([]string, 0, len(items))
|
||||
for _, it := range items {
|
||||
kept = append(kept, it.ID+" ("+it.Class+")")
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
||||
}
|
||||
// How what it runs is ready (ADR 0240): each declared check, and what is judged by liveness alone.
|
||||
var checks []string
|
||||
for _, r := range m.Resources {
|
||||
if h, has, _ := catalogue.ReadHealth(r); has {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
if len(checks) > 0 {
|
||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
||||
}
|
||||
if missing := catalogue.Undeclared(m); len(missing) > 0 {
|
||||
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
|
||||
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
|
||||
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
||||
}
|
||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
||||
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UndeclaredHealthLine starts the line `module check` says the count of long-running resources without
|
||||
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
|
||||
const UndeclaredHealthLine = "long-running resources without health:"
|
||||
|
||||
// checkNow is the clock `module check` judges the date by; a test sets it.
|
||||
var checkNow = time.Now
|
||||
|
||||
func joinInvokes(invokes []string) string {
|
||||
if len(invokes) == 1 && invokes[0] == "*" {
|
||||
return "every tool"
|
||||
}
|
||||
s := ""
|
||||
for i, t := range invokes {
|
||||
if i > 0 {
|
||||
s += ", "
|
||||
}
|
||||
s += t
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// manifestsUnder lists every module.json below a directory, for `module check <dir>`.
|
||||
func manifestsUnder(dir string) ([]string, error) {
|
||||
var found []string
|
||||
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git" || d.Name() == "dist") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == "module.json" {
|
||||
found = append(found, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(found)
|
||||
return found, err
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
|
||||
// runs it** (novox/hq issue 283): the same code (builder.Check), the same ask the controller would make of
|
||||
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
|
||||
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
|
||||
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
|
||||
// the mesh runs.
|
||||
//
|
||||
// **The build seat is the reference.** A merge-check.sh that passed on an agent's machine failed on the
|
||||
// seat for three reasons that were each the agent's environment, never the change: a newer Go whose gofmt
|
||||
// lays a file out differently, a sibling checkout at the agent's feature branch where the seat had the
|
||||
// commit the mesh runs, and a different user. Run here, a check sees what the seat will.
|
||||
//
|
||||
// check-here [--tree <checkout>] [--base main] [--registry <artifact store>] [--forge <url of the owner>]
|
||||
// [--number <n>] [--user uid:gid] [--facts store|<file>] [--keep]
|
||||
//
|
||||
// The checkout's HEAD is what is checked, and it must be committed: the seat checks a commit, never a
|
||||
// working tree.
|
||||
func checkHereCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("check-here", flag.ContinueOnError)
|
||||
tree := set.String("tree", ".", "the change's checkout; its HEAD is checked")
|
||||
base := set.String("base", "main", "the branch the change would merge into")
|
||||
registry := set.String("registry", os.Getenv("MESH_REGISTRY"), "the artifact store holding the facts and the toolchains")
|
||||
forge := set.String("forge", "", "where the repositories beside it are cloned from, as <forge>/<repository>.git; "+
|
||||
"the checkout's origin without its own name when not given")
|
||||
number := set.Int("number", 0, "the pull request's number, when there is one")
|
||||
// The build seat's service runs as root, and its check containers run as the builder does.
|
||||
user := set.String("user", "0:0", "the user the check's containers run as: the build seat's")
|
||||
keep := set.Bool("keep", false, "keep the workspace afterwards")
|
||||
factsFrom := set.String("facts", "store", "the facts snapshot: `store`, the one the artifact store holds — "+
|
||||
"what the seat reads — or a file")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *registry == "" {
|
||||
return errors.New("check-here reads the facts and the toolchains from the artifact store: --registry <host:port> or MESH_REGISTRY")
|
||||
}
|
||||
dir, err := filepath.Abs(*tree)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
git := func(args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.Output()
|
||||
return strings.TrimSpace(string(out)), err
|
||||
}
|
||||
if dirty, err := git("status", "--porcelain", "--untracked-files=no"); err != nil {
|
||||
return fmt.Errorf("%s is not a checkout: %w", dir, err)
|
||||
} else if dirty != "" {
|
||||
return errors.New("the checkout has changes not committed: the build seat checks a commit, so commit first")
|
||||
}
|
||||
head, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
origin, err := git("remote", "get-url", "origin")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the checkout has no origin to say which repository it is: %w", err)
|
||||
}
|
||||
owner, repo, prefix := ownerRepoOf(origin)
|
||||
if *forge == "" {
|
||||
*forge = prefix
|
||||
}
|
||||
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
|
||||
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
|
||||
}
|
||||
changedText, err := git("diff", "--name-only", "origin/"+*base+"...HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var paths, removed []string
|
||||
for _, p := range strings.Split(changedText, "\n") {
|
||||
if p = strings.TrimSpace(p); p == "" {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, p)
|
||||
if _, err := os.Stat(filepath.Join(dir, p)); os.IsNotExist(err) {
|
||||
removed = append(removed, p)
|
||||
}
|
||||
}
|
||||
|
||||
_ = os.Setenv("MESH_REGISTRY", *registry)
|
||||
f, err := readFacts(ctx, *factsFrom)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the facts snapshot cannot be read: %w", err)
|
||||
}
|
||||
entries, read, edges, err := graphOfFacts(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p := link.PullUpdated{Owner: owner, Repo: repo, Number: *number, Base: *base, Commit: head, Paths: paths,
|
||||
Removed: removed, ModuleDirs: moduleDirsIn(dir, paths), ModuleDirsSaid: true}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
_, scriptErr := os.Stat(filepath.Join(dir, builder.CheckScript))
|
||||
if !scope.gated() && !scope.Mesh {
|
||||
fmt.Printf("%s: %s, and the repository is not the mesh's: the build seat runs nothing for it\n",
|
||||
owner+"/"+repo, noModuleTouched)
|
||||
return nil
|
||||
}
|
||||
if !scope.gated() && scriptErr != nil {
|
||||
fmt.Printf("%s: %s; %s\n", owner+"/"+repo, noModuleTouched, noMergeCheck)
|
||||
return nil
|
||||
}
|
||||
|
||||
toolchains := map[string]string{}
|
||||
for language, reference := range f.Versions.Toolchains {
|
||||
toolchains[language] = catalogue.Rerouted(reference, *registry)
|
||||
}
|
||||
if len(toolchains) == 0 {
|
||||
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
|
||||
"issue 283, and the seat's toolchain cannot be known here")
|
||||
}
|
||||
beside := map[string]builder.Beside{}
|
||||
for d, ref := range f.Beside {
|
||||
from := d
|
||||
if d == "mesh-controller-main" {
|
||||
from = "mesh-controller"
|
||||
}
|
||||
beside[d] = builder.Beside{Repository: strings.TrimSuffix(*forge, "/") + "/" + from + ".git", Ref: ref}
|
||||
}
|
||||
id := fmt.Sprintf("check-here-%d", time.Now().UnixNano())
|
||||
spec := builder.CheckSpec{ID: id, Repository: dir, Ref: head, Owner: owner, Repo: repo, Number: *number,
|
||||
Paths: paths, Beside: beside, Modules: scope.Modules, New: scope.New, Manifests: scope.Manifests,
|
||||
Base: *base, Judge: scope.Judge, Toolchain: toolchains["go"], Toolchains: toolchains, User: *user}
|
||||
|
||||
workspace, err := os.MkdirTemp("", "mesh-check-here-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !*keep {
|
||||
defer removeWorkspace(spec.Toolchain, workspace, *user)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "checking %s/%s at %.8s as the build seat would, against the facts of %s, in %s\n",
|
||||
owner, repo, head, f.Taken.Format(time.RFC3339), workspace)
|
||||
v, err := builder.Check(ctx, builder.Command, spec, workspace, *registry, builder.GitCredential{},
|
||||
func(step, message string) {
|
||||
if step != "output" {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the check could not run — on the seat an error, never a pass: %w", err)
|
||||
}
|
||||
fmt.Println(v.Report)
|
||||
fmt.Println()
|
||||
fmt.Printf("mesh/merge-gate: %s — %s\n", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary)
|
||||
if v.Repo != nil {
|
||||
fmt.Printf("mesh/repo-check: %s — %s\n", strings.ToUpper(v.Repo.Verdict), v.Repo.Summary)
|
||||
}
|
||||
for _, l := range []*builder.Layer{v.Gate, v.Repo} {
|
||||
if l != nil && l.Verdict != "pass" && l.Verdict != "warning" {
|
||||
return errors.New("the build seat would not pass this change")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownerRepoOf reads owner, repository and the owner's URL from a remote: ssh://git@host:222/novox/mesh-host.git
|
||||
// → novox, mesh-host, ssh://git@host:222/novox.
|
||||
func ownerRepoOf(remote string) (string, string, string) {
|
||||
trimmed := strings.TrimSuffix(strings.TrimSuffix(remote, "/"), ".git")
|
||||
cut := strings.LastIndexAny(trimmed, "/:")
|
||||
if cut < 0 {
|
||||
return "", trimmed, ""
|
||||
}
|
||||
repo, prefix := trimmed[cut+1:], trimmed[:cut]
|
||||
owner := prefix
|
||||
if at := strings.LastIndexAny(prefix, "/:"); at >= 0 {
|
||||
owner = prefix[at+1:]
|
||||
}
|
||||
return owner, repo, prefix
|
||||
}
|
||||
|
||||
// removeWorkspace removes what the check left, written as the seat's user: by a container of that user
|
||||
// when it is not this one.
|
||||
func removeWorkspace(image, workspace, user string) {
|
||||
if image != "" && user != fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) {
|
||||
// Everything in it — the check's HOME is the workspace, so the toolchain's own files are there too.
|
||||
_ = exec.Command("docker", "run", "--rm", "--user", user, "--volume", workspace+":/workspace", image,
|
||||
"sh", "-c", "rm -rf /workspace/* /workspace/.[!.]*").Run()
|
||||
}
|
||||
_ = os.RemoveAll(workspace)
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||
// with a known fault is named, and one without passes, with no store opened.
|
||||
func TestModuleCheckNamesAFaultAndNeedsNoMesh(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
good := filepath.Join(dir, "good.json")
|
||||
bad := filepath.Join(dir, "bad.json")
|
||||
os.WriteFile(good, []byte(`{"module":"shop","version":"1","tools":["price"],"invokes":["mesh-catalog.catalog_modules"]}`), 0o600)
|
||||
os.WriteFile(bad, []byte(`{"module":"till","version":"1","invokes":["shop"]}`), 0o600)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{good}, &out); err != nil {
|
||||
t.Fatalf("a sound manifest was refused: %v\n%s", err, out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "shop: ok, 1 tool(s), invokes mesh-catalog.catalog_modules") {
|
||||
t.Fatalf("the report does not say what it checked:\n%s", out.String())
|
||||
}
|
||||
|
||||
out.Reset()
|
||||
err := moduleCheck([]string{good, bad}, &out)
|
||||
if err == nil {
|
||||
t.Fatal("a manifest invoking a module and no tool passed")
|
||||
}
|
||||
if !strings.Contains(out.String(), `till invokes "shop", which does not name a tool`) {
|
||||
t.Fatalf("the fault is not named in the manifest's words:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The rules between manifests run over what was given together: a seat two modules declare is
|
||||
// refused, which no single-manifest check can see.
|
||||
func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a := filepath.Join(dir, "a.json")
|
||||
b := filepath.Join(dir, "b.json")
|
||||
os.WriteFile(a, []byte(`{"module":"a","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
||||
os.WriteFile(b, []byte(`{"module":"b","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{a, b}, &out); err == nil {
|
||||
t.Fatalf("two declarations of one seat passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "a seat name means one protocol") {
|
||||
t.Fatalf("the cross-manifest rule was not the one named:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
}
|
||||
paths, err := manifestsUnder(root)
|
||||
if err != nil || len(paths) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck(paths, &out); err != nil {
|
||||
t.Fatalf("the catalogue does not pass its own check: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
|
||||
// novox/hq ADR 0155: the check moves to registration once the catalogue passes it. Both
|
||||
// ways in — `module add` and a build's result — go through this, and a name declared on
|
||||
// purpose passes with its reason.
|
||||
named := catalogue.Manifest{Module: "idp", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}},
|
||||
}}
|
||||
err := namesNoInstallation(named)
|
||||
if err == nil || !strings.Contains(err.Error(), "login.mesh-one.be") ||
|
||||
!strings.Contains(err.Error(), catalogue.NamesOnPurpose) {
|
||||
t.Fatalf("a definition naming an installation is refused with the name and the way out; got %v", err)
|
||||
}
|
||||
meant := catalogue.Manifest{Module: "site", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "image": "registry.mesh-one.be/org/site@sha256:cc",
|
||||
catalogue.NamesOnPurpose: map[string]any{
|
||||
"registry.mesh-one.be": "built outside the mesh until its repository is a build source here"}},
|
||||
}}
|
||||
if err := namesNoInstallation(meant); err != nil {
|
||||
t.Fatalf("a name declared on purpose passes; got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged
|
||||
// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it —
|
||||
// refused it. Held here, so a verb added to the table without the manifest fails this repository's
|
||||
// own suite rather than its next pull request's gate.
|
||||
func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
|
||||
var out strings.Builder
|
||||
if err := moduleCheck([]string{"../../module.json"}, &out); err != nil {
|
||||
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A pull request's merge check (novox/hq to-be 45 §9, ADR 0237 as amended 2026-10-06): the forge
|
||||
// announces every pull request's new head, the controller decides what is checked, asks the build seat to
|
||||
// check it, and says the verdict as `checked`, which the forge's holder sets as the pull request's
|
||||
// statuses. **Before merge, never after**: every check the mesh had ran after a merge, on a machine.
|
||||
//
|
||||
// **The mesh's module graph decides, not the repository.** The controller holds the graph — every module,
|
||||
// the repository and directory it is built from — and maps the pull request's changed paths onto it by
|
||||
// the planner's own answer (reachOfMerge, touchedBy — the one place a changed file is mapped onto modules,
|
||||
// for the merge handler, the release planner, the merge gate and this check; issue 280): **a changed file
|
||||
// touches exactly the modules whose build reads it** — a module's own directory (the whole repository for
|
||||
// one built from its root), or a repository its recipe packages. A file no build reads — a script at the
|
||||
// root, a README — touches no module. A directory the change adds a module.json in, which the graph does
|
||||
// not hold yet (said by the head, issue 278), is a new module and is checked too.
|
||||
//
|
||||
// - touches a module: the build seat runs **the gate** — `mesh/merge-gate`, the touched manifests, every
|
||||
// machine composed with the change, the replays — and the repository's own merge-check.sh beside it;
|
||||
// - touches none, in a repository that is the mesh's (it sources a module on some branch, or shares the
|
||||
// core's owner): the gate is a pass that says so — a fact, not a missing check — and the repository's
|
||||
// own merge-check.sh runs as `mesh/repo-check`, a warning when it has none;
|
||||
// - touches none, anywhere else: the gate is a pass that says so, and nothing more is said.
|
||||
//
|
||||
// What is checked is decided here and run there (internal/builder/check.go).
|
||||
|
||||
// checkTimeout is how long one check may run on the build seat. Said here so the ask's watchdog (S6)
|
||||
// and the builder agree on what late means.
|
||||
const checkTimeout = 45 * time.Minute
|
||||
|
||||
// noModuleTouched is the gate's word for a change that touches nothing of the graph.
|
||||
const noModuleTouched = "the change touches no module of the mesh's graph"
|
||||
|
||||
// noMergeCheck is the repository layer's word for a repository of the mesh with no merge-check.sh.
|
||||
const noMergeCheck = "the repository declares no merge-check.sh: none of its own tests run before it merges"
|
||||
|
||||
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
|
||||
// each beside it — and whose owner is the mesh's own.
|
||||
var coreModules = map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host",
|
||||
"node-tools": "mesh-tools", "nats": "mesh-catalog"}
|
||||
|
||||
// checkScope is what a pull request reaches of the mesh's graph, as the planner reckons it.
|
||||
type checkScope struct {
|
||||
// Modules are the modules a merge of the change would move itself — built from the repository into
|
||||
// the pull request's base and reading a changed file, or packaging the repository's source — and
|
||||
// Dependents those the plan would build after them; New the directories it adds a module in.
|
||||
Modules []string
|
||||
Dependents []string
|
||||
New []string
|
||||
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
|
||||
Manifests []string
|
||||
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
|
||||
// module's build reads.
|
||||
Width, Tiers int
|
||||
Unread []string
|
||||
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
|
||||
// core's, or the change adds a module to it.
|
||||
Mesh bool
|
||||
// Judge is who judges the gate (link.JudgeSelf, link.JudgeValidator, or the running controller).
|
||||
Judge string
|
||||
// From is a module built from the repository, for how the mesh clones it; nil when none is.
|
||||
From *inventory.Entry
|
||||
// Reach is the planner's whole answer, which the change plan is made from.
|
||||
Reach mergeReach
|
||||
}
|
||||
|
||||
func (s checkScope) gated() bool { return len(s.Modules)+len(s.New) > 0 }
|
||||
|
||||
// pullScope is what a pull request reaches: **the planner's own answer** (reachOfMerge), asked as if the
|
||||
// head were merged into the base — never a mapping of its own, so a change to what a merge touches
|
||||
// changes what is checked with it (novox/hq ADR 0238).
|
||||
func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) checkScope {
|
||||
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL, Commit: p.Commit,
|
||||
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
|
||||
ModuleDirsSaid: p.ModuleDirsSaid}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
|
||||
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
|
||||
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
|
||||
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
|
||||
switch e.Manifest.Module {
|
||||
case "mesh-controller":
|
||||
s.Judge = link.JudgeSelf
|
||||
case "mesh-host":
|
||||
if s.Judge == "" {
|
||||
s.Judge = link.JudgeValidator
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, d := range r.Added {
|
||||
s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile))
|
||||
}
|
||||
sort.Strings(s.Manifests)
|
||||
s.Manifests = slices.Compact(s.Manifests)
|
||||
|
||||
// Whose repository it is, for how it is cloned and whether its own check is the mesh's to run.
|
||||
owners := map[string]bool{}
|
||||
for i, e := range entries {
|
||||
if e.Provided {
|
||||
continue
|
||||
}
|
||||
if _, core := coreModules[e.Manifest.Module]; core {
|
||||
if owner := sourceOwner(e.Source.Repository); owner != "" {
|
||||
owners[owner] = true
|
||||
}
|
||||
}
|
||||
if sameRepository(e.Source.Repository, m) && s.From == nil {
|
||||
s.From = &entries[i]
|
||||
}
|
||||
}
|
||||
s.Mesh = s.From != nil || owners[strings.ToLower(p.Owner)] || s.gated()
|
||||
return s
|
||||
}
|
||||
|
||||
// sourceOwner is the owner of a recorded repository, a path on the git seat or a URL: novox/mesh-host → novox.
|
||||
func sourceOwner(repository string) string {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(parts[len(parts)-2])
|
||||
}
|
||||
|
||||
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
|
||||
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted
|
||||
// with the verdict whatever the verdict is.
|
||||
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Print(planText(plan))
|
||||
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
|
||||
sayChecked(ctx, direct)
|
||||
return nil
|
||||
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
|
||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
|
||||
sayChecked(ctx, direct)
|
||||
return nil
|
||||
}
|
||||
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
||||
scope.Width, scope.Tiers)
|
||||
}
|
||||
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
|
||||
p.Commit, seat, what, request.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
|
||||
// and what is read beside it.
|
||||
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
|
||||
entries []inventory.Entry) (link.BuildRequest, error) {
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
clone := func(s inventory.Source) (string, error) {
|
||||
if s.Seat == "" {
|
||||
return s.Repository, nil
|
||||
}
|
||||
return clonedFromSeat(world, s.Seat, s.Repository)
|
||||
}
|
||||
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
|
||||
source := inventory.Source{Seat: gitSeat, Repository: p.Owner + "/" + p.Repo}
|
||||
if scope.From != nil {
|
||||
source = scope.From.Source
|
||||
}
|
||||
repository, err := clone(source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
if dir == "mesh-controller" {
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
"mesh-lab")}); err == nil {
|
||||
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Ref: p.Commit,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||
// builds from; and beside the controller its main, for a judge the running controller predates, and the
|
||||
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
|
||||
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
|
||||
func besideRefs(dir, running string) map[string]string {
|
||||
switch dir {
|
||||
case "mesh-catalog":
|
||||
return map[string]string{dir: "main"}
|
||||
case "mesh-controller":
|
||||
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
|
||||
}
|
||||
return map[string]string{dir: running}
|
||||
}
|
||||
|
||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
||||
func siblingOf(repository, name string) string {
|
||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
||||
return repository[:cut+1] + name
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// sourceOnSeat is a source's seat form, nil for one on no seat.
|
||||
func sourceOnSeat(s inventory.Source) *link.SourceOnSeat {
|
||||
if s.Seat == "" {
|
||||
return nil
|
||||
}
|
||||
return &link.SourceOnSeat{Seat: s.Seat, Repository: s.Repository}
|
||||
}
|
||||
|
||||
// checkEvents is where the serving controller says a check's verdict; nil in a command.
|
||||
var checkEvents link.Bus
|
||||
|
||||
// maxCheckReport is how much of a check's report travels in its verdict: enough for the failures and
|
||||
// the machines, never a log.
|
||||
const maxCheckReport = 60 << 10
|
||||
|
||||
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
||||
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
||||
func checked(ctx context.Context, result link.BuildResult) {
|
||||
sayChecked(ctx, checkedOf(result))
|
||||
}
|
||||
|
||||
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
|
||||
// could not run.
|
||||
func checkedOf(result link.BuildResult) link.Checked {
|
||||
c := link.Checked{ID: result.ID, On: result.On, Commit: result.Ref}
|
||||
if result.Checked != nil {
|
||||
c.Owner, c.Repo, c.Number = result.Checked.Owner, result.Checked.Repo, result.Checked.Number
|
||||
}
|
||||
switch {
|
||||
case result.Check != nil:
|
||||
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
|
||||
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
|
||||
if c.Gate == nil {
|
||||
// A build seat from before the layers: its verdict is the gate's.
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
}
|
||||
case result.Failed != "":
|
||||
// The check could not run: an error, never read as a pass — on both layers it was asked for.
|
||||
c.Verdict, c.Summary = "error", "the check could not run: "+firstLine(result.Failed)
|
||||
default:
|
||||
c.Verdict, c.Summary = "error", "the build seat answered the check with no verdict"
|
||||
}
|
||||
if c.Verdict == "" {
|
||||
c.Verdict = "error"
|
||||
}
|
||||
if result.Check == nil {
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
}
|
||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
|
||||
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
|
||||
}
|
||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Dependents) == 0 {
|
||||
c.Gate.Dependents = result.Checked.Dependents
|
||||
}
|
||||
if result.Checked != nil {
|
||||
c.Plan = result.Checked.Plan
|
||||
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
|
||||
if g := result.Checked; g.Group != "" {
|
||||
c.Group = g.Group
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
|
||||
Commit: result.Ref})
|
||||
for _, m := range g.Members {
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
|
||||
Commit: m.Ref})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, l := range []*link.CheckLayer{c.Gate, c.RepoCheck} {
|
||||
if l != nil && l.Verdict == "" {
|
||||
l.Verdict = "error"
|
||||
}
|
||||
}
|
||||
if len(c.Report) > maxCheckReport {
|
||||
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func prefixedAll(prefix string, items []string) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, i := range items {
|
||||
out = append(out, prefix+i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
|
||||
func sayChecked(ctx context.Context, c link.Checked) {
|
||||
repo := "none"
|
||||
if c.RepoCheck != nil {
|
||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||
}
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
if checkEvents == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
|
||||
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **The mesh's module graph decides what a pull request's check runs**, not the repository (novox/hq
|
||||
// ADR 0237 as amended): a change is mapped onto the graph by the rule a merge is (issue 278), the gate
|
||||
// runs when it touches a module — a new one included — and a repository that is the mesh's and touches
|
||||
// none still has its own merge-check.sh run.
|
||||
func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) {
|
||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
const controller = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
const host = "http://forge.internal:20000/novox/mesh-host.git"
|
||||
photos := fromRepo("photos", "http://forge.internal:20000/novox/photos.git", "")
|
||||
photos.Source.Ref = "nox-mesh"
|
||||
snake := fromRepo("snake", "jschoubben/snake", "")
|
||||
snake.Source.Seat = "git"
|
||||
entries := []inventory.Entry{
|
||||
fromRepo("gitea", catalogue, "modules/gitea"),
|
||||
fromRepo("keycloak", catalogue, "modules/keycloak"),
|
||||
fromRepo("nats", catalogue, "modules/nats"),
|
||||
fromRepo("mesh-controller", controller, ""),
|
||||
fromRepo("mesh-host", host, ""),
|
||||
photos, snake,
|
||||
}
|
||||
pull := func(owner, repo, base string, paths []string, dirs ...string) link.PullUpdated {
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Base: base, Commit: "abc", Paths: paths,
|
||||
ModuleDirs: dirs, ModuleDirsSaid: true}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
p link.PullUpdated
|
||||
modules, new, manifest string
|
||||
mesh bool
|
||||
judge string
|
||||
}{
|
||||
{"one module's own files", pull("novox", "mesh-catalog", "main", []string{"modules/gitea/index.ts"}, "modules/gitea"),
|
||||
"gitea", "", "modules/gitea/module.json", true, ""},
|
||||
{"a file no module's build reads touches no module (issue 280)",
|
||||
pull("novox", "mesh-catalog", "main", []string{"merge-check.sh", "README.md"}), "", "", "", true, ""},
|
||||
{"files the announcer could not list: everything built from it",
|
||||
link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", PathsTruncated: true,
|
||||
Paths: []string{"README.md"}}, "gitea,keycloak,nats", "",
|
||||
"modules/gitea/module.json,modules/keycloak/module.json,modules/nats/module.json", true, ""},
|
||||
{"a new module, said by the head", pull("novox", "mesh-catalog", "main",
|
||||
[]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, "modules/newmod"),
|
||||
"", "modules/newmod", "modules/newmod/module.json", true, ""},
|
||||
{"the controller judges itself", pull("novox", "mesh-controller", "main", []string{"cmd/x.go"}),
|
||||
"mesh-controller", "", "module.json", true, "self"},
|
||||
{"the node-engine is judged with its validator", pull("novox", "mesh-host", "main", []string{"validate/v.go"}),
|
||||
"mesh-host", "", "module.json", true, "validator"},
|
||||
{"the core's owner, no module: the mesh's, its own check alone", pull("novox", "hq", "main", []string{"README.md"}),
|
||||
"", "", "", true, ""},
|
||||
{"a branch nothing is built from: the mesh's repository, no module", pull("novox", "photos", "master",
|
||||
[]string{"server/x.js"}), "", "", "", true, ""},
|
||||
{"the branch a module is built from", pull("novox", "photos", "nox-mesh", []string{"server/x.js"}),
|
||||
"photos", "", "module.json", true, ""},
|
||||
{"a repository on the forge's seat", pull("jschoubben", "snake", "main", []string{"index.html"}),
|
||||
"snake", "", "module.json", true, ""},
|
||||
{"a repository of nobody's, touching nothing", pull("someone", "dotfiles", "main", []string{"x"}),
|
||||
"", "", "", false, ""},
|
||||
{"a repository adding a module at its root", pull("someone", "newapp", "main", []string{"module.json", "x.js"}),
|
||||
"", ".", "module.json", true, ""},
|
||||
} {
|
||||
s := pullScope(c.p, entries, nil, nil)
|
||||
got := []string{strings.Join(s.Modules, ","), strings.Join(s.New, ","), strings.Join(s.Manifests, ",")}
|
||||
want := []string{c.modules, c.new, c.manifest}
|
||||
for i, what := range []string{"modules", "new", "manifests"} {
|
||||
if got[i] != want[i] {
|
||||
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
|
||||
}
|
||||
}
|
||||
if s.Mesh != c.mesh || s.Judge != c.judge {
|
||||
t.Errorf("%s: the mesh's %v judged by %q, wanted %v by %q", c.what, s.Mesh, s.Judge, c.mesh, c.judge)
|
||||
}
|
||||
if s.gated() != (c.modules != "" || c.new != "") {
|
||||
t.Errorf("%s: gated %v", c.what, s.gated())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose build packages another repository's source is touched by a change to it.
|
||||
func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
|
||||
entries := []inventory.Entry{fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")}
|
||||
read := map[string][]inventory.ReadRepository{"node-tools": {{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}}
|
||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "abc", Paths: []string{"go/x.go"}}, entries, read, nil)
|
||||
if strings.Join(s.Modules, ",") != "node-tools" || len(s.Manifests) != 0 {
|
||||
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
// Each layer is said; a check that could not run is an error on both, never a pass; a build seat from
|
||||
// before the layers is read as the gate.
|
||||
func TestAChecksLayersAreEachSaidAndAnErrorIsNeverAPass(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3, Modules: []string{"gitea"}}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
||||
if c.Verdict != "error" || c.Gate == nil || c.Gate.Verdict != "error" || c.RepoCheck == nil || c.RepoCheck.Verdict != "error" {
|
||||
t.Fatalf("a check that could not run said %+v", c)
|
||||
}
|
||||
if strings.Join(c.Gate.Modules, ",") != "gitea" {
|
||||
t.Errorf("the gate names %v", c.Gate.Modules)
|
||||
}
|
||||
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "warning", Summary: "wide"}})
|
||||
if c.Gate == nil || c.Gate.Verdict != "warning" || c.RepoCheck != nil {
|
||||
t.Fatalf("an outcome without layers said %+v", c)
|
||||
}
|
||||
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "pass",
|
||||
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "its merge-check.sh failed"}}})
|
||||
if c.RepoCheck.Verdict != "fail" || c.Gate.Verdict != "pass" {
|
||||
t.Fatalf("the layers said %+v / %+v", c.Gate, c.RepoCheck)
|
||||
}
|
||||
}
|
||||
|
||||
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
|
||||
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
|
||||
// module in both, and what stands on a touched module is named as its dependents in both.
|
||||
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
|
||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
nats := fromRepo("nats", catalogue, "modules/nats")
|
||||
nats.Source.BuiltFrom = "old"
|
||||
gitea := fromRepo("gitea", catalogue, "modules/gitea")
|
||||
gitea.Source.BuiltFrom = "old"
|
||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
||||
entries := []inventory.Entry{nats, gitea, tools}
|
||||
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
||||
read := map[string][]inventory.ReadRepository{}
|
||||
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
paths []string
|
||||
moved, dependents string
|
||||
width int
|
||||
unread string
|
||||
}{
|
||||
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
|
||||
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
|
||||
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
|
||||
} {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
||||
s := pullScope(p, entries, read, edges)
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
|
||||
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
|
||||
}
|
||||
if got := strings.Join(s.Dependents, ","); got != c.dependents {
|
||||
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
|
||||
}
|
||||
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
|
||||
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
|
||||
}
|
||||
if got := strings.Join(s.Unread, ","); got != c.unread {
|
||||
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
|
||||
}
|
||||
}
|
||||
|
||||
// A repository whose build another module's recipe packages: that module is reached through the
|
||||
// planner's `read`, and what stands on it after it.
|
||||
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
|
||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
|
||||
Paths: []string{"src/index.ts"}}, entries, read, edges)
|
||||
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
|
||||
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a build of a commit off its repository's
|
||||
// default branch — a pull request's head, a branch built by hand, a rebuild or replay of one — is recorded
|
||||
// and never registered, so nothing can send it; a check or a dry run never is either.
|
||||
func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
|
||||
on := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main", OnTrunk: true}
|
||||
if err := publishable(on, ""); err != nil {
|
||||
t.Errorf("a build on the trunk was refused: %v", err)
|
||||
}
|
||||
off := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main"}
|
||||
if err := publishable(off, ""); !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "main") {
|
||||
t.Errorf("a build off the trunk was let through: %v", err)
|
||||
}
|
||||
for _, r := range []link.BuildResult{
|
||||
{ID: "c", Trunk: "main", OnTrunk: true, Check: &link.CheckOutcome{Verdict: "pass"}},
|
||||
{ID: "d", Trunk: "main", OnTrunk: true, Checked: &link.CheckRequest{}},
|
||||
{ID: "e", Trunk: "main", OnTrunk: true, DryRun: true},
|
||||
} {
|
||||
if publishable(r, "") == nil {
|
||||
t.Errorf("%s, a check or a dry run, was publishable", r.ID)
|
||||
}
|
||||
}
|
||||
// A module that follows a branch other than the default: that branch is its trunk, and the default
|
||||
// is not.
|
||||
follows := link.BuildResult{ID: "g", Commit: "abc", Trunk: "master", Branches: []string{"nox-mesh"}}
|
||||
if err := publishable(follows, "nox-mesh"); err != nil {
|
||||
t.Errorf("a commit on the branch a module follows was refused: %v", err)
|
||||
}
|
||||
if err := publishable(link.BuildResult{ID: "h", Commit: "abc", Trunk: "master", OnTrunk: true,
|
||||
Branches: []string{"master"}}, "nox-mesh"); err == nil {
|
||||
t.Error("a commit on the default but not on the branch the module follows was published")
|
||||
}
|
||||
if err := publishable(link.BuildResult{ID: "i", Commit: "abc", Trunk: "main", Branches: []string{"feat/x"}}, ""); err == nil {
|
||||
t.Error("a feature branch's commit was published")
|
||||
}
|
||||
// A build seat older than the rule says nothing: let through and said, so the rule can reach the mesh.
|
||||
if err := publishable(link.BuildResult{ID: "f", Commit: "abc"}, ""); err != nil {
|
||||
t.Errorf("a build seat that said nothing was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Letting the artifact store go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||
//
|
||||
// **Run where the records change.** A build is the moment new bytes landed in the store and the
|
||||
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
||||
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
||||
//
|
||||
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
||||
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
||||
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
||||
// having failed.
|
||||
|
||||
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
||||
// what it let go of. Says what it did and what it could not; returns nothing, because nothing
|
||||
// upstream should branch on it.
|
||||
func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||
references, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
|
||||
return
|
||||
}
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
|
||||
return
|
||||
}
|
||||
if len(references) == 0 && len(kept) == 0 {
|
||||
return
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue to find the artifact store: %v\n", err)
|
||||
return
|
||||
}
|
||||
// As the mesh reaches it from the network. Empty means the store is not on the network — on a
|
||||
// mesh being raised it is not yet, and there the store holds one build of anything and has
|
||||
// nothing to collect.
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil || address == "" {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not find the artifact store to collect from: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
|
||||
// never collected has the whole history to get through, and a person waiting on `build` should
|
||||
// not pay for it. Two bounds, and what is left over is simply offered again next time —
|
||||
// builds are frequent, and the point is that the store stops growing, not that it empties
|
||||
// tonight.
|
||||
within, stop := context.WithTimeout(ctx, sweepBudget)
|
||||
defer stop()
|
||||
store := artifacts.Store{Address: address}
|
||||
|
||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||
wrote, missing, err := holdKept(within, store, kept)
|
||||
if wrote > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
|
||||
}
|
||||
if missing > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||
"`collection` lists them\n", missing)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
var done []string
|
||||
var left, skipped int
|
||||
for i, reference := range references {
|
||||
if i >= mostPerSweep || within.Err() != nil {
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
err := store.LetGo(within, reference)
|
||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||
// again for ever.
|
||||
done = append(done, reference)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||
// reason to stop, because the store was never asked. One of these at the front of
|
||||
// the oldest-first order ended every sweep until this.
|
||||
skipped++
|
||||
if skipped == 1 {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"the sweep will not address %s and went on: %v\n", reference, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||
// front of whoever was building something.
|
||||
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
|
||||
reference, err)
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
|
||||
if len(done) > 0 {
|
||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||
// the sweep ran out of budget would mean asking about them again for ever.
|
||||
if err := inv.MarkCollected(ctx, done); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
|
||||
len(done), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
|
||||
len(done))
|
||||
}
|
||||
if left > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
|
||||
}
|
||||
}
|
||||
|
||||
// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
|
||||
// Answers how many holders it wrote and how many kept archives the store does not have.
|
||||
//
|
||||
// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
|
||||
// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
|
||||
// reference the store cannot be asked about is skipped as the deletion loop skips one
|
||||
// (novox/hq issue 226).
|
||||
func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
|
||||
for _, reference := range kept {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
|
||||
}
|
||||
did, err := store.Hold(ctx, reference)
|
||||
switch {
|
||||
case err == nil:
|
||||
if did {
|
||||
wrote++
|
||||
}
|
||||
case errors.Is(err, artifacts.Gone):
|
||||
missing++
|
||||
case errors.Is(err, artifacts.ErrNotOurs):
|
||||
default:
|
||||
return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
|
||||
}
|
||||
}
|
||||
return wrote, missing, nil
|
||||
}
|
||||
|
||||
// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
|
||||
// several times a day converges within days of this landing; small enough that no single build
|
||||
// waits on the whole backlog.
|
||||
const mostPerSweep = 200
|
||||
|
||||
// sweepBudget is the longest a sweep will keep a build waiting.
|
||||
const sweepBudget = 60 * time.Second
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
)
|
||||
|
||||
// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
|
||||
// (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The question to answer before the store's collector runs for real.** The collector deletes
|
||||
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
|
||||
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
|
||||
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
|
||||
// dry run go.
|
||||
//
|
||||
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
|
||||
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
|
||||
// new verb in the seat's row.
|
||||
|
||||
type collectionReport struct {
|
||||
// Store is the artifact store as this machine reached it; empty when it is not on the network.
|
||||
Store string `json:"store"`
|
||||
// KeptArchives is how many archives the mesh keeps, for either reason.
|
||||
KeptArchives int `json:"kept_archives"`
|
||||
// Held is how many of them the store holds by their manifest.
|
||||
Held int `json:"held"`
|
||||
// Unheld are the kept archives the collector would delete tonight if it ran for real.
|
||||
Unheld []string `json:"unheld"`
|
||||
// Missing are kept archives the store does not have at all.
|
||||
Missing []string `json:"missing"`
|
||||
// Unasked is how many could not be asked about, and why the asking stopped.
|
||||
Unasked int `json:"unasked"`
|
||||
Stopped string `json:"stopped,omitempty"`
|
||||
// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
|
||||
// collected — split by kind.
|
||||
Eligible int `json:"eligible"`
|
||||
EligibleImages int `json:"eligible_images"`
|
||||
EligibleArchives int `json:"eligible_archives"`
|
||||
// SafeToCollect is whether every kept archive was asked about and every one is held.
|
||||
SafeToCollect bool `json:"safe_to_collect"`
|
||||
}
|
||||
|
||||
func collectionCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("collection", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("collection [--json]")
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eligible, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
|
||||
for _, reference := range eligible {
|
||||
if strings.Contains(reference, "/blobs/") {
|
||||
report.EligibleArchives++
|
||||
} else {
|
||||
report.EligibleImages++
|
||||
}
|
||||
}
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if report.Store == "" {
|
||||
report.Unasked = len(kept)
|
||||
report.Stopped = "this mesh has no artifact store on its network"
|
||||
} else {
|
||||
report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
|
||||
}
|
||||
report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
|
||||
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(report)
|
||||
}
|
||||
printCollection(report)
|
||||
return nil
|
||||
}
|
||||
|
||||
// askHeld asks the store about each kept archive, stopping at the first answer that is not about
|
||||
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
|
||||
// identical failures says less than one line.
|
||||
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
|
||||
for i, reference := range kept {
|
||||
held, err := store.Held(ctx, reference)
|
||||
switch {
|
||||
case err == nil && held:
|
||||
report.Held++
|
||||
case err == nil:
|
||||
report.Unheld = append(report.Unheld, reference)
|
||||
case errors.Is(err, artifacts.Gone):
|
||||
report.Missing = append(report.Missing, reference)
|
||||
case errors.Is(err, artifacts.ErrNotOurs):
|
||||
// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
|
||||
report.Unasked++
|
||||
default:
|
||||
return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
|
||||
}
|
||||
}
|
||||
return report.Unasked, ""
|
||||
}
|
||||
|
||||
func printCollection(r collectionReport) {
|
||||
store := r.Store
|
||||
if store == "" {
|
||||
store = "(not on the network)"
|
||||
}
|
||||
fmt.Printf("artifact store %s\n", store)
|
||||
fmt.Printf("kept archives %d\n", r.KeptArchives)
|
||||
fmt.Printf(" held %d\n", r.Held)
|
||||
fmt.Printf(" unheld %d\n", len(r.Unheld))
|
||||
fmt.Printf(" missing %d\n", len(r.Missing))
|
||||
if r.Unasked > 0 {
|
||||
fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped)
|
||||
}
|
||||
fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
|
||||
if len(r.Unheld) > 0 {
|
||||
fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
|
||||
for _, reference := range r.Unheld {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
}
|
||||
if len(r.Missing) > 0 {
|
||||
fmt.Println("\nmissing — kept by the mesh, not in the store:")
|
||||
for _, reference := range r.Missing {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
if r.SafeToCollect {
|
||||
fmt.Println("every kept archive is held: the store's collector may run for real")
|
||||
} else {
|
||||
fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
||||
//
|
||||
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||
// while, with a reason, and that is recorded as a hand act.
|
||||
|
||||
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||
var conditionsFrom *conditions.Keeper
|
||||
|
||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||
// does nothing).
|
||||
Changed: statusFrom.nudge,
|
||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||
}
|
||||
|
||||
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
||||
// it was given before it returns.
|
||||
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||
if conditionsFrom != nil {
|
||||
return f(conditionsFrom)
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
k, err := keeperOn(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
k.Close(flushing)
|
||||
}()
|
||||
return f(k)
|
||||
})
|
||||
}
|
||||
|
||||
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
||||
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
||||
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||
if conditionsFrom != nil {
|
||||
return conditionsFrom.Open(ctx)
|
||||
}
|
||||
if _, err := broker.BusAddress(); err != nil {
|
||||
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||
}
|
||||
var out []conditions.Condition
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, err = conditions.Read(reading, store)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// conditionsUsage is how the verb is typed.
|
||||
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
||||
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
||||
"conditions history [--days N] [--key K] [--json]"
|
||||
|
||||
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
||||
func conditionsCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
return listConditions(ctx, args)
|
||||
case "show":
|
||||
return showCondition(ctx, args)
|
||||
case "silence":
|
||||
return silenceCondition(ctx, args)
|
||||
case "history":
|
||||
return conditionHistory(ctx, args)
|
||||
}
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
|
||||
func listConditions(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||
severity := set.String("severity", "", "only urgent, or only warning")
|
||||
machine := set.String("machine", "", "only those about this machine")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
||||
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
||||
}
|
||||
open, err := openConditions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, c := range open {
|
||||
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
||||
(*machine == "" || concerns(c, *machine)) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Condition{}
|
||||
}
|
||||
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if len(open) == 0 {
|
||||
fmt.Println("no open conditions")
|
||||
} else {
|
||||
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, line := range conditionLines(out, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||
func concerns(c conditions.Condition, machine string) bool {
|
||||
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||
return true
|
||||
}
|
||||
for _, part := range strings.Split(c.Key, ".") {
|
||||
if part == machine {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
||||
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||
var out []string
|
||||
for _, c := range list {
|
||||
times := ""
|
||||
if c.Count > 1 {
|
||||
times = fmt.Sprintf(", raised %d times", c.Count)
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
||||
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
||||
if c.SilencedAt(now) {
|
||||
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
||||
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func showCondition(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New("conditions show <key>")
|
||||
}
|
||||
key := rest[0]
|
||||
var c conditions.Condition
|
||||
var found bool
|
||||
if conditionsFrom != nil {
|
||||
c, found, err = conditionsFrom.Get(ctx, key)
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
||||
"history --key %s` what became of it", key, key)
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(c)
|
||||
}
|
||||
now := time.Now()
|
||||
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
if c.Subject.Machine != "" {
|
||||
fmt.Printf(", on %s", c.Subject.Machine)
|
||||
}
|
||||
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||
now.Sub(c.LastObserved).Round(time.Second))
|
||||
if c.Count > 1 {
|
||||
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
}
|
||||
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||
if c.Silenced != nil {
|
||||
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
c.Silenced.By, c.Silenced.Why)
|
||||
}
|
||||
if len(c.Tried) > 0 {
|
||||
fmt.Println("\n tried:")
|
||||
for _, t := range c.Tried {
|
||||
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n evidence, newest first:")
|
||||
for _, e := range c.Evidence {
|
||||
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolverWords(r string) string {
|
||||
switch r {
|
||||
case conditions.ResolverSelf:
|
||||
return "itself: it clears when observation says it is resolved"
|
||||
case conditions.ResolverOperator:
|
||||
return "the operator: nothing in the mesh will repair it"
|
||||
case conditions.ResolverAgent:
|
||||
return "an agent"
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
||||
// who and why before it is done, its cause the condition's kind unless one is given.
|
||||
func silenceCondition(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
||||
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
||||
acts := addHandActFlags(set)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
||||
}
|
||||
key := rest[0]
|
||||
if err := acts.require("conditions silence"); err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := parseFor(*forFlag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d > conditions.MaxSilence {
|
||||
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
||||
}
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
c, found, err := k.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
||||
}
|
||||
if strings.TrimSpace(*acts.cause) == "" {
|
||||
*acts.cause = c.Kind
|
||||
}
|
||||
*acts.condition = key
|
||||
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
||||
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
||||
"`status` still says it; it clears when observation says it is resolved\n",
|
||||
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// parseFor reads a duration, days included.
|
||||
func parseFor(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
||||
}
|
||||
if days, ok := strings.CutSuffix(s, "d"); ok {
|
||||
n, err := strconv.Atoi(days)
|
||||
if err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("%q is not a number of days", s)
|
||||
}
|
||||
return time.Duration(n) * 24 * time.Hour, nil
|
||||
}
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func conditionHistory(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||
days := set.Int("days", 7, "how many days back, at most 90")
|
||||
key := set.String("key", "", "only this condition")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("conditions history [--days N] [--key K] [--json]")
|
||||
}
|
||||
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
||||
var events []conditions.Event
|
||||
read := func(h conditions.History) error {
|
||||
var err error
|
||||
events, err = h.Since(ctx, since)
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if conditionsFrom != nil {
|
||||
events, err = conditionsFrom.HistorySince(ctx, since)
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
_, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return read(history)
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out []conditions.Event
|
||||
for _, e := range events {
|
||||
if *key == "" || e.Key == *key {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Event{}
|
||||
}
|
||||
return printJSON(map[string]any{"history": out, "days": *days})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for _, e := range out {
|
||||
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
||||
switch e.Change {
|
||||
case conditions.ChangeRaised, conditions.ChangeReopened:
|
||||
line += " — " + e.Summary
|
||||
case conditions.ChangeSeverity:
|
||||
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
||||
case conditions.ChangeResolver:
|
||||
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
||||
default:
|
||||
if e.Why != "" {
|
||||
line += " — " + e.Why
|
||||
}
|
||||
}
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
||||
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
||||
// is not "none open".
|
||||
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
||||
if unread != "" {
|
||||
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
urgent := 0
|
||||
for _, c := range list {
|
||||
if c.Severity == conditions.Urgent {
|
||||
urgent++
|
||||
}
|
||||
}
|
||||
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
||||
for _, line := range conditionLines(list, now) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
||||
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
||||
}
|
||||
|
||||
// orHealer is who tried, as an attempt names it.
|
||||
func orHealer(by string) string {
|
||||
if by == "" {
|
||||
return "a healer"
|
||||
}
|
||||
return by
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
|
||||
// by what is open.
|
||||
|
||||
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "conditions --json"},
|
||||
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
|
||||
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
|
||||
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
|
||||
"conditions history --json --days 3 --key machine.ace.silent"},
|
||||
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
|
||||
"conditions silence machine.ace.silent --for 2h --why on the train"},
|
||||
{map[string]any{"run": "true"}, "doctor run --json"},
|
||||
{map[string]any{}, "doctor --json"},
|
||||
} {
|
||||
verb := "conditions"
|
||||
if strings.HasPrefix(c.want, "doctor") {
|
||||
verb = "doctor"
|
||||
}
|
||||
argv, err := argvFor(verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
|
||||
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
|
||||
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
|
||||
} {
|
||||
if argv, err := argvFor(c.verb, c.args); err == nil {
|
||||
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
|
||||
t.Error("a silence is not a hand act")
|
||||
}
|
||||
}
|
||||
|
||||
// **A silence through the verb is recorded and bounded**; the condition stays open.
|
||||
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
|
||||
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
|
||||
t.Fatal("silenced without saying why")
|
||||
}
|
||||
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
|
||||
t.Fatal("silenced for more than a week")
|
||||
}
|
||||
said := printed(t, func() error {
|
||||
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
|
||||
})
|
||||
if !strings.Contains(said, "is silenced until") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
c, found, _ := k.Get(ctx, "machine.ace.silent")
|
||||
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
|
||||
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
|
||||
t.Fatalf("%s", listed)
|
||||
}
|
||||
}
|
||||
|
||||
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.Fail = errors.New("the bus is away")
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() || asked.conditionsUnread == "" {
|
||||
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.Fail = nil
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A finding one look can be wrong about is raised on the second look in a row** (novox/hq issue 277).
|
||||
//
|
||||
// D2 raised its resolver urgent on one unanswered question, asked once, while the resolver's machine
|
||||
// was loaded by a push and a build starting; thirty questions right after were answered at once. A
|
||||
// single sample of something that is answered over the network, or timed on a machine under load,
|
||||
// is not the invariant failing. So a source marks such a finding `Confirm`, and this holds it back:
|
||||
//
|
||||
// - raised when the source's previous look saw it too — two runs of the self-check in a row (five
|
||||
// minutes apart), or two ticks of the watchdogs (half a minute) — at the severity the source says;
|
||||
// - kept while it is already open, however it is seen, so a condition the next look still sees is
|
||||
// never cleared and raised again (flapping is not news; a reopening within ReopenWithin still is);
|
||||
// - cleared, as everything is, by the look that no longer sees it.
|
||||
//
|
||||
// A finding held back is not a pass: the self-check's verdict names it as unconfirmed. A finding that is
|
||||
// a definite answer — a resolver that answered wrongly, a stream that is not there — is not marked, and
|
||||
// is raised at once.
|
||||
type confirming struct {
|
||||
mu sync.Mutex
|
||||
// last is, by source, the keys of the Confirm findings its previous look saw.
|
||||
last map[string]map[string]bool
|
||||
}
|
||||
|
||||
// pass splits one source's findings into what is raised now and what is held for the next look, and
|
||||
// remembers what it saw. An open condition that cannot be read is kept: unknown is not a reason to
|
||||
// hold a finding back.
|
||||
func (c *confirming) pass(ctx context.Context, keeper *conditions.Keeper, source string,
|
||||
found []conditions.Observation) (raise, held []conditions.Observation) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.last == nil {
|
||||
c.last = map[string]map[string]bool{}
|
||||
}
|
||||
before, seen := c.last[source], map[string]bool{}
|
||||
for _, o := range found {
|
||||
if !o.Confirm {
|
||||
raise = append(raise, o)
|
||||
continue
|
||||
}
|
||||
key := o.Key()
|
||||
seen[key] = true
|
||||
if before[key] || isOpen(ctx, keeper, key) {
|
||||
raise = append(raise, o)
|
||||
continue
|
||||
}
|
||||
held = append(held, o)
|
||||
}
|
||||
c.last[source] = seen
|
||||
return raise, held
|
||||
}
|
||||
|
||||
// isOpen says whether a condition is open; one that cannot be read is taken as open.
|
||||
func isOpen(ctx context.Context, keeper *conditions.Keeper, key string) bool {
|
||||
if keeper == nil {
|
||||
return false
|
||||
}
|
||||
_, open, err := keeper.Get(ctx, key)
|
||||
return open || err != nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A fresh assignment is pushed before its credential exists (novox/hq issue 203): `assign` recorded
|
||||
// the module, `push` sealed a random own secret where the bus credential belongs, and the process
|
||||
// crash-looped until a person ran `module issue` and pushed again. Now assigning a module that speaks
|
||||
// on the bus issues its credential in the same act — or, when the bus cannot be reached from here,
|
||||
// says which verb to run — and a push never seals a placeholder in a credential's place.
|
||||
|
||||
func aTalker() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "talker", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/talker/broker"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/talker", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAssigningAModuleThatSpeaksOnTheBusNamesItsCredential(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
|
||||
// No bus is known to this process, so the credential cannot be issued here: the assignment
|
||||
// stands and says exactly what must happen before a push — never silently.
|
||||
said, err := assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "module issue talker --node laptop") {
|
||||
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
|
||||
}
|
||||
|
||||
// And the push refuses to send it, naming the same verb, rather than sealing a placeholder.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err == nil {
|
||||
t.Fatal("a push sealed a placeholder where talker's bus credential belongs")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "module issue talker --node laptop") || !strings.Contains(err.Error(), "issue 203") {
|
||||
t.Fatalf("the refusal does not say what to run: %v", err)
|
||||
}
|
||||
|
||||
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
|
||||
// does not mint again: a credential rotates on purpose, never by habit.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "laptop.talker", Kind: inventory.BusModule, Node: "laptop", Module: "talker"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err = assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "module issue") {
|
||||
t.Fatalf("a module with a minted credential was told to issue one:\n%s", said)
|
||||
}
|
||||
again, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again != hash {
|
||||
t.Fatal("re-assigning rotated the credential")
|
||||
}
|
||||
}
|
||||
|
||||
// A module that declares no broker secret is left alone: nothing to issue, nothing said.
|
||||
func TestAssigningAModuleThatDoesNotSpeakSaysNothingOfCredentials(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
register(t, open, helloWeb())
|
||||
said, err := assign(t.Context(), open, "laptop", "hello-web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "credential") {
|
||||
t.Fatalf("a module without a broker secret was told about credentials:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,931 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module declares the data it holds, and the mesh protects and watches it from that declaration
|
||||
// (novox/hq ADR 0233).
|
||||
//
|
||||
// The self-check's D13 composes what every machine declares, asks each machine's backup holder what
|
||||
// it measured of every item — size, newest write, newest good backup, the redundant storage it is on —
|
||||
// and keeps both. From that, and from what every provider says it holds for its consumers, it raises,
|
||||
// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking):
|
||||
//
|
||||
// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least
|
||||
// shrinkFloor less; `data-missing`: its path is gone;
|
||||
// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a
|
||||
// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on
|
||||
// empty databases while their real ones sat on another machine (issue 273);
|
||||
// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot
|
||||
// be compared;
|
||||
// - `data-quiet`: an item said to be written all the time has not been, within its bound;
|
||||
// - `backup-stale`: an item's newest good backup is older than its bound, or there is none;
|
||||
// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read;
|
||||
// `protection-missing`: an item said to be protected by redundancy is on storage that is not;
|
||||
// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person.
|
||||
//
|
||||
// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no
|
||||
// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by
|
||||
// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with
|
||||
// anything in it; this is the record of what it kept. An operator's path is never retired or deleted.
|
||||
|
||||
// The condition kinds of D13.
|
||||
const (
|
||||
kindDataShrank = "data-shrank"
|
||||
kindEmptyReplacement = "empty-replacement"
|
||||
kindDataHeldTwice = "data-held-twice"
|
||||
kindDataQuiet = "data-quiet"
|
||||
kindBackupStale = "backup-stale"
|
||||
kindDataUnmeasured = "data-unmeasured"
|
||||
// kindDataMissing is a watched item whose path is gone.
|
||||
kindDataMissing = "data-missing"
|
||||
// kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read.
|
||||
kindArrayDegraded = "array-degraded"
|
||||
// kindProtectionMissing is an item said to be protected by redundancy, on storage that is not.
|
||||
kindProtectionMissing = "protection-missing"
|
||||
)
|
||||
|
||||
// probeDataID is the self-check's id for this probe.
|
||||
const probeDataID = "D13"
|
||||
|
||||
// The bounds the findings are read against.
|
||||
var (
|
||||
// shrinkWindow is how far back the largest size is looked for.
|
||||
shrinkWindow = 7 * 24 * time.Hour
|
||||
// shrinkFloor is the least loss that is worth saying: two empty databases differ by a few
|
||||
// megabytes, and half of almost nothing is noise.
|
||||
shrinkFloor int64 = 16 << 20
|
||||
// dataAsk is how long one machine's holder, or one provider, is given to answer.
|
||||
dataAsk = 8 * time.Second
|
||||
)
|
||||
|
||||
// keyOfItem is one item's condition id: its machine, module and item.
|
||||
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
|
||||
|
||||
// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data).
|
||||
type holderAnswer struct {
|
||||
Module string `json:"module"`
|
||||
Data []holderItem `json:"data"`
|
||||
}
|
||||
|
||||
// holderItem is one item as the holder measured it.
|
||||
type holderItem struct {
|
||||
Item string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path"`
|
||||
SizeBytes *int64 `json:"size_bytes"`
|
||||
LastWrite *time.Time `json:"last_write"`
|
||||
MeasuredAt *time.Time `json:"measured_at"`
|
||||
LastBackup *time.Time `json:"last_backup"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233).
|
||||
Precision string `json:"precision,omitempty"`
|
||||
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
|
||||
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
|
||||
}
|
||||
|
||||
// readHolder reads a holder's answer into measurements by module and item.
|
||||
func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) {
|
||||
var modules []holderAnswer
|
||||
if err := json.Unmarshal(raw, &modules); err != nil {
|
||||
return nil, fmt.Errorf("its answer is not readable: %w", err)
|
||||
}
|
||||
out := map[string]map[string]inventory.Measurement{}
|
||||
for _, m := range modules {
|
||||
for _, it := range m.Data {
|
||||
if out[m.Module] == nil {
|
||||
out[m.Module] = map[string]inventory.Measurement{}
|
||||
}
|
||||
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
|
||||
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy,
|
||||
Precision: it.Precision}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaredOn is every data item a machine's composition declares, and the module there that holds
|
||||
// node-backup to measure them — empty for none.
|
||||
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) {
|
||||
var out []inventory.DeclaredData
|
||||
held := ""
|
||||
for _, m := range plan.Modules {
|
||||
for _, c := range m.Claims {
|
||||
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
|
||||
held = m.Module
|
||||
}
|
||||
}
|
||||
for _, it := range m.DataItems() {
|
||||
out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class,
|
||||
Owned: it.OwnedByModule(), Protection: it.Protection()})
|
||||
}
|
||||
}
|
||||
return out, held
|
||||
}
|
||||
|
||||
// consumerCopy is one provider's account of one consumer: where, how big, and whether still active.
|
||||
type consumerCopy struct {
|
||||
Node, Module, Consumer string
|
||||
Size *int64
|
||||
Retired bool
|
||||
// Class is how precious the consumer's data is: the stricter of what the provider keeps for its
|
||||
// consumers and what the consumer says it keeps there (`kept-by`).
|
||||
Class string
|
||||
}
|
||||
|
||||
// probeData is D13.
|
||||
func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("no bus to ask the machines over")
|
||||
}
|
||||
open := d.open
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
now := time.Now()
|
||||
delivered, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
type machine struct {
|
||||
name string
|
||||
declared []inventory.DeclaredData
|
||||
held bool
|
||||
measured map[string]map[string]inventory.Measurement
|
||||
askErr error
|
||||
}
|
||||
var machines []*machine
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// A machine that cannot be worked out declares nothing this run — which is not the same as
|
||||
// declaring nothing: retiring its data on that would be acting on an unreadable result.
|
||||
continue
|
||||
}
|
||||
declared, holder := declaredOn(plan)
|
||||
// **A holder is asked only once its machine has been sent it and had time to report** (novox/hq
|
||||
// issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it
|
||||
// measured" about it was a warning for a push nobody had made yet.
|
||||
held := holder != "" && delivered[n.Name].settled(holder, now)
|
||||
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
|
||||
}
|
||||
// Every holder asked at once, as D8 asks every ban list.
|
||||
var wg sync.WaitGroup
|
||||
for _, m := range machines {
|
||||
if !m.held || !heard[m.name] {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(m *machine) {
|
||||
defer wg.Done()
|
||||
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||
defer cancel()
|
||||
raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name)
|
||||
if err == nil {
|
||||
m.measured, err = readHolder(raw)
|
||||
}
|
||||
m.askErr = err
|
||||
}(m)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var out []conditions.Observation
|
||||
for _, m := range machines {
|
||||
if m.askErr != nil {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
|
||||
Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
|
||||
"nothing about that data is known", m.name),
|
||||
Said: firstLine(m.askErr.Error())})
|
||||
}
|
||||
why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
|
||||
change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err)
|
||||
}
|
||||
for _, r := range change.Retired {
|
||||
log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+
|
||||
"delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why)
|
||||
}
|
||||
for _, r := range change.Reenabled {
|
||||
log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine)
|
||||
}
|
||||
}
|
||||
|
||||
records, err := open.inventory.Data(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bindings, err := open.inventory.Bindings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||
copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func orUnknownPath(p string) string {
|
||||
if p == "" {
|
||||
return "a path its backup holder never named"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at
|
||||
// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding.
|
||||
func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var asked []providerInstance
|
||||
for _, p := range instances {
|
||||
m := shelf[p.Module]
|
||||
keeps := false
|
||||
for provision := range m.Grants {
|
||||
keeps = keeps || m.KeepsConsumerData(provision)
|
||||
}
|
||||
if keeps {
|
||||
asked = append(asked, p)
|
||||
}
|
||||
}
|
||||
states := make([]*link.RetirementState, len(asked))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range asked {
|
||||
wg.Add(1)
|
||||
go func(i int, p providerInstance) {
|
||||
defer wg.Done()
|
||||
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||
defer cancel()
|
||||
if s, err := askRetirement(asking, conn, p); err == nil {
|
||||
states[i] = &s
|
||||
}
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
var out []consumerCopy
|
||||
for i, p := range asked {
|
||||
s := states[i]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
class := consumersClass(shelf[p.Module])
|
||||
for _, c := range s.Held {
|
||||
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c,
|
||||
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])}
|
||||
if size, ok := s.HeldSizes[c]; ok && size >= 0 {
|
||||
size := size
|
||||
cp.Size = &size
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
for _, r := range s.Retired {
|
||||
if r.Kind != "" && r.Kind != "consumer" {
|
||||
continue
|
||||
}
|
||||
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true,
|
||||
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])}
|
||||
if r.SizeBytes != nil && *r.SizeBytes >= 0 {
|
||||
cp.Size = r.SizeBytes
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning
|
||||
// for anything else watched (the operator's ranking, ADR 0233).
|
||||
func severityOf(class string) conditions.Severity {
|
||||
if class == catalogue.ClassIrreplaceable {
|
||||
return conditions.Urgent
|
||||
}
|
||||
return conditions.Warning
|
||||
}
|
||||
|
||||
// consumersClass is the most precious class a provider keeps any of its consumers' data as.
|
||||
func consumersClass(m catalogue.Manifest) string {
|
||||
class := catalogue.ClassNone
|
||||
for provision := range m.Grants {
|
||||
if c, ok := m.ConsumerDataOf(provision); ok {
|
||||
class = catalogue.StricterClass(class, c.Class)
|
||||
} else if m.KeepsConsumerData(provision) {
|
||||
class = catalogue.StricterClass(class, catalogue.ClassValuable)
|
||||
}
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read
|
||||
// through where each is bound: by provider module and consumer identity, the class of that consumer's
|
||||
// data there; and by provider item key (machine/module/item), the class the item holding it is held to.
|
||||
func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) {
|
||||
upgraded, keptBy := map[string]string{}, map[string]string{}
|
||||
for _, b := range bindings {
|
||||
m, ok := shelf[b.Consumer]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
k, said := m.KeptByOf(b.Provision)
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module))
|
||||
key := b.Provider.Module + "/" + identity
|
||||
keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class)
|
||||
if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" {
|
||||
if _, own := shelf[b.Provider.Module].DataItem(pc.In); own {
|
||||
item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In
|
||||
upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class)
|
||||
}
|
||||
}
|
||||
}
|
||||
return upgraded, keptBy
|
||||
}
|
||||
|
||||
// dataFindings is every condition the data on record raises now. A function of what is known, so the
|
||||
// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer
|
||||
// of its module keeps data in it more precious than its own class says (`kept-by`), by its key.
|
||||
func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest,
|
||||
copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
byItem := map[string][]inventory.DataRecord{}
|
||||
arrays := map[string][]inventory.DataRecord{}
|
||||
type shrunk struct {
|
||||
machine, dataset, class string
|
||||
size, peak int64
|
||||
items []string
|
||||
}
|
||||
shrunkDatasets := map[string]shrunk{}
|
||||
for _, r := range records {
|
||||
if r.DeletedAt != nil {
|
||||
continue
|
||||
}
|
||||
class := catalogue.StricterClass(r.Class, upgraded[r.Key()])
|
||||
r.Class = class
|
||||
byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r)
|
||||
item, declared := shelf[r.Module].DataItem(r.Item)
|
||||
id := keyOfItem(r.Machine, r.Module, r.Item)
|
||||
if r.Retired() {
|
||||
if now.Sub(*r.RetiredAt) > cleanupAfter {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
|
||||
Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
|
||||
"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
|
||||
r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
|
||||
r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module),
|
||||
Said: "kept at " + orUnknownPath(r.Path)})
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !catalogue.Watched(class) {
|
||||
continue
|
||||
}
|
||||
severity := severityOf(class)
|
||||
if r.Redundancy != nil {
|
||||
where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
|
||||
arrays[where] = append(arrays[where], r)
|
||||
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
|
||||
Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
|
||||
"and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
|
||||
r.Item, r.Module, r.Machine, class),
|
||||
Said: orUnknownPath(r.Path) + " is on no redundant storage the backup holder can read"})
|
||||
}
|
||||
if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
|
||||
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.Item,
|
||||
r.Module, r.Machine, class),
|
||||
Said: orUnknownPath(r.Path) + " does not exist: " + firstLine(r.MeasureError)})
|
||||
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
|
||||
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
|
||||
// Several items on one dataset share its size: one condition for the dataset, as loud as the
|
||||
// most precious item on it.
|
||||
k := r.Machine + "/" + inventory.Dataset(r.Precision)
|
||||
ds := shrunkDatasets[k]
|
||||
ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak
|
||||
ds.class = catalogue.StricterClass(ds.class, class)
|
||||
ds.items = append(ds.items, r.Module+"/"+r.Item)
|
||||
shrunkDatasets[k] = ds
|
||||
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
|
||||
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
|
||||
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
|
||||
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
|
||||
"good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak),
|
||||
int(shrinkWindow.Hours()/24))})
|
||||
}
|
||||
if !declared {
|
||||
continue
|
||||
}
|
||||
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet,
|
||||
Kind: kindDataQuiet, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
|
||||
"whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339),
|
||||
within)})
|
||||
}
|
||||
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
|
||||
// plan, said only where the machine was measured — where a holder is there to take it.
|
||||
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
|
||||
within := item.BackupWithin()
|
||||
switch {
|
||||
case r.LastBackup == nil && now.Sub(r.FirstSeen) > within:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
|
||||
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
|
||||
r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))})
|
||||
case r.LastBackup != nil && now.Sub(*r.LastBackup) > within:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
|
||||
"of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, k := range keysSorted(shrunkDatasets) {
|
||||
ds := shrunkDatasets[k]
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
|
||||
Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than half of "+
|
||||
"what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak),
|
||||
int(shrinkWindow.Hours()/24)),
|
||||
Said: "the dataset " + ds.dataset})
|
||||
}
|
||||
// The redundant storage watched data is on: one condition per array, as loud as the most precious
|
||||
// item on it — the array, not each item, is what degrades.
|
||||
for _, where := range keysSorted(arrays) {
|
||||
rs := arrays[where]
|
||||
red := rs[0].Redundancy
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
continue
|
||||
}
|
||||
class, machine := catalogue.ClassValuable, rs[0].Machine
|
||||
var names []string
|
||||
for _, r := range rs {
|
||||
class = catalogue.StricterClass(class, r.Class)
|
||||
names = append(names, r.Module+"/"+r.Item)
|
||||
}
|
||||
state := "could not be read"
|
||||
if red.Healthy != nil {
|
||||
state = "is NOT healthy"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
|
||||
Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
|
||||
Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, strings.Join(names, ", "),
|
||||
state),
|
||||
Said: fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said))})
|
||||
}
|
||||
// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
|
||||
// an empty replacement. Only against a retired copy — a module running on two machines on purpose
|
||||
// keeps two different sets of data.
|
||||
for _, key := range keysSorted(byItem) {
|
||||
rs := byItem[key]
|
||||
for _, a := range rs {
|
||||
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) {
|
||||
continue
|
||||
}
|
||||
for _, o := range rs {
|
||||
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) ||
|
||||
!replacedByLess(*a.Size, *o.Size) {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||
Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
|
||||
"an empty replacement of its data. Move the data, or assign it back where its data is",
|
||||
a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, consumerFindings(copies)...)
|
||||
return out
|
||||
}
|
||||
|
||||
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than
|
||||
// half of it, and at least shrinkFloor less.
|
||||
func replacedByLess(inUse, kept int64) bool {
|
||||
return inUse*2 < kept && kept-inUse >= shrinkFloor
|
||||
}
|
||||
|
||||
// consumerFindings is the same question of consumers' data at providers: one consumer, the same
|
||||
// provider module on two machines.
|
||||
func consumerFindings(copies []consumerCopy) []conditions.Observation {
|
||||
by := map[string][]consumerCopy{}
|
||||
for _, c := range copies {
|
||||
k := c.Module + "/" + c.Consumer
|
||||
by[k] = append(by[k], c)
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, k := range keysSorted(by) {
|
||||
cs := by[k]
|
||||
if len(cs) < 2 {
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, a := range cs {
|
||||
if a.Retired || a.Size == nil {
|
||||
continue
|
||||
}
|
||||
for _, o := range cs {
|
||||
if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
|
||||
continue
|
||||
}
|
||||
state := "active"
|
||||
if o.Retired {
|
||||
state = "retired"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||
Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)),
|
||||
Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+
|
||||
"consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+
|
||||
"the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node,
|
||||
sizeWords(o.Size), o.Node)})
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if found {
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
continue
|
||||
}
|
||||
var active []consumerCopy
|
||||
for _, c := range cs {
|
||||
if !c.Retired {
|
||||
active = append(active, c)
|
||||
}
|
||||
}
|
||||
if len(active) >= 2 {
|
||||
var where []string
|
||||
var also []string
|
||||
for _, c := range active {
|
||||
where = append(where, c.Node+" ("+sizeWords(c.Size)+")")
|
||||
also = append(also, c.Node)
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice,
|
||||
Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses",
|
||||
active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keysSorted[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ---- the `data` verb ---------------------------------------------------------------------------
|
||||
|
||||
const dataUsage = "data [--json] [--machine <name>] [--retired]"
|
||||
|
||||
// dataRow is one item as `data` lists it.
|
||||
type dataRow struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
Item string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Protection string `json:"protection,omitempty"`
|
||||
// Array is the redundant storage it is on and its state, where its holder could tell.
|
||||
Array string `json:"array,omitempty"`
|
||||
Unmeasured string `json:"unmeasured,omitempty"`
|
||||
// Precision says what the size is: exact, a dataset's whole size, partial, or none.
|
||||
Precision string `json:"precision,omitempty"`
|
||||
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
||||
LastWrite string `json:"last-write,omitempty"`
|
||||
MeasuredAt string `json:"measured-at,omitempty"`
|
||||
LastBackup string `json:"last-backup,omitempty"`
|
||||
BackupDue string `json:"backup-within,omitempty"`
|
||||
Retired string `json:"retired,omitempty"`
|
||||
RetiredWhy string `json:"retired-why,omitempty"`
|
||||
Deleted string `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last
|
||||
// found it.
|
||||
func dataCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("data", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
only := set.String("machine", "", "one machine")
|
||||
retiredOnly := set.Bool("retired", false, "only what is retired")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(dataUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
records, err := open.inventory.Data(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows := dataRows(records, shelf, *only, *retiredOnly)
|
||||
if *asJSON {
|
||||
return printJSON(map[string]any{"data": rows})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run")
|
||||
return nil
|
||||
}
|
||||
for _, r := range rows {
|
||||
state := ""
|
||||
switch {
|
||||
case r.Deleted != "":
|
||||
state = " DELETED " + r.Deleted
|
||||
case r.Retired != "":
|
||||
state = " RETIRED " + r.Retired + " — " + r.RetiredWhy
|
||||
}
|
||||
fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class,
|
||||
sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state)
|
||||
if r.Array != "" {
|
||||
fmt.Printf(" on %s\n", r.Array)
|
||||
}
|
||||
if r.Precision != "" && r.Precision != "exact" {
|
||||
fmt.Printf(" size: %s\n", r.Precision)
|
||||
}
|
||||
if r.Unmeasured != "" {
|
||||
fmt.Printf(" not measured: %s\n", r.Unmeasured)
|
||||
}
|
||||
if r.Class == catalogue.ClassCache {
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt),
|
||||
orNever(r.LastBackup), within(r.BackupDue))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow {
|
||||
rows := []dataRow{}
|
||||
stamp := func(t *time.Time) string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
for _, r := range records {
|
||||
if only != "" && r.Machine != only {
|
||||
continue
|
||||
}
|
||||
if retiredOnly && !r.Retired() {
|
||||
continue
|
||||
}
|
||||
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
|
||||
Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
|
||||
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
|
||||
Deleted: stamp(r.DeletedAt)}
|
||||
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
|
||||
row.BackupDue = it.BackupWithin().String()
|
||||
}
|
||||
if red := r.Redundancy; red != nil {
|
||||
state := "state unread"
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
state = "healthy"
|
||||
} else if red.Healthy != nil {
|
||||
state = "NOT HEALTHY"
|
||||
}
|
||||
row.Array = red.Kind + " " + red.Where + ", " + state
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func orNothingWord(s string) string {
|
||||
if s == "" || s == "none" {
|
||||
return "nothing"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func orNever(s string) string {
|
||||
if s == "" {
|
||||
return "never"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func within(s string) string {
|
||||
if s == "" {
|
||||
return " (not backed up)"
|
||||
}
|
||||
return " (bound " + s + ")"
|
||||
}
|
||||
|
||||
// ---- cleanup of retired own data ---------------------------------------------------------------
|
||||
|
||||
// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first
|
||||
// takes a last restore point of it, tagged as retired, and only then removes it — in the background,
|
||||
// because a large item outlasts any call — and the second says how that went. Module tools, not seat
|
||||
// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete.
|
||||
const (
|
||||
ToolDeleteRetired = "backup_delete_retired"
|
||||
ToolDeletedOutcome = "backup_deleted"
|
||||
)
|
||||
|
||||
// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person.
|
||||
var deletionWait = 8 * time.Minute
|
||||
|
||||
// deletionPoll is how often it asks.
|
||||
var deletionPoll = 5 * time.Second
|
||||
|
||||
// deletion is a holder's account of one deletion.
|
||||
type deletion struct {
|
||||
Started bool `json:"started"`
|
||||
Running bool `json:"running"`
|
||||
Done bool `json:"done"`
|
||||
OK bool `json:"ok"`
|
||||
Snapshot string `json:"snapshot"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// retiredData is every retired item on record, as `cleanup list` shows them.
|
||||
func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow {
|
||||
var out []retiredRow
|
||||
for _, r := range records {
|
||||
if !r.Retired() {
|
||||
continue
|
||||
}
|
||||
out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind,
|
||||
RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24),
|
||||
SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer.
|
||||
const retiredDataKind = "own-data"
|
||||
|
||||
// holderOn is the module holding node-backup on a machine.
|
||||
func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) {
|
||||
held, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, h := range held {
|
||||
if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine {
|
||||
return h.Module, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+
|
||||
"(after a last restore point)", catalogue.BackupSeat, machine)
|
||||
}
|
||||
|
||||
// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and
|
||||
// never one not retired. The holder takes a last restore point of it first, so the deletion can be
|
||||
// undone until a person forgets that restore point; the record says deleted only once the holder says
|
||||
// it is.
|
||||
func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error {
|
||||
inv := open.inventory
|
||||
if !r.Retired() {
|
||||
return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done",
|
||||
r.Item, r.Module, r.Machine)
|
||||
}
|
||||
if r.Path == "" {
|
||||
return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted",
|
||||
r.Item, r.Module, r.Machine)
|
||||
}
|
||||
if plan, _, err := planFor(ctx, open, r.Machine); err == nil {
|
||||
for _, m := range plan.Modules {
|
||||
if _, still := m.DataItem(r.Item); still && m.Module == r.Module {
|
||||
return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done",
|
||||
r.Module, r.Machine, r.Item)
|
||||
}
|
||||
}
|
||||
}
|
||||
holder, err := holderOn(ctx, inv, r.Machine)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item})
|
||||
args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item,
|
||||
"why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController}
|
||||
ask := func(tool string) (deletion, error) {
|
||||
var d deletion
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error)
|
||||
}
|
||||
return d, unmarshalAnswer(answer, &d)
|
||||
}
|
||||
d, err := ask(ToolDeleteRetired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(deletionPoll):
|
||||
}
|
||||
if d, err = ask(ToolDeletedOutcome); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !d.Done:
|
||||
fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+
|
||||
"showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n",
|
||||
holder, r.Machine, r.Path)
|
||||
return nil
|
||||
case !d.OK:
|
||||
return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error)
|
||||
}
|
||||
if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil {
|
||||
return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err)
|
||||
}
|
||||
fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n",
|
||||
holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot))
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its
|
||||
// own directories on the machine:
|
||||
// kept, and retired at the self-check's next run.
|
||||
func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string {
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()}
|
||||
}
|
||||
var out []string
|
||||
for _, r := range records {
|
||||
if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned {
|
||||
continue
|
||||
}
|
||||
for _, m := range modules {
|
||||
if r.Module == m {
|
||||
out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+
|
||||
"never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)",
|
||||
r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,465 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func bytesOf(n int64) *int64 { return &n }
|
||||
|
||||
func when(t time.Time) *time.Time { return &t }
|
||||
|
||||
func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
|
||||
t.Helper()
|
||||
out := map[string]catalogue.Manifest{}
|
||||
for _, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[m.Module] = m
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const houseManifest = `{"module":"house","version":"1",
|
||||
"data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]},
|
||||
"resources":[{"id":"config","type":"directory","mode":"0700"}]}`
|
||||
|
||||
func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation {
|
||||
out := map[string]conditions.Observation{}
|
||||
for _, o := range linted(obs) {
|
||||
out[o.Kind] = o
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound,
|
||||
// by one changed rule, to the store on the control node, which made each an empty database — while
|
||||
// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired
|
||||
// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines
|
||||
// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says
|
||||
// its data there is irreplaceable (`kept-by`).
|
||||
func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) {
|
||||
var copies []consumerCopy
|
||||
for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} {
|
||||
copies = append(copies,
|
||||
consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"},
|
||||
consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"})
|
||||
}
|
||||
copies[1].Class = "irreplaceable" // the photo site's own database says so
|
||||
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
|
||||
if len(got) != 5 {
|
||||
t.Fatalf("%d findings for five empty replacements: %+v", len(got), got)
|
||||
}
|
||||
for i, o := range got {
|
||||
want := conditions.Warning
|
||||
if strings.Contains(o.ID, "mesh_home_board") {
|
||||
want = conditions.Urgent
|
||||
}
|
||||
_ = i
|
||||
if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" ||
|
||||
len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") {
|
||||
t.Errorf("%+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// While the old copy is still active (the first ten minutes), it is the same finding.
|
||||
copies[0].Retired = false
|
||||
copies[1].Class = "valuable"
|
||||
if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement {
|
||||
t.Fatalf("with the old copy still active: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and
|
||||
// a retired one at the old: nothing to say here; `cleanup` covers the old one.
|
||||
func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) {
|
||||
copies := []consumerCopy{
|
||||
{Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true},
|
||||
{Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)},
|
||||
}
|
||||
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||
t.Fatalf("a deliberate move raised %+v", got)
|
||||
}
|
||||
// Two small databases differing by less than the floor are not a finding either.
|
||||
copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20)
|
||||
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||
t.Fatalf("noise between two empty databases raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning,
|
||||
// since which one is empty cannot be told.
|
||||
func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) {
|
||||
copies := []consumerCopy{
|
||||
{Node: "home", Module: "minio", Consumer: "mesh_home_photos"},
|
||||
{Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"},
|
||||
}
|
||||
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
|
||||
if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The same incident for a module's own data: a module unassigned from one machine and assigned on
|
||||
// another starts over in an empty directory while its full one is kept, retired, where it was.
|
||||
func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) {
|
||||
now := time.Now()
|
||||
retired := now.Add(-time.Hour)
|
||||
records := []inventory.DataRecord{
|
||||
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||
Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)},
|
||||
{Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||
Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)},
|
||||
}
|
||||
got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
|
||||
o, ok := got[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
// The same of a valuable item is a warning.
|
||||
records[0].Class, records[1].Class = "valuable", "valuable"
|
||||
if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning {
|
||||
t.Fatalf("a valuable empty replacement: %+v", o)
|
||||
}
|
||||
records[0].Class, records[1].Class = "irreplaceable", "irreplaceable"
|
||||
// Two machines running a module on purpose, both active, keep two sets of data: nothing to say.
|
||||
records[0].RetiredAt = nil
|
||||
if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" {
|
||||
t.Fatalf("two active copies were read as a replacement: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss,
|
||||
// or a loss under the floor, is not a finding.
|
||||
func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||
now := time.Now()
|
||||
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)}
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank]
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
for _, peak := range []int64{500 << 20, 20 << 20} {
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||
t.Errorf("a peak of %d raised a shrink", peak)
|
||||
}
|
||||
}
|
||||
small := r
|
||||
small.Size = bytesOf(1 << 20)
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||
t.Error("a loss under the floor raised a shrink")
|
||||
}
|
||||
}
|
||||
|
||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||
now := time.Now()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)),
|
||||
LastBackup: when(now.Add(-72 * time.Hour))}
|
||||
got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))
|
||||
if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent {
|
||||
t.Fatalf("irreplaceable: %+v", got)
|
||||
}
|
||||
valuable := r
|
||||
valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning ||
|
||||
got[kindBackupStale].Severity != conditions.Warning {
|
||||
t.Fatalf("valuable: %+v", got)
|
||||
}
|
||||
never := r
|
||||
never.LastBackup = nil
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") {
|
||||
t.Fatalf("never backed up: %+v", o)
|
||||
}
|
||||
fresh := never
|
||||
fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now)
|
||||
if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An item retired more than thirty days waits for a person; less, it is only listed.
|
||||
func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) {
|
||||
now := time.Now()
|
||||
old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour)
|
||||
records := []inventory.DataRecord{
|
||||
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old},
|
||||
{Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent},
|
||||
}
|
||||
got := linted(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The holder's answer reads into measurements, by module and item.
|
||||
func TestTheHoldersAnswerIsRead(t *testing.T) {
|
||||
raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3,
|
||||
"data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps",
|
||||
"size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`)
|
||||
got, err := readHolder(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got["postgres"]["store"]
|
||||
if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil {
|
||||
t.Fatalf("%+v", m)
|
||||
}
|
||||
// An older holder, which says no data, reads as nothing measured rather than a failure.
|
||||
if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 {
|
||||
t.Fatalf("%v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it
|
||||
// measured.
|
||||
type fakeHolder struct {
|
||||
mu sync.Mutex
|
||||
modules []map[string]any
|
||||
}
|
||||
|
||||
func (f *fakeHolder) set(modules ...map[string]any) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.modules = modules
|
||||
}
|
||||
|
||||
func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) {
|
||||
t.Helper()
|
||||
sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func measuredHouse(path string, size int64, at time.Time) map[string]any {
|
||||
return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{
|
||||
"item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size,
|
||||
"last_write": at, "measured_at": at, "last_backup": at}}}
|
||||
}
|
||||
|
||||
// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty
|
||||
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
|
||||
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
|
||||
// up on another machine with an empty directory while the full one waits retired, that is urgent.
|
||||
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
now := time.Now()
|
||||
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||
var d13 probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeDataID {
|
||||
d13 = p
|
||||
}
|
||||
}
|
||||
run := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||
obs, err := probeData(probing, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||
if obs := run(); len(obs) != 0 {
|
||||
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "house")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||
}
|
||||
laptop.set()
|
||||
run()
|
||||
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||
}
|
||||
records, _ := inv.Data(ctx)
|
||||
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
|
||||
// Assigned on the anchor, onto an empty directory.
|
||||
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||
obs := findingsByKind(run())
|
||||
o, ok := obs[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
|
||||
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
|
||||
// whose path is gone is said.
|
||||
func TestTheArrayUnderDataIsWatched(t *testing.T) {
|
||||
now := time.Now()
|
||||
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"},
|
||||
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}`
|
||||
shelf := shelfFor(t, media)
|
||||
sick := false
|
||||
on := func(item string, healthy *bool) inventory.DataRecord {
|
||||
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false,
|
||||
Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}}
|
||||
}
|
||||
got := linted(dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
well := true
|
||||
if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("a healthy array raised %+v", got)
|
||||
}
|
||||
plain := on("films", nil)
|
||||
plain.Redundancy = nil
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("redundancy said and not found: %+v", o)
|
||||
}
|
||||
gone := on("films", &well)
|
||||
gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0)
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("a vanished library: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class:
|
||||
// the photo site's objects make the object store's data an urgent matter.
|
||||
func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
|
||||
objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"},
|
||||
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
||||
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`
|
||||
photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`
|
||||
shelf := shelfFor(t, objects, photos)
|
||||
bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket",
|
||||
Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}}
|
||||
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||
if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" {
|
||||
t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy)
|
||||
}
|
||||
now := time.Now()
|
||||
r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true,
|
||||
Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))}
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("the photos' store without a backup: %+v", o)
|
||||
}
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning {
|
||||
t.Fatalf("a valuable store without a backup: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition
|
||||
// naming every item on it, not one per item; and a partial walk's lower bound is never compared.
|
||||
func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) {
|
||||
now := time.Now()
|
||||
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"},
|
||||
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}`
|
||||
shelf := shelfFor(t, media, houseManifest)
|
||||
well := true
|
||||
on := func(item string, size int64) inventory.DataRecord {
|
||||
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable",
|
||||
Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||
Precision: "dataset tank/media: its whole size",
|
||||
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}}
|
||||
}
|
||||
films, shows := on("films", 30<<40), on("shows", 30<<40)
|
||||
peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40}
|
||||
got := linted(dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now))
|
||||
if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now),
|
||||
LastBackup: when(now), Precision: "partial: measured partially"}
|
||||
if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("a partial size was compared: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,643 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The controller's part in a delivery (novox/hq ADR 0239, to-be 47).
|
||||
//
|
||||
// **A delivery is mesh-delivery's; the walk is the controller's.** A delivery is one commit in one
|
||||
// repository from its pull request's head to every machine, and a delivery group a few of them sharing a
|
||||
// branch name; their states, order, holds and record belong to the module holding the `mesh-delivery`
|
||||
// seat. The controller keeps what it already owned — the planner, the gate, registration, sending, the
|
||||
// rollback — and the **walk**: one trunk commit's plan, tier by tier across the machines, one machine
|
||||
// first and judged at the gate (ADR 0236). What changes here is when a walk starts.
|
||||
//
|
||||
// - A walk that moves the controller, the node-engine, the node tools, the bus or mesh-delivery itself is
|
||||
// **on the controller's own path**: started by the merge, as every plan was. mesh-delivery cannot gate
|
||||
// or deliver itself, and nothing the core needs to be repaired may wait for it.
|
||||
// - Any other walk, **while the seat has a holder on record**, is opened by the merge and waits — nothing
|
||||
// asked, nothing registered, nothing sent — until mesh-delivery says `deliver`, or a person says
|
||||
// `plans go`. Nothing of it is registered before then, so no other send can carry it to a machine
|
||||
// before its turn (ADR 0236 §4a carries everything registered).
|
||||
// - With no holder on record, every walk starts at the merge, exactly as before this existed.
|
||||
//
|
||||
// The verbs mesh-delivery asks with are here: `delivery plan` (the planner's one answer for a diffset),
|
||||
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
|
||||
// go`, `delivery stop` and `delivery walks`.
|
||||
|
||||
// onTheControllersPath are the modules whose walk never waits for the delivery's owner, and what each is.
|
||||
var onTheControllersPath = map[string]string{
|
||||
"mesh-controller": "the controller",
|
||||
"mesh-host": "the node-engine",
|
||||
"node-tools": "the node tools",
|
||||
"nats": "the bus",
|
||||
catalogue.DeliverySeat: "the delivery's owner",
|
||||
}
|
||||
|
||||
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
|
||||
// holder on record. Read from the catalogue the merge handler already holds; never from whether the
|
||||
// holder answers, so a walk does not start by itself because mesh-delivery is down — that wait is said.
|
||||
func deliverySeatHeld(entries []inventory.Entry) bool {
|
||||
for _, e := range entries {
|
||||
if len(e.On) > 0 && e.Manifest.ClaimsSeat(catalogue.DeliverySeat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// awaitsFor is what a new walk waits for: nil when it starts at once — no holder on record, or a module
|
||||
// on the controller's own path among those it moves.
|
||||
func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDelivery {
|
||||
if !deliverySeatHeld(entries) {
|
||||
return nil
|
||||
}
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}
|
||||
}
|
||||
|
||||
// waitingNote is what a walk waiting for its word says, wherever it is read.
|
||||
func waitingNote(p inventory.Plan) string {
|
||||
return fmt.Sprintf("published; its walk waits for %s's word — `plans go %s --why …` starts it by hand",
|
||||
p.Delivery.Awaits, p.ID)
|
||||
}
|
||||
|
||||
// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its
|
||||
// first tier. Refused for a walk that does not wait.
|
||||
func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
switch {
|
||||
case !p.Open():
|
||||
return p, fmt.Errorf("%s is %s: there is no walk to start", p.ID, p.State)
|
||||
case p.Delivery == nil || p.Delivery.Awaits == "":
|
||||
return p, fmt.Errorf("%s waits for nobody: it started at its merge", p.ID)
|
||||
case p.Delivery.Go != nil:
|
||||
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
|
||||
p.Delivery.Go.Local().Format("15:04:05"))
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
|
||||
p.Note = "let go by " + by + "; its first tier is asked next"
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
|
||||
// builds and registers; nothing further is asked or sent.
|
||||
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
if !p.Open() {
|
||||
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
if p.Delivery == nil {
|
||||
p.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// orderMember is one member of a group, as mesh-delivery says it.
|
||||
type orderMember struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
PathsTruncated bool `json:"paths_truncated,omitempty"`
|
||||
Removed []string `json:"removed,omitempty"`
|
||||
ModuleDirs []string `json:"module_dirs,omitempty"`
|
||||
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
|
||||
CloneURL string `json:"clone_url,omitempty"`
|
||||
// After are the repositories its pull request says it goes after (`after: <repository>`).
|
||||
After []string `json:"after,omitempty"`
|
||||
}
|
||||
|
||||
// pull is the member as the forge announced it.
|
||||
func (m orderMember) pull() link.PullUpdated {
|
||||
owner, repo, _ := strings.Cut(m.Repository, "/")
|
||||
base := m.Base
|
||||
if base == "" {
|
||||
base = "main"
|
||||
}
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head,
|
||||
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
||||
}
|
||||
|
||||
// orderPair is one "before" among a group's members, and why.
|
||||
type orderPair struct {
|
||||
Before string `json:"before"`
|
||||
After string `json:"after"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// orderReach is what a member moves, as the planner says.
|
||||
type orderReach struct {
|
||||
Moved []string `json:"moved,omitempty"`
|
||||
Dependents []string `json:"dependents,omitempty"`
|
||||
New []string `json:"new,omitempty"`
|
||||
Manifests []string `json:"manifests,omitempty"`
|
||||
}
|
||||
|
||||
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one.
|
||||
type groupOrder struct {
|
||||
Order []string `json:"order"`
|
||||
Pairs []orderPair `json:"pairs,omitempty"`
|
||||
Cycle []string `json:"cycle,omitempty"`
|
||||
Reach map[string]orderReach `json:"reach,omitempty"`
|
||||
}
|
||||
|
||||
// The reasons a pair is ordered, in the words the delivery plan shows.
|
||||
const (
|
||||
orderDeclared = "declared"
|
||||
orderBuiltBy = "built by"
|
||||
orderVersionSkew = "version skew"
|
||||
orderEngineFirst = "engine before controller"
|
||||
)
|
||||
|
||||
// orderOf is a group's order (novox/hq ADR 0239 decision 4), pure. A member goes before another when:
|
||||
//
|
||||
// - its pull request is named in the other's `after:` lines (declared);
|
||||
// - it moves a module the other's moved modules are built by, stand on, package or declare (built by);
|
||||
// - it moves the controller and the other changes any module's manifest (version skew: the newer
|
||||
// controller parses what the newer manifest says) — the node-engine's excepted, which goes first;
|
||||
// - it moves the node-engine and the other moves the controller (the witness reads nothing the controller
|
||||
// does not yet grant, and a controller sends nothing an older engine would refuse — ADR 0236's rollout
|
||||
// order).
|
||||
//
|
||||
// Otherwise, by repository then id, so every reading gives one order. A pair both ways is a cycle: the
|
||||
// members in it are named, and none of them is ordered.
|
||||
func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge) groupOrder {
|
||||
out := groupOrder{Reach: reach}
|
||||
byID := map[string]orderMember{}
|
||||
for _, m := range members {
|
||||
byID[m.ID] = m
|
||||
}
|
||||
add := func(before, after, why string) {
|
||||
if before == after {
|
||||
return
|
||||
}
|
||||
for _, p := range out.Pairs {
|
||||
if p.Before == before && p.After == after {
|
||||
return
|
||||
}
|
||||
}
|
||||
out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why})
|
||||
}
|
||||
named := func(said, repository string) bool {
|
||||
said = strings.TrimSuffix(strings.TrimSpace(said), ".git")
|
||||
if strings.EqualFold(said, repository) {
|
||||
return true
|
||||
}
|
||||
_, repo, _ := strings.Cut(repository, "/")
|
||||
return strings.EqualFold(said, repo)
|
||||
}
|
||||
for _, a := range members {
|
||||
for _, b := range members {
|
||||
if a.ID == b.ID {
|
||||
continue
|
||||
}
|
||||
ra, rb := reach[a.ID], reach[b.ID]
|
||||
for _, said := range b.After {
|
||||
if named(said, a.Repository) {
|
||||
add(a.ID, b.ID, orderDeclared)
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
if slices.Contains(ra.Moved, e.To) && slices.Contains(rb.Moved, e.From) && e.From != e.To {
|
||||
add(a.ID, b.ID, orderBuiltBy)
|
||||
break
|
||||
}
|
||||
}
|
||||
if slices.Contains(ra.Moved, "mesh-controller") && len(rb.Manifests) > 0 &&
|
||||
!slices.Contains(rb.Moved, "mesh-controller") && !slices.Contains(rb.Moved, "mesh-host") {
|
||||
add(a.ID, b.ID, orderVersionSkew)
|
||||
}
|
||||
if slices.Contains(ra.Moved, "mesh-host") && slices.Contains(rb.Moved, "mesh-controller") &&
|
||||
!slices.Contains(ra.Moved, "mesh-controller") {
|
||||
add(a.ID, b.ID, orderEngineFirst)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out.Pairs, func(i, j int) bool {
|
||||
if out.Pairs[i].Before != out.Pairs[j].Before {
|
||||
return out.Pairs[i].Before < out.Pairs[j].Before
|
||||
}
|
||||
return out.Pairs[i].After < out.Pairs[j].After
|
||||
})
|
||||
// Kahn's walk, the next always the first by repository and id among those with nothing before them.
|
||||
waiting := map[string]int{}
|
||||
for _, m := range members {
|
||||
waiting[m.ID] = 0
|
||||
}
|
||||
for _, p := range out.Pairs {
|
||||
waiting[p.After]++
|
||||
}
|
||||
done := map[string]bool{}
|
||||
for len(done) < len(members) {
|
||||
var ready []orderMember
|
||||
for _, m := range members {
|
||||
if !done[m.ID] && waiting[m.ID] == 0 {
|
||||
ready = append(ready, m)
|
||||
}
|
||||
}
|
||||
if len(ready) == 0 {
|
||||
for _, m := range members {
|
||||
if !done[m.ID] {
|
||||
out.Cycle = append(out.Cycle, m.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out.Cycle)
|
||||
return out
|
||||
}
|
||||
sort.Slice(ready, func(i, j int) bool {
|
||||
if ready[i].Repository != ready[j].Repository {
|
||||
return ready[i].Repository < ready[j].Repository
|
||||
}
|
||||
return ready[i].ID < ready[j].ID
|
||||
})
|
||||
next := ready[0]
|
||||
done[next.ID] = true
|
||||
out.Order = append(out.Order, next.ID)
|
||||
for _, p := range out.Pairs {
|
||||
if p.Before == next.ID {
|
||||
waiting[p.After]--
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reachOfMembers is each member's reach, as the planner says it.
|
||||
func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) map[string]orderReach {
|
||||
out := map[string]orderReach{}
|
||||
for _, m := range members {
|
||||
s := pullScope(m.pull(), entries, read, edges)
|
||||
out[m.ID] = orderReach{Moved: s.Modules, Dependents: s.Dependents, New: s.New, Manifests: s.Manifests}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner:
|
||||
//
|
||||
// delivery plan --repository owner/repo --head <commit> [--base main] --paths a,b [--module-dirs …] [--removed …]
|
||||
// delivery order --members <json>
|
||||
// delivery check --group <id> --members <json>
|
||||
// delivery go <plan> [--by <who>] [--why <text>]
|
||||
// delivery stop <plan> --why <text> [--by <who>]
|
||||
// delivery walks [-n 50] [--plan <id>]
|
||||
func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("delivery plan|order|check|go|stop|walks")
|
||||
}
|
||||
sub, rest := args[0], args[1:]
|
||||
set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError)
|
||||
repository := set.String("repository", "", "owner/repository")
|
||||
base := set.String("base", "main", "the branch it merges into")
|
||||
head := set.String("head", "", "the commit at hand")
|
||||
paths := set.String("paths", "", "the files it changes, comma-separated")
|
||||
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
|
||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
by := set.String("by", catalogue.DeliverySeat, "who says it")
|
||||
why := set.String("why", "", "why")
|
||||
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
|
||||
planID := set.String("plan", "", "one walk")
|
||||
positionals, err := parseAround(set, rest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var members []orderMember
|
||||
if *membersJSON != "" {
|
||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||
return fmt.Errorf("the members are not readable: %w", err)
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
answer := func(v any) error {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(v)
|
||||
}
|
||||
switch sub {
|
||||
case "plan":
|
||||
if *repository == "" || *head == "" && *paths == "" {
|
||||
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
|
||||
}
|
||||
m := orderMember{ID: *repository + "@" + *head, Repository: *repository, Base: *base, Head: *head,
|
||||
Paths: splitList(*paths), Removed: splitList(*removed)}
|
||||
if d := moduleDirsOf(*moduleDirs); d != nil {
|
||||
m.ModuleDirs, m.ModuleDirsSaid = *d, true
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s := pullScope(m.pull(), entries, read, edges)
|
||||
plan := changePlanOf(*repository, *base, *head, s.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
return answer(map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
|
||||
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()})
|
||||
case "order":
|
||||
if len(members) == 0 {
|
||||
return errors.New("delivery order --members <json>: a group has members")
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges))
|
||||
case "check":
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
// own path for a pull request, run because the delivery's owner asked.
|
||||
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"rechecked": members[0].ID})
|
||||
}
|
||||
if *group == "" || len(members) < 2 {
|
||||
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
|
||||
}
|
||||
id, err := askGroupCheck(ctx, open, *group, members)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"asked": id, "group": *group})
|
||||
case "go":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
|
||||
short(p.Commit), *by)
|
||||
return nil
|
||||
case "stop":
|
||||
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
|
||||
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
|
||||
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
|
||||
return nil
|
||||
case "walks":
|
||||
var walks []inventory.Plan
|
||||
if *planID != "" {
|
||||
p, err := inv.PlanByID(ctx, *planID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
walks = []inventory.Plan{p}
|
||||
} else {
|
||||
if walks, err = inv.OpenPlans(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, *limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if !slices.ContainsFunc(walks, func(w inventory.Plan) bool { return w.ID == p.ID }) {
|
||||
walks = append(walks, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||
"own-path": sortedKeysOf(ownPathWords())})
|
||||
}
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||
}
|
||||
|
||||
// ownPathWords is the controller's own path as words, for an answer.
|
||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||
|
||||
// theGraph is what the planner reads.
|
||||
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
|
||||
[]inventory.Edge, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return entries, read, edges, nil
|
||||
}
|
||||
|
||||
// groupPrimary is the head a group's composed check is run from: the one moving the controller, which then
|
||||
// judges the group by itself; else the one moving the node-engine, whose validator judges; else the first.
|
||||
func groupPrimary(members []orderMember, reach map[string]orderReach) int {
|
||||
for _, want := range []string{"mesh-controller", "mesh-host"} {
|
||||
for i, m := range members {
|
||||
if slices.Contains(reach[m.ID].Moved, want) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// askGroupCheck asks the build seat for a group's composed check: every head laid over the mesh in turn,
|
||||
// judged as one future state (novox/hq ADR 0239). The verdict comes back as `checked` with the group's id,
|
||||
// for mesh-delivery; the forge's holder sets no status from it.
|
||||
func askGroupCheck(ctx context.Context, open *stores, group string, members []orderMember) (string, error) {
|
||||
entries, read, edges, err := theGraph(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
reach := reachOfMembers(members, entries, read, edges)
|
||||
primary := groupPrimary(members, reach)
|
||||
p := members[primary].pull()
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// The gate runs over what any member moves; a judge from the primary alone.
|
||||
for _, m := range members {
|
||||
r := reach[m.ID]
|
||||
scope.Modules = appendNew(scope.Modules, r.Moved...)
|
||||
scope.Dependents = appendNew(scope.Dependents, r.Dependents...)
|
||||
}
|
||||
request, err := checkRequestFor(ctx, open, p, scope, entries)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Check.Group = group
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for i, m := range members {
|
||||
if i == primary {
|
||||
continue
|
||||
}
|
||||
mp := m.pull()
|
||||
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
|
||||
source := inventory.Source{Seat: gitSeat, Repository: mp.Owner + "/" + mp.Repo}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && sameRepository(e.Source.Repository, link.SourceMoved{Owner: mp.Owner, Repo: mp.Repo}) {
|
||||
source = e.Source
|
||||
break
|
||||
}
|
||||
}
|
||||
url := source.Repository
|
||||
if source.Seat != "" {
|
||||
url, err = clonedFromSeat(world, source.Seat, source.Repository)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
|
||||
}
|
||||
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths})
|
||||
}
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "group %s: asked %s to check %d head(s) composed together, as %s\n", group, seat,
|
||||
len(members), request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// appendNew appends what is not there yet.
|
||||
func appendNew(to []string, items ...string) []string {
|
||||
for _, i := range items {
|
||||
if !slices.Contains(to, i) {
|
||||
to = append(to, i)
|
||||
}
|
||||
}
|
||||
return to
|
||||
}
|
||||
|
||||
// shelfOf is the catalogue's manifests by name.
|
||||
func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
return shelf
|
||||
}
|
||||
|
||||
// sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the
|
||||
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
||||
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
||||
// comparison. Records arriving out of order are told apart by the plan's revision.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, p)
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, body); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// sayingOnTheBus runs a command that keeps walks or says verdicts with the bus to say them on: the serving
|
||||
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
|
||||
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
|
||||
// still runs; what it did is found by comparison.
|
||||
func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
if checkEvents != nil {
|
||||
return f()
|
||||
}
|
||||
ran := false
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bus := link.OverNATS{Conn: conn, JS: js}
|
||||
checkEvents = bus
|
||||
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
|
||||
ran = true
|
||||
return f()
|
||||
})
|
||||
if !ran {
|
||||
fmt.Fprintf(os.Stderr, "the bus cannot be reached (%v): what this does is not said, and is found by comparison\n", err)
|
||||
return f()
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table
|
||||
// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the
|
||||
// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery.<id>.stalled`; healer
|
||||
// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only
|
||||
// the next observation says whether it worked (ADR 0231).
|
||||
|
||||
// probeDeliveriesID is the probe that reads the delivery's owner.
|
||||
const probeDeliveriesID = "D14"
|
||||
|
||||
// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope.
|
||||
const kindDeliveryStalled = "stalled"
|
||||
|
||||
// deliveryOwnerWithin is how long the owner is given to answer.
|
||||
var deliveryOwnerWithin = 10 * time.Second
|
||||
|
||||
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
|
||||
type stalledLine struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||
|
||||
// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted;
|
||||
// a seam a test replaces. The holder's own refusal is an error naming it.
|
||||
var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
||||
granted = granted || v.Verb == verb
|
||||
}
|
||||
if !granted {
|
||||
return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
if conn == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error)
|
||||
}
|
||||
return unwrapToolResult(answer.Result), nil
|
||||
}
|
||||
|
||||
// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the
|
||||
// protocol's content list holding it as text.
|
||||
func unwrapToolResult(raw json.RawMessage) json.RawMessage {
|
||||
var wrapped struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) {
|
||||
return json.RawMessage(wrapped.Content[0].Text)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record,
|
||||
// or none answers (D3 says that one).
|
||||
func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) {
|
||||
if !held {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{})
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return nil, nil // the holder not running is D3's holder-silent, said once there
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lines []stalledLine
|
||||
if err := json.Unmarshal(raw, &lines); err != nil {
|
||||
return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// stalledObservations are the conditions of what is stalled.
|
||||
func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
|
||||
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
|
||||
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
|
||||
if l.operatorsOnly() {
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeDeliveries is D14: every delivery held past its state's bound is said.
|
||||
func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var conn *nats.Conn
|
||||
if d.js != nil {
|
||||
conn = d.js.Conn()
|
||||
}
|
||||
lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stalledObservations(lines), nil
|
||||
}
|
||||
|
||||
// --- H2, for a delivery ---------------------------------------------------------------------------------
|
||||
|
||||
// connOf is the bus a healer asks over.
|
||||
func (h *healing) connOf() *nats.Conn {
|
||||
if h.js == nil {
|
||||
return nil
|
||||
}
|
||||
return h.js.Conn()
|
||||
}
|
||||
|
||||
// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table
|
||||
// lets H2 take; never one whose state is the operator's.
|
||||
func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
lines, err := deliveriesStalled(ctx, h.connOf(), true)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
for _, l := range lines {
|
||||
if l.ID != c.Subject.ID {
|
||||
continue
|
||||
}
|
||||
if l.operatorsOnly() {
|
||||
return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil
|
||||
}
|
||||
|
||||
// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the
|
||||
// transition its table names. A refusal is no repair, said; the next observation says whether it worked.
|
||||
func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key})
|
||||
if err != nil {
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return "", "", err
|
||||
}
|
||||
return "closed nothing", oneLine(err.Error()), nil
|
||||
}
|
||||
var said string
|
||||
if json.Unmarshal(raw, &said) != nil {
|
||||
said = string(raw)
|
||||
}
|
||||
return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's
|
||||
// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`.
|
||||
|
||||
// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records
|
||||
// what close was asked.
|
||||
func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string {
|
||||
t.Helper()
|
||||
var closed []string
|
||||
was := askDeliveryOwner
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
switch verb {
|
||||
case "stalled":
|
||||
raw, _ := json.Marshal(lines)
|
||||
return raw, nil
|
||||
case "close":
|
||||
closed = append(closed, args["id"].(string))
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil
|
||||
}
|
||||
t.Fatalf("asked the owner %s", verb)
|
||||
return nil, nil
|
||||
}
|
||||
t.Cleanup(func() { askDeliveryOwner = was })
|
||||
return &closed
|
||||
}
|
||||
|
||||
func holdTheDeliverySeat(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
m := catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var twoStalled = []stalledLine{
|
||||
{ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s",
|
||||
H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"},
|
||||
{ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s",
|
||||
H2: "none: the state is the operator's", Says: "it waits for the operator"},
|
||||
}
|
||||
|
||||
func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
ownerAnswers(t, twoStalled, nil)
|
||||
d := &doctor{open: open}
|
||||
got, err := probeDeliveries(ctx, d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("with no holder on record D14 said %+v %v", got, err)
|
||||
}
|
||||
holdTheDeliverySeat(t, open)
|
||||
got, err = probeDeliveries(ctx, d)
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("D14 said %+v %v", got, err)
|
||||
}
|
||||
if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning ||
|
||||
got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") {
|
||||
t.Fatalf("the first is %+v", got[0])
|
||||
}
|
||||
if got[1].Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("a held delivery is not the operator's: %+v", got[1])
|
||||
}
|
||||
// The holder not running is D3's to say, not D14's.
|
||||
was := askDeliveryOwner
|
||||
askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) {
|
||||
return nil, link.ErrNothingServes
|
||||
}
|
||||
defer func() { askDeliveryOwner = was }()
|
||||
if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 {
|
||||
t.Fatalf("an owner that is down was said by D14: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a
|
||||
// refusal is no repair.
|
||||
func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
closed := ownerAnswers(t, twoStalled, nil)
|
||||
for _, o := range stalledObservations(twoStalled) {
|
||||
o.Source = probeDeliveriesID
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) {
|
||||
t.Fatalf("close was asked for %v", *closed)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// A refusal: closed nothing, said so.
|
||||
c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled")
|
||||
ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering"))
|
||||
act, said, err := repairDelivery(ctx, h, c)
|
||||
if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") {
|
||||
t.Fatalf("a refused close reads %q %q %v", act, said, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else
|
||||
// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read.
|
||||
func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
|
||||
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, verb := range []string{"stalled", "close"} {
|
||||
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
|
||||
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
}
|
||||
if _, err := askDeliveryOwner(t.Context(), nil, "stop", nil); err == nil || !strings.Contains(err.Error(), "grant") {
|
||||
t.Fatalf("a verb the grant does not name was asked: %v", err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := deliveriesStalled(t.Context(), conn, true); err != nil {
|
||||
t.Fatalf("an owner not running is D3's, not an error: %v", err)
|
||||
}
|
||||
lines, _ := json.Marshal(twoStalled)
|
||||
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}})
|
||||
sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) {
|
||||
body, _ := json.Marshal(link.Answer{Result: wrapped})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
got, err := deliveriesStalled(ctx, conn, true)
|
||||
if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" {
|
||||
t.Fatalf("over the bus: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
|
||||
type recordingDelivery struct {
|
||||
did []string
|
||||
grantErr error
|
||||
declareErr error
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
|
||||
for _, s := range sending {
|
||||
r.did = append(r.did, "grant "+s.node)
|
||||
}
|
||||
return r.grantErr
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
|
||||
if r.declareErr != nil {
|
||||
return "", r.declareErr
|
||||
}
|
||||
r.did = append(r.did, "declare "+s.node)
|
||||
return "digest-" + s.node, nil
|
||||
}
|
||||
|
||||
func ready(names ...string) []readyNode {
|
||||
var out []readyNode
|
||||
for _, n := range names {
|
||||
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
|
||||
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
|
||||
// carries the controller's own right to issue them, and goes first.
|
||||
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
|
||||
d := &recordingDelivery{}
|
||||
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
|
||||
if !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
|
||||
}
|
||||
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
|
||||
t.Fatalf("the digests sent were not answered: %v", digests)
|
||||
}
|
||||
|
||||
held := &recordingDelivery{}
|
||||
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
|
||||
if !reflect.DeepEqual(held.did, want) {
|
||||
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
|
||||
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
|
||||
// grant that would let the controller issue memberships is never held behind them.
|
||||
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
|
||||
// A failure naming no machine (the buckets): everything but the bus's machine.
|
||||
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
|
||||
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
|
||||
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
|
||||
!strings.Contains(err.Error(), "anchor, laptop not sent") {
|
||||
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
|
||||
}
|
||||
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
|
||||
}
|
||||
|
||||
// A membership that failed for one machine: that machine alone.
|
||||
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
|
||||
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
|
||||
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
|
||||
t.Fatalf("one machine's refused membership was not said: %v", err)
|
||||
}
|
||||
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
|
||||
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
|
||||
}
|
||||
|
||||
// The announced upgrade that hit it is asked again.
|
||||
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
|
||||
t.Fatal("an announcement whose send stopped at its grants is not asked again")
|
||||
}
|
||||
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
|
||||
t.Fatal("any failure is asked again, not only a grant's")
|
||||
}
|
||||
|
||||
// Nothing to send is nothing granted either.
|
||||
none := &recordingDelivery{grantErr: errors.New("never asked")}
|
||||
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
|
||||
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Whether the bus's machine goes first is read from the user list alone, by its digest.
|
||||
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
list := "users: [a, b]"
|
||||
if userListBehind(list, digestOf([]byte(list))) {
|
||||
t.Fatal("the list it was sent reads as behind")
|
||||
}
|
||||
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
|
||||
t.Fatal("a changed or never-sent list reads as current")
|
||||
}
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
// checked against. Among the machines it is moved to the front; not among them it is added only
|
||||
// when it is behind.
|
||||
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
names []string
|
||||
holder string
|
||||
behind bool
|
||||
want []string
|
||||
}{
|
||||
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
|
||||
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
|
||||
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
|
||||
} {
|
||||
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every
|
||||
// reading — the build agent before what it builds, the controller before a manifest that needs it, the
|
||||
// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered.
|
||||
func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "cat", Repository: "novox/mesh-catalog"},
|
||||
{ID: "ctl", Repository: "novox/mesh-controller"},
|
||||
{ID: "host", Repository: "novox/mesh-host"},
|
||||
{ID: "agent", Repository: "novox/build-agent"},
|
||||
{ID: "app", Repository: "novox/app", After: []string{"lab"}},
|
||||
{ID: "lab", Repository: "novox/lab"},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
|
||||
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
|
||||
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
|
||||
"agent": {Moved: []string{"build-agent"}},
|
||||
"app": {Moved: []string{"app"}},
|
||||
"lab": {Moved: []string{"lab"}},
|
||||
}
|
||||
edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
|
||||
got := orderOf(members, reach, edges)
|
||||
if len(got.Cycle) > 0 {
|
||||
t.Fatalf("a cycle where there is none: %v", got.Cycle)
|
||||
}
|
||||
// By repository among those with nothing before them: the build agent, the lab, then what waited on both.
|
||||
want := []string{"agent", "lab", "app", "host", "ctl", "cat"}
|
||||
if !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
|
||||
}
|
||||
why := map[string]string{}
|
||||
for _, p := range got.Pairs {
|
||||
why[p.Before+">"+p.After] = p.Why
|
||||
}
|
||||
for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew,
|
||||
"ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} {
|
||||
if why[pair] != reason {
|
||||
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
|
||||
}
|
||||
}
|
||||
// The same members in another order read the same.
|
||||
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
|
||||
if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) {
|
||||
t.Fatalf("another reading ordered %v", again.Order)
|
||||
}
|
||||
|
||||
// A declared order against an inferred one is a cycle: named, and nothing ordered.
|
||||
members[1].After = []string{"mesh-catalog"}
|
||||
got = orderOf(members, reach, edges)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) {
|
||||
t.Fatalf("the cycle is %v, ordered %v", got.Cycle, got.Order)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
|
||||
// record, and never one that moves a module on the controller's own path.
|
||||
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
|
||||
holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
|
||||
unassigned := holder
|
||||
unassigned.On = nil
|
||||
for _, c := range []struct {
|
||||
entries []inventory.Entry
|
||||
moved []string
|
||||
waits bool
|
||||
}{
|
||||
{nil, []string{"gitea"}, false},
|
||||
{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
|
||||
{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"node-tools"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"nats"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
|
||||
} {
|
||||
d := awaitsFor(c.entries, c.moved)
|
||||
if (d != nil) != c.waits {
|
||||
t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
|
||||
c.moved, d != nil, c.waits)
|
||||
}
|
||||
if d != nil && d.Awaits != catalogue.DeliverySeat {
|
||||
t.Errorf("waits for %q", d.Awaits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
|
||||
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
|
||||
// hand when the owner is down, and the owner's stop ends one as stopped.
|
||||
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
withConditionsInMemory(t)
|
||||
for _, name := range []string{"app", "mesh-delivery"} {
|
||||
m := catalogue.Manifest{Module: name, Version: "1"}
|
||||
if name == "mesh-delivery" {
|
||||
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
merge := func(commit string, paths ...string) inventory.Plan {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
|
||||
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 1)
|
||||
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
|
||||
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
|
||||
}
|
||||
return recent[0]
|
||||
}
|
||||
|
||||
// finish ends a walk as done, so the next merge has nothing of it to take over.
|
||||
finish := func(id string) {
|
||||
t.Helper()
|
||||
w, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.State = inventory.PlanDone
|
||||
if err := inv.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No holder on record: the merge starts its walk, as before.
|
||||
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
||||
if p.Waiting() || len(*asked) != 1 {
|
||||
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either.
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p = merge("c2bbbbbbbb", "modules/app/index.ts")
|
||||
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
|
||||
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
|
||||
}
|
||||
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
|
||||
t.Errorf("a waiting walk reads %q", line)
|
||||
}
|
||||
|
||||
// The delivery's word starts it.
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
||||
t.Fatal("a walk was let go twice")
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the word did not start the walk: %v", *asked)
|
||||
}
|
||||
got, err := inv.PlanByID(ctx, p.ID)
|
||||
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
|
||||
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
||||
}
|
||||
|
||||
// The delivery's owner's own merge never waits for it — and takes over what the older walk had not
|
||||
// built (app, folded in: ADR 0218), which goes with it on the controller's own path.
|
||||
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
||||
if p.Waiting() || len(*asked) != 4 {
|
||||
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The owner down: a person starts a waiting walk, with why.
|
||||
finish(p.ID)
|
||||
p = merge("c4dddddddd", "modules/app/index.ts")
|
||||
if !p.Waiting() {
|
||||
t.Fatal("the walk did not wait")
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a walk was started by hand without why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 5 {
|
||||
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
||||
}
|
||||
|
||||
// The owner's stop: failed, said as stopped by it.
|
||||
finish(p.ID)
|
||||
p = merge("c5eeeeeeee", "modules/app/index.ts")
|
||||
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
|
||||
"mesh-delivery for jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
|
||||
!strings.Contains(got.Note, "the operator stopped it") {
|
||||
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
|
||||
}
|
||||
|
||||
// Unassigned: the mesh is back on the controller's own path.
|
||||
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asks := len(*asked)
|
||||
if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
|
||||
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
|
||||
}
|
||||
}
|
||||
|
||||
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
|
||||
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
|
||||
[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
|
||||
{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
|
||||
[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
|
||||
{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
|
||||
{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
|
||||
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
|
||||
{"delivery-check", map[string]any{"group": "feat/x", "members": "[]"},
|
||||
[]string{"delivery", "check", "--group", "feat/x", "--members", "[]"}},
|
||||
{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"},
|
||||
[]string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}},
|
||||
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
|
||||
} {
|
||||
got, err := argvFor(c.verb, c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("deliver", map[string]any{}); err == nil {
|
||||
t.Error("deliver without a walk was composed")
|
||||
}
|
||||
if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
|
||||
t.Error("a stop without why was composed")
|
||||
}
|
||||
}
|
||||
|
||||
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
|
||||
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
|
||||
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
|
||||
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
var said inventory.Plan
|
||||
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
|
||||
said.Delivery.Go == nil {
|
||||
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a walk let go by a verb was not said")
|
||||
}
|
||||
if checkEvents != nil || inventory.PlanSaved != nil {
|
||||
t.Fatal("the command's own bus was left in place")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A merge that rebuilds a base rebuilds what stands on it, through every layer, and nothing else
|
||||
// (novox/hq issue 186): the runtime image moving means every module built on it moves too, and a
|
||||
// module built on one of those moves as well.
|
||||
func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) {
|
||||
entry := func(name string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: name}}
|
||||
}
|
||||
entries := []inventory.Entry{entry("mesh-tools"), entry("shop"), entry("shop-plugin"), entry("postgres"), entry("unrelated")}
|
||||
against := map[string][]string{
|
||||
"shop": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
|
||||
"shop-plugin": {catalogue.ArtifactStoreScheme + "shop/runtime@sha256:b"},
|
||||
"postgres": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
|
||||
"unrelated": {catalogue.ArtifactStoreScheme + "alpine/base@sha256:c"},
|
||||
}
|
||||
got := dependentsOf([]inventory.Entry{entry("mesh-tools")}, entries, against)
|
||||
var names []string
|
||||
for _, e := range got {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
want := map[string]bool{"shop": true, "shop-plugin": true, "postgres": true}
|
||||
if len(names) != len(want) {
|
||||
t.Fatalf("rebuilt %v; wanted exactly the three that stand on the runtime, directly or through shop", names)
|
||||
}
|
||||
for _, n := range names {
|
||||
if !want[n] {
|
||||
t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names)
|
||||
}
|
||||
}
|
||||
// The dependents come in base order when the merge orders them: the runtime, then shop, then
|
||||
// the plugin that stands on shop.
|
||||
ordered := orderByBases(append([]inventory.Entry{entry("mesh-tools")}, got...), against)
|
||||
pos := map[string]int{}
|
||||
for i, e := range ordered {
|
||||
pos[e.Manifest.Module] = i
|
||||
}
|
||||
if !(pos["mesh-tools"] < pos["shop"] && pos["shop"] < pos["shop-plugin"]) {
|
||||
t.Fatalf("not in base order: %v", ordered)
|
||||
}
|
||||
// Nothing moved: nothing follows.
|
||||
if more := dependentsOf(nil, entries, against); len(more) != 0 {
|
||||
t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
|
||||
//
|
||||
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
|
||||
// design — every machine's declaration composes and validates, every resolver answers, every seat's
|
||||
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
|
||||
// condition it raises when the invariant does not hold. The serving controller runs the registry every
|
||||
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
|
||||
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
|
||||
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
|
||||
// machine: a controller that stops checking is itself said, through a channel that does not pass
|
||||
// through it.
|
||||
//
|
||||
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
|
||||
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
|
||||
|
||||
// The self-check's clocks.
|
||||
var (
|
||||
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
|
||||
// starts the first run waits, so what the controller hears at its start has arrived.
|
||||
doctorEvery = 5 * time.Minute
|
||||
doctorFirstAfter = time.Minute
|
||||
// probeWithin is each probe's bound.
|
||||
probeWithin = 30 * time.Second
|
||||
)
|
||||
|
||||
// The verdicts a probe can have.
|
||||
const (
|
||||
verdictPass = "pass"
|
||||
verdictFail = "fail"
|
||||
verdictFailedToRun = "failed-to-run"
|
||||
verdictDeferred = "deferred"
|
||||
)
|
||||
|
||||
// probe is one live invariant.
|
||||
type probe struct {
|
||||
ID string
|
||||
Asserts string
|
||||
From string
|
||||
// Kind is the condition kind raised when the invariant does not hold.
|
||||
Kind string
|
||||
// Raises are the other kinds its findings carry, each its own (a consumer missing, one far behind):
|
||||
// what a healer may be registered against (healers.go).
|
||||
Raises []string
|
||||
Phase int
|
||||
// Deferred says why it is not run yet; empty for one that is.
|
||||
Deferred string
|
||||
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
|
||||
// controller's grant names every one (a test over this registry): a probe whose question the bus
|
||||
// refuses checks nothing (D8, 2026-10-06).
|
||||
Asks []broker.SeatVerb
|
||||
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||
}
|
||||
|
||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
|
||||
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
|
||||
Kind: "archives-unheld", Phase: 1, run: probeArchives},
|
||||
{ID: "D5", Asserts: "exactly one lease holder — the key names this controller at its epoch, and the record " +
|
||||
"holds no other epoch open; no message from a stale epoch refused in the last interval",
|
||||
From: "issue 204", Kind: "lease-split", Phase: 2, run: probeLease},
|
||||
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Raises: []string{"consumer-lost", kindConsumerBehind},
|
||||
Phase: 1, run: probeConsumers},
|
||||
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
|
||||
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
|
||||
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
|
||||
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
|
||||
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
|
||||
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
|
||||
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
|
||||
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding " +
|
||||
"its cleanup", From: "ADR 0230", Kind: kindCleanupWaiting, Phase: 2, run: probeRetired},
|
||||
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
|
||||
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
|
||||
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
|
||||
{ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " +
|
||||
"written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " +
|
||||
"empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " +
|
||||
"valuable is retired, not forgotten", From: "issue 273, ADR 0233",
|
||||
Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale,
|
||||
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
|
||||
Phase: 2, run: probeData,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
|
||||
// The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by
|
||||
// the controller, and H2 works it through the owner's `close`.
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
// judged by on its first machine, run against every machine between upgrades too.
|
||||
{ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " +
|
||||
"ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth},
|
||||
{ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " +
|
||||
"build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth},
|
||||
{ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus",
|
||||
From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth},
|
||||
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " +
|
||||
"it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4,
|
||||
run: probeBusHealth},
|
||||
// The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its
|
||||
// condition until a newer build passes; each rollback a witness stands by is said.
|
||||
{ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " +
|
||||
"a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack,
|
||||
Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates},
|
||||
// The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus.
|
||||
{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
|
||||
"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
type probeVerdict struct {
|
||||
ID string `json:"id"`
|
||||
Verdict string `json:"verdict"`
|
||||
Found []string `json:"found,omitempty"`
|
||||
// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before:
|
||||
// raised if the next run sees them too (confirm.go). Not a pass, and not yet a condition.
|
||||
Unconfirmed []string `json:"unconfirmed,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
}
|
||||
|
||||
// doctorCounts are a run's verdicts, counted.
|
||||
type doctorCounts struct {
|
||||
Passed int `json:"passed"`
|
||||
Failed int `json:"failed"`
|
||||
FailedToRun int `json:"failed-to-run"`
|
||||
Deferred int `json:"deferred"`
|
||||
}
|
||||
|
||||
// doctorRun is one run of the registry, and the body of its heartbeat.
|
||||
type doctorRun struct {
|
||||
Run string `json:"run"`
|
||||
At time.Time `json:"at"`
|
||||
Started time.Time `json:"started"`
|
||||
Took string `json:"took"`
|
||||
IntervalSeconds int `json:"interval-seconds"`
|
||||
Counts doctorCounts `json:"counts"`
|
||||
Probes []probeVerdict `json:"probes"`
|
||||
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
|
||||
Controller string `json:"controller"`
|
||||
Why string `json:"why"`
|
||||
// Unsaid is how many condition transitions this controller could not say, since it started.
|
||||
Unsaid int `json:"unsaid,omitempty"`
|
||||
}
|
||||
|
||||
// doctor is the registry and what its probes need.
|
||||
type doctor struct {
|
||||
open *stores
|
||||
js *broker.JetStream
|
||||
keeper *conditions.Keeper
|
||||
teller conditions.Teller
|
||||
watchdogs *watchdogs
|
||||
host string
|
||||
// confirm holds back what one run alone saw of a finding a single look can be wrong about.
|
||||
confirm confirming
|
||||
|
||||
running sync.Mutex
|
||||
mu sync.Mutex
|
||||
last *doctorRun
|
||||
ended time.Time
|
||||
}
|
||||
|
||||
// lastRunEnded is when the last run ended; zero before the first.
|
||||
func (d *doctor) lastRunEnded() time.Time {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.ended
|
||||
}
|
||||
|
||||
// lastRun is the last run's verdict; nil before the first.
|
||||
func (d *doctor) lastRun() *doctorRun {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.last
|
||||
}
|
||||
|
||||
// keep runs the registry on its schedule until ctx ends.
|
||||
func (d *doctor) keep(ctx context.Context) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(doctorFirstAfter):
|
||||
}
|
||||
tick := time.NewTicker(doctorEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
// Only the controller acting checks the mesh on a schedule: one standing by would say a
|
||||
// heartbeat for a self-check that is not the mesh's.
|
||||
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
|
||||
d.runOnce(ctx, "the schedule")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var doctorRuns struct {
|
||||
sync.Mutex
|
||||
n uint64
|
||||
}
|
||||
|
||||
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
|
||||
// at a time: a person's `doctor run` during a scheduled one waits for it.
|
||||
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||
d.running.Lock()
|
||||
defer d.running.Unlock()
|
||||
doctorRuns.Lock()
|
||||
doctorRuns.n++
|
||||
n := doctorRuns.n
|
||||
doctorRuns.Unlock()
|
||||
started := time.Now()
|
||||
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
|
||||
|
||||
type result struct {
|
||||
obs []conditions.Observation
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
results := make([]result, len(probeRegistry))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range probeRegistry {
|
||||
if p.run == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int, p probe) {
|
||||
defer wg.Done()
|
||||
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
|
||||
defer cancel()
|
||||
began := time.Now()
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||
}
|
||||
}()
|
||||
obs, err := p.run(probing, d)
|
||||
done <- result{obs: obs, err: err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
r.took = time.Since(began)
|
||||
results[i] = r
|
||||
case <-probing.Done():
|
||||
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
|
||||
}
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var blind []conditions.Observation
|
||||
for i, p := range probeRegistry {
|
||||
v := probeVerdict{ID: p.ID}
|
||||
r := results[i]
|
||||
switch {
|
||||
case p.run == nil:
|
||||
v.Verdict = verdictDeferred
|
||||
run.Counts.Deferred++
|
||||
case r.err != nil:
|
||||
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
|
||||
run.Counts.FailedToRun++
|
||||
// A probe that could not run once — a question timed out on a loaded machine — is said in
|
||||
// the verdict at once, and raised as a condition when the next run cannot run it either.
|
||||
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
|
||||
Token: "failed", Severity: conditions.Warning, Confirm: true,
|
||||
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||
Said: firstLine(r.err.Error())})
|
||||
default:
|
||||
raise, held := d.confirm.pass(ctx, d.keeper, p.ID, kindedAs(r.obs, p.Kind))
|
||||
if err := d.keeper.Reconcile(ctx, p.ID, raise); err != nil {
|
||||
v.Error = "what it found could not be kept: " + err.Error()
|
||||
}
|
||||
for _, o := range held {
|
||||
v.Unconfirmed = append(v.Unconfirmed, o.Summary)
|
||||
}
|
||||
if len(raise) == 0 {
|
||||
v.Verdict = verdictPass
|
||||
run.Counts.Passed++
|
||||
} else {
|
||||
v.Verdict = verdictFail
|
||||
run.Counts.Failed++
|
||||
for _, o := range raise {
|
||||
v.Found = append(v.Found, o.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.run != nil {
|
||||
v.Took = r.took.Round(time.Millisecond).String()
|
||||
}
|
||||
run.Probes = append(run.Probes, v)
|
||||
}
|
||||
blind, _ = d.confirm.pass(ctx, d.keeper, sourceDoctor, blind)
|
||||
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
|
||||
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
|
||||
}
|
||||
ended := time.Now()
|
||||
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
|
||||
d.mu.Lock()
|
||||
d.last, d.ended = &run, ended
|
||||
d.mu.Unlock()
|
||||
d.sayHeartbeat(ctx, run)
|
||||
return run
|
||||
}
|
||||
|
||||
// sourceDoctor is what raises a probe's own failure to run.
|
||||
const sourceDoctor = "doctor"
|
||||
|
||||
// kindedAs gives each observation of a probe the probe's kind where it named none.
|
||||
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(obs))
|
||||
for _, o := range obs {
|
||||
if o.Kind == "" {
|
||||
o.Kind = kind
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
|
||||
// says it outward: the watcher hears nothing.
|
||||
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
|
||||
if d.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
|
||||
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
|
||||
"will say the self-check is silent: %v\n", run.Run, err)
|
||||
}
|
||||
}
|
||||
|
||||
// doctorFrom is the serving controller's self-check; nil in any other process.
|
||||
var doctorFrom *doctor
|
||||
|
||||
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
|
||||
func doctorCommand(ctx context.Context, args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("doctor [run|probes|signals] [--json]")
|
||||
}
|
||||
answer, err := doctorAnswer(ctx, sub)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(answer)
|
||||
}
|
||||
fmt.Print(doctorText(answer))
|
||||
return nil
|
||||
}
|
||||
|
||||
// doctorAnswer is what the verb answers, as data.
|
||||
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
switch sub {
|
||||
case "":
|
||||
if doctorFrom != nil {
|
||||
if run := doctorFrom.lastRun(); run != nil {
|
||||
return verdictAnswer(*run, time.Now()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||
}
|
||||
run, err := lastHeartbeat(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return verdictAnswer(run, time.Now()), nil
|
||||
case "run":
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
local, closeIt, err := localDoctor(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeIt()
|
||||
d = local
|
||||
}
|
||||
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||
case "probes":
|
||||
return probesAnswer(), nil
|
||||
case "signals":
|
||||
if doctorFrom == nil || doctorFrom.watchdogs == nil {
|
||||
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
|
||||
"hears them: ask it through the mesh-controller seat's doctor verb")
|
||||
}
|
||||
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
|
||||
}
|
||||
|
||||
// verdictAnswer is a run as the verb answers it, with its age.
|
||||
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it, " +
|
||||
"and one a single look can be wrong about — an unanswered question, a slow answer — is raised when two " +
|
||||
"runs in a row see it"}
|
||||
}
|
||||
|
||||
// probesAnswer is the registry.
|
||||
func probesAnswer() map[string]any {
|
||||
var out []map[string]any
|
||||
for _, p := range probeRegistry {
|
||||
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
|
||||
if len(p.Raises) > 0 {
|
||||
row["raises"] = p.Raises
|
||||
}
|
||||
if p.Deferred != "" {
|
||||
row["deferred"] = p.Deferred
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
|
||||
}
|
||||
|
||||
// signalsAnswer is every row of the signals table with the age of its newest signal.
|
||||
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
|
||||
var rows []map[string]any
|
||||
for _, r := range signalsTable {
|
||||
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
|
||||
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
|
||||
switch {
|
||||
case r.Deferred != "":
|
||||
row["deferred"] = r.Deferred
|
||||
case f == nil:
|
||||
row["newest"] = "not yet looked at"
|
||||
default:
|
||||
if err := r.needs(f); err != nil {
|
||||
row["blind"] = err.Error()
|
||||
} else if newest := r.newest(f); newest.IsZero() {
|
||||
row["newest"] = "none heard"
|
||||
} else {
|
||||
row["newest"] = newest.UTC().Format(time.RFC3339)
|
||||
row["age"] = f.now.Sub(newest).Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
out := map[string]any{"signals": rows, "every": watchEvery.String()}
|
||||
if !ticked.IsZero() {
|
||||
out["looked"] = ticked.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// doctorText is an answer as a person reads it.
|
||||
func doctorText(answer any) string {
|
||||
body, _ := json.Marshal(answer)
|
||||
var b strings.Builder
|
||||
var verdict struct {
|
||||
Run doctorRun `json:"run"`
|
||||
Age string `json:"age"`
|
||||
}
|
||||
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
|
||||
r := verdict.Run
|
||||
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
|
||||
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
|
||||
for _, p := range r.Probes {
|
||||
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
|
||||
for _, f := range p.Found {
|
||||
fmt.Fprintf(&b, " %s\n", f)
|
||||
}
|
||||
for _, f := range p.Unconfirmed {
|
||||
fmt.Fprintf(&b, " unconfirmed, raised if the next run sees it too: %s\n", f)
|
||||
}
|
||||
if p.Error != "" {
|
||||
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
pretty, _ := json.MarshalIndent(answer, "", " ")
|
||||
return string(pretty) + "\n"
|
||||
}
|
||||
|
||||
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
|
||||
// the serving controller answers `doctor` from.
|
||||
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
|
||||
var run doctorRun
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
|
||||
}
|
||||
return json.Unmarshal(msg.Data, &run)
|
||||
})
|
||||
return run, err
|
||||
}
|
||||
|
||||
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
|
||||
// its own connection, and its own keeper, closed after.
|
||||
func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
k, err := keeperOn(ctx, js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
jsCtx := js.Context()
|
||||
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
|
||||
host: controlHost(ctx, open.inventory)}
|
||||
return d, func() {
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
k.Close(flushing)
|
||||
js.Close()
|
||||
open.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
|
||||
func sortedFound(obs []conditions.Observation) []conditions.Observation {
|
||||
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
|
||||
return obs
|
||||
}
|
||||
|
||||
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
|
||||
type probeAsksKey struct{}
|
||||
|
||||
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
|
||||
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
|
||||
p, ok := ctx.Value(probeAsksKey{}).(probe)
|
||||
if !ok {
|
||||
return "a caller outside the self-check", false
|
||||
}
|
||||
for _, v := range p.Asks {
|
||||
if v.Seat == seat && v.Verb == verb {
|
||||
return p.ID, true
|
||||
}
|
||||
}
|
||||
return p.ID, false
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/net/dns/dnsmessage"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
|
||||
|
||||
// withProbes runs the test with a registry of its own.
|
||||
func withProbes(t *testing.T, probes ...probe) {
|
||||
t.Helper()
|
||||
before := probeRegistry
|
||||
probeRegistry = probes
|
||||
t.Cleanup(func() { probeRegistry = before })
|
||||
}
|
||||
|
||||
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||
var broken atomic.Bool
|
||||
broken.Store(true)
|
||||
withProbes(t,
|
||||
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return []conditions.Observation{failing}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return nil, errors.New("the store is away")
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
<-ctx.Done()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||
)
|
||||
before := probeWithin
|
||||
probeWithin = 200 * time.Millisecond
|
||||
t.Cleanup(func() { probeWithin = before })
|
||||
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||
// A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot
|
||||
// run it either (novox/hq issue 277): one timed-out question is not a probe gone blind.
|
||||
first := d.runOnce(t.Context(), "a test")
|
||||
if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", first.Counts)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" {
|
||||
t.Fatalf("after one run, open: %+v", open)
|
||||
}
|
||||
run := d.runOnce(t.Context(), "a test")
|
||||
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", run.Counts)
|
||||
}
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key+"="+c.Kind)
|
||||
}
|
||||
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||
"probe.P4.failed=probe-failed"} {
|
||||
if !slices.Contains(keys, want) {
|
||||
t.Errorf("%s is not open: %v", want, keys)
|
||||
}
|
||||
}
|
||||
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||
if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent {
|
||||
t.Fatalf("said %v", told.Names)
|
||||
}
|
||||
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||
t.Fatal("the run is not the last verdict")
|
||||
}
|
||||
body, _ := json.Marshal(run)
|
||||
var shape map[string]any
|
||||
_ = json.Unmarshal(body, &shape)
|
||||
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
|
||||
// Mended: the next run clears every one of them.
|
||||
broken.Store(false)
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("a passing run left open %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **The registry says what each probe asserts**, and a probe not built says why and when.
|
||||
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, p := range probeRegistry {
|
||||
if seen[p.ID] {
|
||||
t.Errorf("%s twice", p.ID)
|
||||
}
|
||||
seen[p.ID] = true
|
||||
if p.Asserts == "" || p.From == "" || p.Kind == "" {
|
||||
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
|
||||
}
|
||||
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
|
||||
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
|
||||
if !seen[id] {
|
||||
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
|
||||
// stopped is S10 (signals_test.go).
|
||||
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
|
||||
w := &watchdogs{started: time.Now().Add(-time.Hour)}
|
||||
d := &doctor{watchdogs: w, host: "anchor"}
|
||||
got, err := probeWatchdogs(t.Context(), d)
|
||||
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
w.ticked = time.Now()
|
||||
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
|
||||
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
|
||||
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{"rival-one", "rival-two"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||
linted(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
|
||||
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||
answerAs := func(rcode dnsmessage.RCode) string {
|
||||
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
go func() {
|
||||
buf := make([]byte, 1500)
|
||||
for {
|
||||
n, from, err := conn.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var q dnsmessage.Message
|
||||
if q.Unpack(buf[:n]) != nil {
|
||||
continue
|
||||
}
|
||||
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
|
||||
if q.Questions[0].Type == dnsmessage.TypeA {
|
||||
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
|
||||
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
|
||||
} else {
|
||||
reply.RCode = rcode
|
||||
}
|
||||
packed, _ := reply.Pack()
|
||||
_, _ = conn.WriteTo(packed, from)
|
||||
}
|
||||
}()
|
||||
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||
return port
|
||||
}
|
||||
before := resolverPort
|
||||
t.Cleanup(func() { resolverPort = before })
|
||||
|
||||
resolverPort = answerAs(dnsmessage.RCodeSuccess)
|
||||
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
|
||||
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
|
||||
t.Fatalf("%v %v %v", v4, rcode, err)
|
||||
}
|
||||
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
|
||||
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
|
||||
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
|
||||
}
|
||||
resolverPort = answerAs(dnsmessage.RCodeNameError)
|
||||
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
|
||||
t.Fatalf("NXDOMAIN read as %v", rcode)
|
||||
}
|
||||
}
|
||||
|
||||
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||
got, err := p(t.Context(), d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := info.Config
|
||||
cfg.MaxMsgsPerSubject = 3
|
||||
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumers, err := probeConsumers(t.Context(), d)
|
||||
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||
}
|
||||
streams, err := probeStreams(t.Context(), d)
|
||||
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
heard := make(chan *nats.Msg, 16)
|
||||
for _, subject := range broker.BusAdvisories {
|
||||
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
api, _ := jetstream.New(conn)
|
||||
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kinds := map[string]string{}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for len(kinds) < 2 {
|
||||
select {
|
||||
case m := <-heard:
|
||||
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||
kinds[a.Kind] = a.Said
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the bus said only %v", kinds)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||
t.Fatalf("%v", kinds)
|
||||
}
|
||||
}
|
||||
|
||||
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
|
||||
// machine read as behind right after a push sent it the current build — it says the digest-named
|
||||
// directory it runs from, and the mesh holds a commit).
|
||||
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
|
||||
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
|
||||
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
|
||||
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
|
||||
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
|
||||
}}
|
||||
delivered := deliveredVersions(m)
|
||||
if !slices.Equal(delivered, []string{"31045596c83a"}) {
|
||||
t.Fatalf("%v", delivered)
|
||||
}
|
||||
commit := "1545b00a9f0c"
|
||||
for _, c := range []struct {
|
||||
reported string
|
||||
behind bool
|
||||
}{
|
||||
{"31045596c83a", false}, // the live case: current, and was called behind
|
||||
{"0123456789ab", true}, // another delivery
|
||||
{"1545b00a", false}, // placed by hand, stamped with the commit
|
||||
{"", false}, // not said
|
||||
} {
|
||||
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
|
||||
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
|
||||
}
|
||||
}
|
||||
if engineBehind("31045596c83a", nil, commit) {
|
||||
t.Error("behind a mesh that holds no delivered build")
|
||||
}
|
||||
}
|
||||
|
||||
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
|
||||
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
|
||||
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
|
||||
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked := 0
|
||||
for _, p := range probeRegistry {
|
||||
for _, v := range p.Asks {
|
||||
asked++
|
||||
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
|
||||
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
|
||||
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
|
||||
}
|
||||
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
|
||||
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
if asked == 0 {
|
||||
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
|
||||
}
|
||||
// And a probe asking what it did not declare is refused before anything is sent.
|
||||
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
|
||||
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
|
||||
!strings.Contains(err.Error(), "does not declare") {
|
||||
t.Fatalf("an undeclared question was asked: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// subjectMatches is the bus's matching of a permission pattern against a subject.
|
||||
func subjectMatches(pattern, subject string) bool {
|
||||
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
|
||||
for i, tok := range p {
|
||||
if tok == ">" {
|
||||
return len(s) > i
|
||||
}
|
||||
if i >= len(s) || (tok != "*" && tok != s[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(s)
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
|
||||
// store at all, so anything that tried to record or register would fail rather than pass quietly.
|
||||
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
|
||||
err := builds{}.Built(t.Context(), link.BuildResult{
|
||||
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
|
||||
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The mark survives the wire both ways: asked as a dry run, answered as one.
|
||||
func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
|
||||
raw, _ := json.Marshal(link.BuildRequest{ID: "build-1", Repository: "r", DryRun: true})
|
||||
var asked link.BuildRequest
|
||||
if err := json.Unmarshal(raw, &asked); err != nil || !asked.DryRun {
|
||||
t.Fatalf("the request lost its dry-run mark: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(link.BuildResult{ID: "build-1", DryRun: true})
|
||||
var answered link.BuildResult
|
||||
if err := json.Unmarshal(raw, &answered); err != nil || !answered.DryRun {
|
||||
t.Fatalf("the outcome lost its dry-run mark: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(link.BuildResult{ID: "build-2"})
|
||||
if string(raw) != `{"id":"build-2","repository":"","on":""}` {
|
||||
t.Fatalf("an ordinary outcome carries a dry-run mark: %s", raw)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||
if asked.IsZero() || result.ID == "" {
|
||||
return
|
||||
}
|
||||
subject := result.Module
|
||||
if subject == "" {
|
||||
subject = result.Repository
|
||||
}
|
||||
detail := "built"
|
||||
if result.Failed != "" {
|
||||
detail = "failed: " + firstLine(result.Failed)
|
||||
}
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
case inventory.DurationHeartbeatGap:
|
||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said
|
||||
// it reads one, and to no other: an older node-engine refuses a key it does not know, whole.
|
||||
|
||||
// bodiesDelivery records each send as the mesh does, and keeps the bodies.
|
||||
type bodiesDelivery struct {
|
||||
recordedDelivery
|
||||
bodies map[string][]byte
|
||||
}
|
||||
|
||||
func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
b.bodies[s.node] = body
|
||||
return b.recordedDelivery.declare(ctx, s, body)
|
||||
}
|
||||
|
||||
func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
epoch := uint64(57)
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return epoch, nil }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
|
||||
anchor, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carried := func(node string) (epoch float64, has bool) {
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal(d.bodies[node], &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epoch, has = envelope["epoch"].(float64)
|
||||
return epoch, has
|
||||
}
|
||||
if e, has := carried("anchor"); !has || e != 57 {
|
||||
t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"])
|
||||
}
|
||||
if _, has := carried("laptop"); has {
|
||||
t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"])
|
||||
}
|
||||
|
||||
// A new holder of the lease is not a change of the machine: neither reads as behind.
|
||||
epoch = 58
|
||||
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
sent, err := inv.Outstanding(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if would[node] != sent {
|
||||
t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A process that may not act composes nothing and sends nothing: its number is not taken.
|
||||
func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") {
|
||||
t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused)
|
||||
}
|
||||
anchor, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||
if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 {
|
||||
t.Fatalf("a controller without the lease took sequence %d", seq)
|
||||
}
|
||||
|
||||
// And at the send itself: the gate every declaration passes.
|
||||
gate := link.ActingGate
|
||||
link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { link.ActingGate = gate })
|
||||
if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil ||
|
||||
!strings.Contains(err.Error(), "lost the lease") {
|
||||
t.Fatalf("a declaration was let through the gate: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,646 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/validate"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The facts snapshot (novox/hq to-be 45 §9, ADR 0227 rule 9): what a merge check needs to judge a change
|
||||
// against the mesh that runs, written by the controller to the artifact store, where the build seat reads
|
||||
// it. internal/facts says what it holds and what it never holds; this composes it from the store and
|
||||
// keeps it current.
|
||||
|
||||
// factsEvery is how often the snapshot is composed. It is kept when it moved — a machine, an
|
||||
// assignment, a seat, a setting, a build — or once a day when nothing did, so its age says the
|
||||
// controller is still writing it (S14).
|
||||
var factsEvery = 10 * time.Minute
|
||||
|
||||
// factsDaily is how old a snapshot of an unchanged mesh may grow before it is written again.
|
||||
const factsDaily = 24 * time.Hour
|
||||
|
||||
// factsStaleAfter is S14's bound: a snapshot older than this is one no check should be fed.
|
||||
const factsStaleAfter = 48 * time.Hour
|
||||
|
||||
// factsExport is what this controller knows of the snapshot it keeps: when the newest was taken, its
|
||||
// content, and the last attempt's error.
|
||||
type factsExport struct {
|
||||
mu sync.Mutex
|
||||
taken time.Time
|
||||
content string
|
||||
digest string
|
||||
err error
|
||||
began time.Time
|
||||
}
|
||||
|
||||
// exportedFacts is this process's export, read by S14.
|
||||
var exportedFacts = &factsExport{}
|
||||
|
||||
func (e *factsExport) last() (taken time.Time, digest string, began time.Time, err error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return e.taken, e.digest, e.began, e.err
|
||||
}
|
||||
|
||||
func (e *factsExport) kept(f snapshot.Facts, content, digest string) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.taken, e.content, e.digest, e.err = f.Taken, content, digest, nil
|
||||
}
|
||||
|
||||
func (e *factsExport) failed(err error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.err = err
|
||||
}
|
||||
|
||||
// exportingFacts keeps the snapshot current for as long as this controller holds the lease: ctx ends
|
||||
// when it stops acting.
|
||||
func exportingFacts(ctx context.Context, open *stores, busVersion func() string) {
|
||||
exportedFacts.mu.Lock()
|
||||
exportedFacts.began = time.Now()
|
||||
exportedFacts.mu.Unlock()
|
||||
// What the store holds already, so a restarted controller neither writes an unchanged snapshot again
|
||||
// nor reads its age as zero.
|
||||
if address, err := factsStore(ctx, open); err == nil {
|
||||
if body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag); err == nil {
|
||||
if f, err := snapshot.Decode(body); err == nil {
|
||||
content, _ := f.Content()
|
||||
exportedFacts.kept(f, content, digest)
|
||||
}
|
||||
}
|
||||
}
|
||||
failing := ""
|
||||
first := time.NewTimer(time.Minute)
|
||||
defer first.Stop()
|
||||
tick := time.NewTicker(factsEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-first.C:
|
||||
case <-tick.C:
|
||||
}
|
||||
wrote, err := exportFacts(ctx, open, busVersion(), false)
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
exportedFacts.failed(err)
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("the facts snapshot cannot be kept: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("the facts snapshot is kept again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
if wrote != "" {
|
||||
fmt.Printf("the facts snapshot moved and is kept as %s\n", short(strings.TrimPrefix(wrote, "sha256:")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// exportFacts composes the snapshot and keeps it when it moved, or when the one kept is a day old, or
|
||||
// when told to. Answers the digest it kept, empty when it kept nothing.
|
||||
func exportFacts(ctx context.Context, open *stores, busVersion string, force bool) (string, error) {
|
||||
f, err := gatherFacts(ctx, open, busVersion)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
content, err := f.Content()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
exportedFacts.mu.Lock()
|
||||
unchanged := content == exportedFacts.content && time.Since(exportedFacts.taken) < factsDaily
|
||||
exportedFacts.mu.Unlock()
|
||||
if unchanged && !force {
|
||||
return "", nil
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
address, err := factsStore(ctx, open)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest, err := (artifacts.Store{Address: address}).PutTagged(ctx, snapshot.Repository, snapshot.Tag, snapshot.MediaType, body)
|
||||
if err != nil && digest == "" {
|
||||
return "", err
|
||||
}
|
||||
exportedFacts.kept(f, content, digest)
|
||||
return digest, err
|
||||
}
|
||||
|
||||
// factsStore is the artifact store as this controller reaches it.
|
||||
func factsStore(ctx context.Context, open *stores) (string, error) {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if address == "" {
|
||||
return "", errors.New("the artifact store is not on the private network, so there is nowhere to keep the facts")
|
||||
}
|
||||
return address, nil
|
||||
}
|
||||
|
||||
// gatherFacts composes one snapshot from the store: read only, nothing made, nothing sent.
|
||||
func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot.Facts, error) {
|
||||
inv := open.inventory
|
||||
f := snapshot.Facts{Format: snapshot.Format, Taken: time.Now().UTC(), Controller: snapshot.Build{Version: version}}
|
||||
f.Versions.Bus = busVersion
|
||||
storeVersion, err := inv.ServerVersion(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, fmt.Errorf("the store will not say its version: %w", err)
|
||||
}
|
||||
f.Versions.Store = storeVersion
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
place := map[string]inventory.Overlay{}
|
||||
for _, o := range overlays {
|
||||
place[o.Name] = o
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
|
||||
// **Every name first**, so text read afterwards — a setting naming a machine, a problem naming a
|
||||
// site — has it replaced wherever it appears.
|
||||
scrub := snapshot.NewScrubber()
|
||||
domains := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
scrub.Machine(n.Name)
|
||||
if n.Account != "" && n.Account != "root" {
|
||||
scrub.Account(n.Account)
|
||||
}
|
||||
d, err := inv.PublicDomainOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
domains[n.Name] = scrub.Domain(d)
|
||||
}
|
||||
for _, o := range overlays {
|
||||
scrub.Site(o.Site)
|
||||
}
|
||||
|
||||
// What a merge check runs in and reads beside it, as the build seat would be asked for it.
|
||||
if held, err := inv.Held(ctx); err == nil {
|
||||
for language, reference := range builder.ToolchainsOf(held) {
|
||||
if f.Versions.Toolchains == nil {
|
||||
f.Versions.Toolchains = map[string]string{}
|
||||
}
|
||||
f.Versions.Toolchains[language] = catalogue.Recorded(reference)
|
||||
}
|
||||
} else {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if dir, core := coreModules[e.Manifest.Module]; core && !e.Provided && e.Source.Repository != "" {
|
||||
for d, ref := range besideRefs(dir, current[e.Manifest.Module].Commit) {
|
||||
if f.Beside == nil {
|
||||
f.Beside = map[string]string{}
|
||||
}
|
||||
f.Beside[d] = ref
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if c, ok := current["mesh-controller"]; ok {
|
||||
f.Controller.Commit = c.Commit
|
||||
}
|
||||
|
||||
gens, gensErr := generators(ctx, open)
|
||||
hostShelf := shelf[hostModule]
|
||||
meshWide := map[string]bool{}
|
||||
engines := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
m := snapshot.Machine{Name: scrub.Machine(n.Name), Length: len(n.Name), Adopted: n.Adopted,
|
||||
AccountHome: scrub.Text(n.AccountHome), NodeEngine: n.HostVersion, PublicDomain: domains[n.Name]}
|
||||
switch n.Account {
|
||||
case "", "root":
|
||||
m.Account = n.Account
|
||||
default:
|
||||
m.Account = scrub.Account(n.Account)
|
||||
}
|
||||
if n.HostVersion != "" {
|
||||
engines[n.HostVersion] = true
|
||||
}
|
||||
m.System = systemOf(hostShelf, n.HostVersion)
|
||||
m.Libc = libcOf(m.System)
|
||||
reported, err := inv.DescribedOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
|
||||
outward, err := inv.OutwardLinksOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, l := range outward {
|
||||
m.OutwardLinks = append(m.OutwardLinks, scrub.Text(l))
|
||||
}
|
||||
capabilities, err := inv.Profile(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, c := range capabilities {
|
||||
kept := snapshot.Capability{Name: c.Name, Present: c.Present}
|
||||
// The detail only where it is a version: everything else a detector says — a ruleset, a
|
||||
// device, a path — is the machine's own business and no check reads it.
|
||||
if c.Present && (c.Name == "container-runtime" || c.Name == "package-manager") {
|
||||
kept.Detail = scrub.Text(c.Detail)
|
||||
}
|
||||
m.Capabilities = append(m.Capabilities, kept)
|
||||
}
|
||||
if o, ok := place[n.Name]; ok {
|
||||
m.Site, m.Hub, m.Public, m.OnNetwork = scrub.Site(o.Site), o.Hub, o.Endpoint != "", o.Address != ""
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Assigned = assigned
|
||||
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
if known && slices.Contains(assigned, broker.RuntimeModule) {
|
||||
m.NodeTools = sent[broker.RuntimeModule]
|
||||
}
|
||||
pins, err := inv.PinsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for provision, c := range pins {
|
||||
m.Pins = append(m.Pins, snapshot.Pin{Provision: provision, Machine: scrub.Machine(c.Node), Module: c.Module})
|
||||
}
|
||||
for _, module := range assigned {
|
||||
held, err := inv.SecretsOf(ctx, n.Name, module)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, h := range held {
|
||||
if h.Origin == inventory.OriginAccepted {
|
||||
m.Accepted = append(m.Accepted, snapshot.Accepted{Module: module, Name: h.Name,
|
||||
Provider: scrub.Machine(h.Provider), Local: h.Local})
|
||||
}
|
||||
}
|
||||
layers, err := inv.SettingsFor(ctx, n.Name, module)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, layer := range layers {
|
||||
if layer.From == catalogue.MeshWideLayer {
|
||||
if !meshWide[module] {
|
||||
meshWide[module] = true
|
||||
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
}
|
||||
continue
|
||||
}
|
||||
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
}
|
||||
}
|
||||
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
|
||||
if ctx.Err() != nil {
|
||||
return snapshot.Facts{}, ctx.Err()
|
||||
}
|
||||
f.Machines = append(f.Machines, m)
|
||||
}
|
||||
for e := range engines {
|
||||
f.Versions.NodeEngines = append(f.Versions.NodeEngines, e)
|
||||
}
|
||||
|
||||
// Seats and their holders, and from them the roles a machine is named by.
|
||||
roles := map[string][]string{}
|
||||
seats := map[string]*snapshot.Seat{}
|
||||
for _, h := range holdings {
|
||||
key := h.Claim + "\x00" + h.Scope
|
||||
s, ok := seats[key]
|
||||
if !ok {
|
||||
s = &snapshot.Seat{Name: h.Claim, Scope: h.Scope}
|
||||
seats[key] = s
|
||||
}
|
||||
s.Holders = append(s.Holders, snapshot.Holder{Machine: scrub.Machine(h.Node), Module: h.Module})
|
||||
if h.Scope == catalogue.ScopeMesh {
|
||||
role := "holds " + h.Claim
|
||||
if h.Claim == catalogue.ControllerSeatName {
|
||||
role = "the control node"
|
||||
}
|
||||
roles[h.Node] = append(roles[h.Node], role)
|
||||
}
|
||||
}
|
||||
for _, s := range seats {
|
||||
f.Seats = append(f.Seats, *s)
|
||||
}
|
||||
for i := range f.Machines {
|
||||
for _, n := range nodes {
|
||||
if scrub.Machine(n.Name) != f.Machines[i].Name {
|
||||
continue
|
||||
}
|
||||
f.Machines[i].Roles = roles[n.Name]
|
||||
if f.Machines[i].Hub {
|
||||
f.Machines[i].Roles = append(f.Machines[i].Roles, "the hub")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every module, as the mesh holds it, and where it is built from.
|
||||
newest := map[string]inventory.Source{}
|
||||
count := map[string]int{}
|
||||
for _, e := range entries {
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
mod := snapshot.Module{Name: e.Manifest.Module, Repository: snapshot.RepositoryName(e.Source.Repository), Path: e.Source.Path,
|
||||
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
|
||||
Manifest: raw}
|
||||
for _, r := range read[e.Manifest.Module] {
|
||||
mod.Reads = append(mod.Reads, snapshot.RepositoryName(r.Repository))
|
||||
}
|
||||
f.Modules = append(f.Modules, mod)
|
||||
if e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
count[e.Source.Repository]++
|
||||
if was, ok := newest[e.Source.Repository]; !ok || e.Source.Seen.After(was.Seen) {
|
||||
newest[e.Source.Repository] = e.Source
|
||||
}
|
||||
}
|
||||
for repository, s := range newest {
|
||||
commit := s.Head
|
||||
if commit == "" {
|
||||
commit = s.BuiltFrom
|
||||
}
|
||||
f.Sources = append(f.Sources, snapshot.Source{Repository: snapshot.RepositoryName(repository), Commit: commit,
|
||||
Modules: count[repository]})
|
||||
}
|
||||
for _, e := range edges {
|
||||
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
|
||||
}
|
||||
f.Sorted()
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// declarationFacts is how one machine's declaration composes now, as the next push would compose it and
|
||||
// without making anything (D1's composition), and whether the node-engine's validator takes it.
|
||||
func declarationFacts(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
|
||||
gensErr error, scrub *snapshot.Scrubber) snapshot.Declaration {
|
||||
var d snapshot.Declaration
|
||||
if gensErr != nil {
|
||||
d.Problems = []string{scrub.Text("the private network cannot be computed: " + oneLine(gensErr.Error()))}
|
||||
return d
|
||||
}
|
||||
declared, problems, err := composedAndValidated(ctx, open, node, gens, Foreseeing)
|
||||
if err != nil {
|
||||
d.Problems = []string{scrub.Text(oneLine(err.Error()))}
|
||||
return d
|
||||
}
|
||||
for _, p := range problems {
|
||||
d.Problems = append(d.Problems, scrub.Text(p))
|
||||
}
|
||||
d.Composes = len(problems) == 0
|
||||
if body, err := declared.Body(); err == nil {
|
||||
d.Digest = fmt.Sprintf("sha256:%x", sha256.Sum256(body))
|
||||
}
|
||||
// Scrubbed like every other word: a resource is named after the machine a grant is for.
|
||||
for _, r := range resourceNames(declared.Resources) {
|
||||
d.Resources = append(d.Resources, scrub.Text(r))
|
||||
}
|
||||
for module, why := range declared.leftOutWhy {
|
||||
if d.LeftOut == nil {
|
||||
d.LeftOut = map[string]string{}
|
||||
}
|
||||
d.LeftOut[module] = scrub.Text(why)
|
||||
}
|
||||
for _, o := range declared.withheld {
|
||||
d.Withheld = append(d.Withheld, scrub.Text(o.String()))
|
||||
}
|
||||
for _, u := range declared.unbound {
|
||||
d.Unbound = append(d.Unbound, scrub.Text(u.String()))
|
||||
}
|
||||
sort.Strings(d.Withheld)
|
||||
sort.Strings(d.Unbound)
|
||||
return d
|
||||
}
|
||||
|
||||
// composedAndValidated composes one machine's declaration — as a push would (Allocating) or as the next
|
||||
// push will without making anything (Foreseeing) — with the order it was last sent, and runs the
|
||||
// node-engine's own validator over the body. An error is that it did not compose; problems are what the
|
||||
// validator refuses.
|
||||
func composedAndValidated(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
|
||||
choosing Choosing) (sendable, []string, error) {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
declared, err := declarationWith(ctx, open, node, plan, settings, gens, choosing)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
if declared.Sequence, err = open.inventory.Sequence(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
return declared, validate.Declaration(body), nil
|
||||
}
|
||||
|
||||
// resourceNames are a declaration's resources as `type:id`, sorted.
|
||||
func resourceNames(resources []map[string]any) []string {
|
||||
out := make([]string, 0, len(resources))
|
||||
for _, r := range resources {
|
||||
kind, _ := r["type"].(string)
|
||||
id, _ := r["id"].(string)
|
||||
out = append(out, kind+":"+id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// systemOf is the system a node-engine of that version was built for, read from the build the mesh
|
||||
// holds: the artifact a resource delivered into `versions/<version>` came from is named for its system
|
||||
// (`host-arch`). Empty when the version is not a delivered one — an engine placed by hand.
|
||||
func systemOf(host catalogue.Manifest, version string) string {
|
||||
if version == "" {
|
||||
return ""
|
||||
}
|
||||
for _, r := range host.Resources {
|
||||
path, _ := r["path"].(string)
|
||||
if !strings.HasSuffix(path, "/versions/"+version) {
|
||||
continue
|
||||
}
|
||||
source, _ := r["source"].(string)
|
||||
for _, part := range strings.Split(source, "/") {
|
||||
if system, ok := strings.CutPrefix(part, "host-"); ok && system != "" {
|
||||
return system
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// libcOf is the C library of a system the node-engine is built for.
|
||||
func libcOf(system string) string {
|
||||
switch system {
|
||||
case "arch":
|
||||
return "glibc"
|
||||
case "alpine":
|
||||
return "musl"
|
||||
case "android":
|
||||
return "bionic"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// factsCommand is `facts`: what the controller keeps, and keeping it now.
|
||||
//
|
||||
// facts the snapshot the artifact store holds: when, which, how many machines
|
||||
// facts show the same, whole, as JSON
|
||||
// facts export compose and keep one now
|
||||
// facts compose compose one and print it, keeping nothing
|
||||
func factsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("facts", flag.ContinueOnError)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := ""
|
||||
if len(rest) > 0 {
|
||||
what = rest[0]
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
switch what {
|
||||
case "", "show":
|
||||
address, err := factsStore(ctx, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if what == "show" {
|
||||
_, err := os.Stdout.Write(body)
|
||||
return err
|
||||
}
|
||||
f, err := snapshot.Decode(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the facts snapshot kept as %s:%s is %s, taken %s (%s ago) by controller %s\n",
|
||||
snapshot.Repository, snapshot.Tag, short(strings.TrimPrefix(digest, "sha256:")),
|
||||
f.Taken.Format(time.RFC3339), ago(time.Since(f.Taken)), orNone(f.Controller.Commit))
|
||||
fmt.Printf(" %d machine(s), %d module(s), %d seat(s); the longest machine name is %d characters\n",
|
||||
len(f.Machines), len(f.Modules), len(f.Seats), f.Longest())
|
||||
fmt.Printf(" the bus runs %s, the store %s\n", orNone(f.Versions.Bus), orNone(f.Versions.Store))
|
||||
for _, m := range f.Machines {
|
||||
state := "composes"
|
||||
if !m.Declaration.Composes {
|
||||
state = "does NOT compose: " + strings.Join(m.Declaration.Problems, "; ")
|
||||
}
|
||||
fmt.Printf(" %-12s %s; %d module(s); %s\n", m.Name, m.Described(), len(m.Assigned), state)
|
||||
}
|
||||
return nil
|
||||
case "export", "compose":
|
||||
busVersion := ""
|
||||
if server, err := connectLink(ctx, nil, nil, nil); err == nil {
|
||||
busVersion = busVersionOf(server)
|
||||
server.Close()
|
||||
}
|
||||
if what == "compose" {
|
||||
f, err := gatherFacts(ctx, open, busVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = os.Stdout.Write(append(body, '\n'))
|
||||
return err
|
||||
}
|
||||
digest, err := exportFacts(ctx, open, busVersion, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the facts snapshot is kept as %s:%s, %s\n", snapshot.Repository, snapshot.Tag, digest)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("facts [show|export|compose], not %q", what)
|
||||
}
|
||||
|
||||
// busVersionOf is the bus server's release, as it told this connection.
|
||||
func busVersionOf(server *link.Server) string {
|
||||
if bus, ok := server.Bus().(link.OverNATS); ok && bus.Conn != nil {
|
||||
return bus.Conn.ConnectedServerVersion()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
)
|
||||
|
||||
// The facts snapshot (novox/hq to-be 45 §9): composed from the store, every machine under a pseudonym
|
||||
// of its name's length, and nothing of the installation in it — no secret, no address, no name.
|
||||
|
||||
// aMeshWithSecrets is aMesh with a provider and its consumers, a value given by hand, settings carrying
|
||||
// a password, an address and a machine's name, and a push's worth of credentials made.
|
||||
func aMeshWithSecrets(t *testing.T) (*stores, []string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"},
|
||||
Resources: []map[string]any{{"id": "config", "type": "file", "path": "/etc/files/config.json",
|
||||
"mode": "0600", "content": "{}", "merge": "json"}}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
secrets := []string{"Hunter2-Is-Not-A-Password-9f8e7d", "0123456789abcdefABCDEF0123456789zz"}
|
||||
if err := open.inventory.SetSettings(ctx, "", "files", map[string]any{
|
||||
"admin_password": secrets[0], "upstream": "10.77.0.9", "hub": "anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "laptop", "files", map[string]any{
|
||||
"note": "reach me at 192.168.1.135, token " + secrets[1]}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A push's worth of composition, which makes the pair credential the consumer is sent.
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, _, err := composedAndValidated(ctx, open, node, gens, Allocating); err != nil {
|
||||
t.Fatalf("%s does not compose: %v", node, err)
|
||||
}
|
||||
}
|
||||
issued, err := open.inventory.SecretsFrom(ctx, "anchor")
|
||||
if err != nil || len(issued) == 0 {
|
||||
t.Fatalf("no credential was made for the consumer: %v", err)
|
||||
}
|
||||
for _, s := range issued {
|
||||
// Sealed, never kept plain (ADR 0004): the sealed blobs are what the store holds, and none may leave.
|
||||
secrets = append(secrets, s.ForConsumer, s.ForProvider)
|
||||
}
|
||||
return open, secrets
|
||||
}
|
||||
|
||||
func TestTheFactsCarryNoSecretNoAddressAndNoName(t *testing.T) {
|
||||
open, secrets := aMeshWithSecrets(t)
|
||||
f, err := gatherFacts(t.Context(), open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := f.Encode()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(body)
|
||||
for _, s := range secrets {
|
||||
if s != "" && strings.Contains(text, s) {
|
||||
t.Errorf("a secret is in the snapshot: %q", s)
|
||||
}
|
||||
}
|
||||
for _, leaked := range []string{"anchor", "laptop", "10.77.0.", "192.168.1.135", ".example:51820"} {
|
||||
if strings.Contains(text, leaked) {
|
||||
t.Errorf("%q is in the snapshot", leaked)
|
||||
}
|
||||
}
|
||||
// Every address it carries is a documentation address.
|
||||
for _, a := range regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b`).FindAllString(text, -1) {
|
||||
if !strings.HasPrefix(a, "192.0.2.") && !strings.HasPrefix(a, "198.51.100.") && !strings.HasPrefix(a, "203.0.113.") {
|
||||
t.Errorf("%s is an address outside the documentation ranges", a)
|
||||
}
|
||||
}
|
||||
|
||||
// What a check needs is there: every machine, its length, its modules, its declaration composing.
|
||||
if len(f.Machines) != 2 || f.Longest() != len("laptop") {
|
||||
t.Fatalf("machines %+v, longest %d", f.Machines, f.Longest())
|
||||
}
|
||||
anchor := snapshot.Pseudonym("machine", "anchor")
|
||||
m, ok := f.Machine(anchor)
|
||||
if !ok || !m.Hub || !strings.Contains(m.Described(), "the hub") || len(m.Name) != len("anchor") {
|
||||
t.Fatalf("the anchor reads as %+v", m)
|
||||
}
|
||||
if !m.Declaration.Composes || m.Declaration.Digest == "" || len(m.Declaration.Resources) == 0 {
|
||||
t.Errorf("the anchor's declaration reads as %+v", m.Declaration)
|
||||
}
|
||||
laptop, _ := f.Machine(snapshot.Pseudonym("machine", "laptop"))
|
||||
if strings.Join(laptop.Assigned, ",") != "files,mesh-wireguard" && !strings.Contains(strings.Join(laptop.Assigned, ","), "files") {
|
||||
t.Errorf("the laptop's assignments read as %v", laptop.Assigned)
|
||||
}
|
||||
var meshWide map[string]any
|
||||
for _, s := range f.Settings {
|
||||
if s.Module == "files" {
|
||||
meshWide = s.Values
|
||||
}
|
||||
}
|
||||
if meshWide["admin_password"] != snapshot.Withheld || meshWide["hub"] != anchor {
|
||||
t.Errorf("the mesh-wide settings read as %v", meshWide)
|
||||
}
|
||||
if f.Versions.Bus != "2.11.17" || f.Versions.Store == "" {
|
||||
t.Errorf("versions read as %+v", f.Versions)
|
||||
}
|
||||
var objects bool
|
||||
for _, mod := range f.Modules {
|
||||
objects = objects || mod.Name == "objects" && len(mod.Manifest) > 0
|
||||
}
|
||||
if !objects {
|
||||
t.Error("the modules the mesh holds are not in the snapshot")
|
||||
}
|
||||
|
||||
// And an unchanged mesh is the same content a moment later.
|
||||
again, err := gatherFacts(t.Context(), open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := f.Content()
|
||||
b, _ := again.Content()
|
||||
if a != b {
|
||||
t.Error("two snapshots of an unchanged mesh differ, so it would be written again every ten minutes")
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package main
|
||||
|
||||
// The broker opening belongs only on the node that listens on it (novox/hq: it leaked onto
|
||||
// every enrolled node's declaration, opening a from-anywhere hole for a port nothing there
|
||||
// serves). foundationPortsFor is the scope.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
func TestTheBrokerHostGetsTheFoundationOpening(t *testing.T) {
|
||||
broker := catalogue.Manifest{Module: "lavinmq", Listens: []catalogue.Listening{
|
||||
{Port: 5671, Protocol: "tcp", From: "mesh"},
|
||||
{Port: 5672, Protocol: "tcp", From: "mesh"},
|
||||
}}
|
||||
got := foundationPortsFor(5671, []catalogue.Manifest{broker})
|
||||
if len(got) != 1 || got[0] != 5671 {
|
||||
t.Fatalf("the node that listens on the broker port keeps it; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatOnlyDialsTheBrokerGetsNoOpening(t *testing.T) {
|
||||
// ace's set: things that reach the broker as a client, none listening on 5671.
|
||||
ace := []catalogue.Manifest{
|
||||
{Module: "plex", Listens: []catalogue.Listening{{Port: 32400, Protocol: "tcp", From: "anywhere"}}},
|
||||
{Module: "postgres", Listens: []catalogue.Listening{{Port: 5432, Protocol: "tcp", From: "mesh"}}},
|
||||
}
|
||||
if got := foundationPortsFor(5671, ace); got != nil {
|
||||
t.Fatalf("a node that only dials out opens nothing for the broker; got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,977 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0236, to-be 45
|
||||
// §8, ADR 0227 rule 8).
|
||||
//
|
||||
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||
// the component's health — the core's health definitions, or a module's own — passing three times over
|
||||
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
|
||||
//
|
||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||
// build goes back to the one the first machine ran before, and that machine is sent it: the ordinary
|
||||
// path, again. Once per build: the verdict is written before the send, and a verdict once written is
|
||||
// not written over. Said as a condition, urgent for the core and when it could not be put back, and as
|
||||
// the event `rolled-back`.
|
||||
|
||||
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
|
||||
var (
|
||||
// gateSettle is how long the first machine must stay healthy, at the least.
|
||||
gateSettle = 2 * time.Minute
|
||||
// gateBound is how long after the apply the build may take to become healthy.
|
||||
gateBound = 10 * time.Minute
|
||||
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
|
||||
gatePasses = 3
|
||||
gateEvery = 40 * time.Second
|
||||
)
|
||||
|
||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
||||
const gateProbe = "DG"
|
||||
|
||||
// The kinds a gate raises.
|
||||
const (
|
||||
kindRolledBack = "rolled-back"
|
||||
kindRollbackFailed = "rollback-failed"
|
||||
)
|
||||
|
||||
// coreComponent is the core component a module is, as a witness names it — controller, node-engine,
|
||||
// node-tools — or empty: the core is judged by its health definitions, anything else by its own health.
|
||||
func coreComponent(module string) string {
|
||||
switch module {
|
||||
case catalogue.ControllerSeatName:
|
||||
return lease.ComponentController
|
||||
case hostModule:
|
||||
return lease.ComponentEngine
|
||||
case broker.RuntimeModule:
|
||||
return lease.ComponentNodeTools
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// health is a judging's word on one machine.
|
||||
type health int
|
||||
|
||||
const (
|
||||
healthGood health = iota
|
||||
// healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy
|
||||
// provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a
|
||||
// build for something it did not do.
|
||||
healthWaiting
|
||||
healthNotYet
|
||||
healthBroken
|
||||
)
|
||||
|
||||
// served is what one machine's node tools answered the bus's discovery with.
|
||||
type served struct {
|
||||
runtime bool
|
||||
tools map[string]bool
|
||||
}
|
||||
|
||||
// gateFacts is what one judging reads, gathered once for every machine it judges.
|
||||
type gateFacts struct {
|
||||
now time.Time
|
||||
reports map[string]inventory.Reported
|
||||
// engines is each machine's node-engine build as it last reported it.
|
||||
engines map[string]string
|
||||
// served is what the bus's discovery answered; servedErr why it could not be asked.
|
||||
served map[string]served
|
||||
servedErr error
|
||||
// open are the open conditions; openErr why they could not be read (nil keeper: not judged).
|
||||
open []conditions.Condition
|
||||
openErr error
|
||||
judged bool
|
||||
// rolledBack is what each machine's witnesses say they decided.
|
||||
rolledBack map[string][]lease.Rollback
|
||||
// holder is who holds the controller lease, for judging the controller.
|
||||
holder *lease.Holder
|
||||
holderErr error
|
||||
// health is each machine's newest health statement (ADR 0240); a machine absent never stated one.
|
||||
// healthErr is why they could not be read.
|
||||
health map[string]inventory.NodeHealth
|
||||
healthErr error
|
||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||
heldOn map[string]string
|
||||
}
|
||||
|
||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||
// variable so a test can hand a judging its facts.
|
||||
var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
inv := open.inventory
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: witnessed.all()}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, n := range nodes {
|
||||
f.engines[n.Name] = n.HostVersion
|
||||
}
|
||||
// What each machine says of its long-running resources (ADR 0240): unreadable is said, never read as
|
||||
// healthy.
|
||||
f.health, f.healthErr = inv.Healths(ctx)
|
||||
if d := doctorFrom; d != nil {
|
||||
if d.keeper != nil {
|
||||
f.judged = true
|
||||
f.open, f.openErr = d.keeper.Open(ctx)
|
||||
}
|
||||
if d.js != nil {
|
||||
f.served, f.servedErr = servedOnTheBus(ctx, d.js.Conn())
|
||||
} else {
|
||||
f.servedErr = errors.New("this controller has no bus to ask")
|
||||
}
|
||||
} else {
|
||||
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
|
||||
}
|
||||
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
||||
if f.healthErr == nil && f.openErr == nil {
|
||||
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
||||
f.heldOn = map[string]string{}
|
||||
for machine := range f.health {
|
||||
for module, p := range hold.heldModules(machine) {
|
||||
f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if theLease != nil {
|
||||
h, found, err := theLease.holder(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
f.holderErr = err
|
||||
case found:
|
||||
f.holder = &h
|
||||
}
|
||||
if component != lease.ComponentController && f.holderErr != nil {
|
||||
f.holderErr = nil // read only for the controller's own judging
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// judgeHealth is one machine's health for a module's new build, from what one judging read: healthy,
|
||||
// not yet (with what is wanting), or healthBroken — a witness put it back, or the machine refused or failed
|
||||
// what it was sent. Pure.
|
||||
func judgeHealth(module, component string, m catalogue.Manifest, machine string, since time.Time, f gateFacts) (health, string) {
|
||||
// A witness's verdict made since the build was sent: the build failed its health there. One that
|
||||
// could not judge at all (unwitnessed) is not a verdict on the build.
|
||||
for _, r := range f.rolledBack[machine] {
|
||||
if component == "" || r.Component != component || r.Outcome == lease.OutcomeUnwitnessed || r.At.Before(since) {
|
||||
continue
|
||||
}
|
||||
return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component,
|
||||
short(r.From), r.Outcome, r.Why)
|
||||
}
|
||||
r, said := f.reports[machine]
|
||||
switch {
|
||||
case !said || r.At == nil || !r.Current:
|
||||
return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine)
|
||||
case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused:
|
||||
return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome)
|
||||
case r.Outcome != inventory.OutcomeApplied:
|
||||
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
|
||||
}
|
||||
// **No new condition about it**: about the machine itself, or naming the module on that machine,
|
||||
// raised since the judging began. The gate's own are not evidence about the build.
|
||||
if f.judged {
|
||||
if f.openErr != nil {
|
||||
return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " +
|
||||
firstLine(f.openErr.Error())
|
||||
}
|
||||
for _, c := range f.open {
|
||||
if c.Source == gateProbe || c.Raised.Before(since) {
|
||||
continue
|
||||
}
|
||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||
(c.Subject.Scope == conditions.ScopeMachine && c.Subject.ID == machine)
|
||||
if !onIt {
|
||||
continue
|
||||
}
|
||||
// A module's own health condition names it whole, its name's dots and all (ADR 0240).
|
||||
ownHealth := c.Subject.Scope == conditions.ScopeModule && c.Subject.ID == module+"."+machine
|
||||
if c.Subject.Scope == conditions.ScopeMachine || ownHealth || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
|
||||
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch component {
|
||||
case lease.ComponentEngine:
|
||||
// The node-engine has reported its current declaration under its own build.
|
||||
want := deliveredVersions(m)
|
||||
if len(want) > 0 && !slices.Contains(want, f.engines[machine]) {
|
||||
return healthNotYet, fmt.Sprintf("%s's node-engine reports build %s, not the new %s", machine,
|
||||
orNotKnown(f.engines[machine]), strings.Join(want, " or "))
|
||||
}
|
||||
case lease.ComponentNodeTools:
|
||||
// The node tools are announced and answer.
|
||||
if f.servedErr != nil {
|
||||
return healthNotYet, "whether the node tools answer cannot be asked: " + firstLine(f.servedErr.Error())
|
||||
}
|
||||
if !f.served[machine].runtime {
|
||||
return healthNotYet, fmt.Sprintf("the node tools on %s do not answer the bus", machine)
|
||||
}
|
||||
case lease.ComponentController:
|
||||
// The new controller holds the lease and says it is ready.
|
||||
switch {
|
||||
case f.holderErr != nil:
|
||||
return healthNotYet, "who holds the controller lease cannot be read: " + firstLine(f.holderErr.Error())
|
||||
case f.holder == nil:
|
||||
return healthNotYet, "no controller holds the lease"
|
||||
case f.holder.Taken.Before(since.Add(-time.Minute)):
|
||||
return healthNotYet, fmt.Sprintf("the lease is held since %s, by a controller older than the new build",
|
||||
f.holder.Taken.UTC().Format(time.RFC3339))
|
||||
case f.holder.Health == nil || !f.holder.Health.Ready:
|
||||
why := "the controller holding the lease does not say it is ready"
|
||||
if f.holder.Health != nil && f.holder.Health.Why != "" {
|
||||
why += ": " + f.holder.Health.Why
|
||||
}
|
||||
return healthNotYet, why
|
||||
}
|
||||
default:
|
||||
// A module's tools answer, where it has any and the machine runs the node tools that serve them.
|
||||
if len(m.Tools) > 0 {
|
||||
if f.servedErr != nil {
|
||||
return healthNotYet, "whether its tools are served cannot be asked: " + firstLine(f.servedErr.Error())
|
||||
}
|
||||
if s := f.served[machine]; s.runtime && !s.tools[module] {
|
||||
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
|
||||
}
|
||||
}
|
||||
// **And what it runs is stated healthy** (ADR 0240 §4): every long-running resource of it on that
|
||||
// machine, in a statement heard since the send. A resource still starting makes the judging wait.
|
||||
if h, why := moduleHealthWord(module, machine, since, f); h != healthGood {
|
||||
return h, why
|
||||
}
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is
|
||||
// the judging's facts without those conditions, on what each names among the modules the send moved,
|
||||
// and whole what holds the machine back as a whole.
|
||||
type machineWord struct {
|
||||
facts gateFacts
|
||||
on map[string]string
|
||||
whole string
|
||||
}
|
||||
|
||||
// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
|
||||
// yet reported applying the node tools it was last sent.
|
||||
const kindCoreBehind = "core-behind"
|
||||
|
||||
// aboutTheMachine sorts the conditions raised about a machine itself since a send was made there
|
||||
// (novox/hq issue 281). The gate read every one of them as the module's it was kept on: a machine-level
|
||||
// condition caused by anything else in the send — or by a tier sent one module at a time — failed that
|
||||
// module, at the bound, with a reason that was never about it.
|
||||
//
|
||||
// - one naming a module the send moved is that module's;
|
||||
// - the core being behind (D10) is the core's: of the node-engine or the node tools when the send
|
||||
// moved them — inside their settle window, which is the gate's bound — and otherwise no evidence about
|
||||
// what was sent: a send not yet applied the machine's reports already say, and a newer core build
|
||||
// that no plan sends is not this send's;
|
||||
// - anything else holds the machine back as a whole: everything the send moved there waits on it, and
|
||||
// fails with it, together, at the bound, with that reason.
|
||||
//
|
||||
// Pure.
|
||||
func aboutTheMachine(machine string, moved []string, since time.Time, f gateFacts) machineWord {
|
||||
w := machineWord{facts: f, on: map[string]string{}}
|
||||
if !f.judged || f.openErr != nil {
|
||||
return w
|
||||
}
|
||||
var core []string
|
||||
for _, m := range moved {
|
||||
if c := coreComponent(m); c == lease.ComponentEngine || c == lease.ComponentNodeTools {
|
||||
core = append(core, m)
|
||||
}
|
||||
}
|
||||
kept := make([]conditions.Condition, 0, len(f.open))
|
||||
for _, c := range f.open {
|
||||
aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine ||
|
||||
slices.Contains(c.Subject.Also, machine))
|
||||
if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) {
|
||||
kept = append(kept, c)
|
||||
continue
|
||||
}
|
||||
said := fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
||||
parts := strings.Split(c.Subject.ID, ".")
|
||||
var named []string
|
||||
for _, m := range moved {
|
||||
if slices.Contains(parts, m) {
|
||||
named = append(named, m)
|
||||
}
|
||||
}
|
||||
coreBehind := c.Kind == kindCoreBehind || strings.HasSuffix(c.Key, "."+kindCoreBehind)
|
||||
switch {
|
||||
case len(named) > 0:
|
||||
for _, m := range named {
|
||||
if _, already := w.on[m]; !already {
|
||||
w.on[m] = said
|
||||
}
|
||||
}
|
||||
case coreBehind && len(core) > 0:
|
||||
for _, m := range core {
|
||||
if _, already := w.on[m]; !already {
|
||||
w.on[m] = said + " (its settle window runs to the gate's bound)"
|
||||
}
|
||||
}
|
||||
case coreBehind:
|
||||
// Not what the send moved: said nowhere against it.
|
||||
default:
|
||||
if w.whole == "" {
|
||||
w.whole = machine + " as a whole: " + said
|
||||
}
|
||||
}
|
||||
}
|
||||
w.facts.open = kept
|
||||
return w
|
||||
}
|
||||
|
||||
// judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass
|
||||
// counted, a pass missed (and why), or the verdict. Answers the verdict once there is one.
|
||||
func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
|
||||
running []string, now time.Time) (string, error) {
|
||||
g := state.Gate
|
||||
if g == nil {
|
||||
// Judged from the send: a witness's verdict, a condition, the bound — all counted from when the
|
||||
// first machine was sent the build.
|
||||
start := now
|
||||
if state.FirstAt != nil {
|
||||
start = *state.FirstAt
|
||||
}
|
||||
var machines []string
|
||||
for _, n := range state.First {
|
||||
if slices.Contains(running, n) {
|
||||
machines = append(machines, n)
|
||||
}
|
||||
}
|
||||
g = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, From: state.Previous,
|
||||
To: state.Commit, Since: &start}
|
||||
state.Gate = g
|
||||
}
|
||||
pairs := []judged{}
|
||||
for _, n := range g.Machines {
|
||||
pairs = append(pairs, judged{module: module, node: n})
|
||||
}
|
||||
return judgeMoves(ctx, open, g, pairs, now)
|
||||
}
|
||||
|
||||
// judged is one module on one machine, as a gate judges it.
|
||||
type judged struct{ module, node string }
|
||||
|
||||
// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and
|
||||
// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is
|
||||
// wanting, and which modules), or the verdict. Answers the verdict once there is one.
|
||||
func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) {
|
||||
if g.Verdict != "" {
|
||||
return g.Verdict, nil
|
||||
}
|
||||
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
|
||||
return "", nil
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) {
|
||||
pairs = append(pairs, judged{module: c.Module, node: c.Node})
|
||||
}
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
facts, err := gatherGateFacts(ctx, open, g.Component)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
byMachine := map[string][]string{}
|
||||
for _, j := range pairs {
|
||||
byMachine[j.node] = append(byMachine[j.node], j.module)
|
||||
}
|
||||
words := map[string]machineWord{}
|
||||
for node, moved := range byMachine {
|
||||
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
|
||||
}
|
||||
worst, why := healthGood, ""
|
||||
var failing []string
|
||||
broken := map[string]bool{}
|
||||
for _, j := range pairs {
|
||||
w := words[j.node]
|
||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||
if h == healthGood {
|
||||
if on, named := w.on[j.module]; named {
|
||||
h, said = healthNotYet, on
|
||||
} else if w.whole != "" {
|
||||
h, said = healthNotYet, w.whole
|
||||
}
|
||||
}
|
||||
if h != healthGood && !slices.Contains(failing, j.module) {
|
||||
failing = append(failing, j.module)
|
||||
}
|
||||
if h == healthBroken {
|
||||
broken[j.module] = true
|
||||
}
|
||||
if h > worst {
|
||||
worst, why = h, said
|
||||
} else if h == worst && h != healthGood && why == "" {
|
||||
why = said
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
|
||||
g.Failing = failing
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
case worst == healthWaiting:
|
||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
case worst == healthNotYet:
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
if now.Sub(*g.Since) > gateBound {
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
|
||||
}
|
||||
default:
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second)), now)
|
||||
}
|
||||
}
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// decide sets a gate's verdict.
|
||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||
g.Took = now.Sub(*g.Since).Round(time.Second).String()
|
||||
}
|
||||
|
||||
// gatePassed keeps a passing build's verdict, so `plans` and the gate's probe can read it.
|
||||
func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule) {
|
||||
g := state.Gate
|
||||
err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
|
||||
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
|
||||
Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
|
||||
if err != nil && state.Build != "" {
|
||||
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
passCarried(ctx, open, p, g, module)
|
||||
fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module,
|
||||
strings.Join(g.Machines, ", "), g.Why)
|
||||
if module == catalogue.ControllerSeatName {
|
||||
carryUserList(ctx, open, p)
|
||||
}
|
||||
}
|
||||
|
||||
// carryUserList sends the machine holding the bus the user list a new controller composes, once that
|
||||
// controller passed its gate (ADR 0236). The controller's own grants travel in that list, and the old
|
||||
// controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new
|
||||
// controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR
|
||||
// 0221): then it is said, as a push would say it.
|
||||
func carryUserList(ctx context.Context, open *stores, p *inventory.Plan) {
|
||||
holder, behind, err := brokerBehind(ctx, open, nil)
|
||||
if err != nil || holder == "" || !behind {
|
||||
if err != nil {
|
||||
fmt.Printf("%s: whether the bus's user list is behind the new controller cannot be read: %v\n", p.ID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
held, err := heldMachines(ctx, open, []string{holder})
|
||||
if err != nil {
|
||||
fmt.Printf("%s: whether %s may be sent the new user list cannot be read: %v\n", p.ID, holder, err)
|
||||
return
|
||||
}
|
||||
if why, isHeld := held[holder]; isHeld {
|
||||
fmt.Printf("%s: the new controller's user list is not carried to %s, which holds the bus: %s — `push %s` "+
|
||||
"carries it\n", p.ID, holder, strings.Join(why, "; "), holder)
|
||||
return
|
||||
}
|
||||
if _, err := sendRollout(ctx, open, []string{holder}); err != nil {
|
||||
fmt.Printf("%s: the new controller's user list could not be carried to %s: %v\n", p.ID, holder, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%s: carried the new controller's user list to %s, which holds the bus\n", p.ID, holder)
|
||||
}
|
||||
|
||||
// gateFailed stops the plan at a build that failed its gate and puts the previous build back on the
|
||||
// machines it was judged on — once per build, said as a condition and an event. The plan is saved
|
||||
// before the send: a controller that is itself the build being put back does not outlive it.
|
||||
func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
|
||||
machines []string, why string) {
|
||||
inv := open.inventory
|
||||
now := time.Now().UTC()
|
||||
if state.Gate == nil {
|
||||
state.Gate = &inventory.PlanGate{Component: coreComponent(module), Machines: machines,
|
||||
From: state.Previous, To: state.Commit, Since: state.FirstAt}
|
||||
}
|
||||
g := state.Gate
|
||||
if g.Verdict == "" {
|
||||
if g.Since == nil {
|
||||
g.Since = &now
|
||||
}
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
}
|
||||
if len(g.Machines) == 0 {
|
||||
g.Machines = machines
|
||||
}
|
||||
state.Why = "failed its gate: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed its gate on %s in tier %d: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why)
|
||||
|
||||
verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous,
|
||||
Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why,
|
||||
Component: g.Component, JudgingFrom: g.Since}
|
||||
// **A send that changed nothing of the module there is no verdict on its build** (novox/hq issue
|
||||
// 280). The machine already ran this build, or one that made the same artifacts from the same
|
||||
// manifest: whatever the gate found wanting, this build did not bring it, and there is nothing to
|
||||
// put back. On 2026-10-06 such a module was marked failed, and the rollback looked for an earlier
|
||||
// build of the very commit it had failed — "no build kept" — while the build it had run before was
|
||||
// kept all along. Said, left as it is, never marked.
|
||||
if unchangedBy(ctx, inv, module, state.Previous, state.Commit) {
|
||||
g.Rollback = gateUnchanged
|
||||
state.Why = "stopped with its send; the send changed nothing of it: " + g.Why
|
||||
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was left as it was — %s already ran "+
|
||||
"%s %s, or a build identical to it, before the send, so nothing of it moved and nothing is put back",
|
||||
strings.Join(g.Machines, ", "), p.Tier, g.Why, module, strings.Join(g.Machines, ", "), module, short(state.Commit))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return
|
||||
}
|
||||
if state.Build == "" {
|
||||
// A plan from before builds were asked by id: nothing to mark, so nothing is put back by the
|
||||
// mesh — said, for a person.
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; not put back: the plan does not know which build it sent"
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
return
|
||||
}
|
||||
if err := inv.RecordGate(ctx, verdict); err != nil {
|
||||
if errors.Is(err, inventory.ErrGateKept) {
|
||||
// Already judged and acted on, by this controller before a restart or by another: never twice.
|
||||
if kept, found, _ := inv.GateOf(ctx, state.Build); found {
|
||||
g.Rollback = kept.Rollback
|
||||
}
|
||||
p.Note += "; its rollback was already made once and is not made again"
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; not put back: its verdict could not be kept, and a rollback that cannot be counted is not made — " + err.Error()
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
return
|
||||
}
|
||||
// The previous build: the one the first machine ran, from the build records.
|
||||
notBack := func(why string) {
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; NOT put back: " + why
|
||||
if err := inv.SetRollback(ctx, state.Build, inventory.NotRolledBack, g.Why+"; not put back: "+why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
sayRollback(ctx, open, module, g, why)
|
||||
}
|
||||
if state.Previous == "" {
|
||||
// A first build there: nothing ran before it, so nothing can be put back, and the machine is left
|
||||
// with it — said as that, not as a build the mesh lost.
|
||||
notBack(fmt.Sprintf("this is the first build of %s that %s was sent, or what it was sent before is not known: "+
|
||||
"there is no earlier build there to put back, so it is left with this one — `unassign` takes it off, "+
|
||||
"a newer merge replaces it", module, strings.Join(g.Machines, ", ")))
|
||||
return
|
||||
}
|
||||
failed, _, err := inv.BuildByID(ctx, state.Build)
|
||||
if err != nil {
|
||||
notBack("the failed build's record cannot be read: " + err.Error())
|
||||
return
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, module, state.Previous, failed)
|
||||
if err != nil {
|
||||
notBack("the build records cannot be read: " + err.Error())
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
notBack(fmt.Sprintf("no build of %s from %s, asked before the failed one, is among the %d newest kept to put "+
|
||||
"back", module, short(state.Previous), inventory.KeptBuilds))
|
||||
return
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, previous); err != nil {
|
||||
notBack(err.Error())
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.RollingBack
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err)
|
||||
}
|
||||
// Put back with the rest of its send, in one send per machine (issue 281).
|
||||
if b, batched := ctx.Value(rollbacksKey{}).(*rollbacks); batched {
|
||||
b.pending = append(b.pending, pendingRollback{module: module, state: state, g: g, previous: previous})
|
||||
b.modules[module] = true
|
||||
for _, n := range g.Machines {
|
||||
if !slices.Contains(b.machines, n) {
|
||||
b.machines = append(b.machines, n)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
|
||||
if err != nil {
|
||||
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.RolledBack
|
||||
p.Note += fmt.Sprintf("; put back to %s on %s", short(previous.Commit), strings.Join(sent, ", "))
|
||||
if err := inv.SetRollback(ctx, state.Build, inventory.RolledBack, g.Why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
}
|
||||
|
||||
// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one
|
||||
// send judges what it moved as one, and what it found wanting goes back in one send per machine — not
|
||||
// in a send for each module, which is the churn that failed the gate in the first place.
|
||||
type rollbacks struct {
|
||||
modules map[string]bool
|
||||
machines []string
|
||||
pending []pendingRollback
|
||||
}
|
||||
|
||||
type pendingRollback struct {
|
||||
module string
|
||||
state *inventory.PlanModule
|
||||
g *inventory.PlanGate
|
||||
previous inventory.Build
|
||||
}
|
||||
|
||||
type rollbacksKey struct{}
|
||||
|
||||
// batchingRollbacks is a context under which gateFailed registers what it puts back and leaves the send
|
||||
// to sendRollbacks.
|
||||
func batchingRollbacks(ctx context.Context) (context.Context, *rollbacks) {
|
||||
b := &rollbacks{modules: map[string]bool{}}
|
||||
return context.WithValue(ctx, rollbacksKey{}, b), b
|
||||
}
|
||||
|
||||
// sendRollbacks sends what a failed send put back, once to each machine, and says each module's rollback.
|
||||
func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *rollbacks) {
|
||||
if len(b.pending) == 0 {
|
||||
return
|
||||
}
|
||||
inv := open.inventory
|
||||
sort.Strings(b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
|
||||
var back []string
|
||||
for _, r := range b.pending {
|
||||
if err != nil {
|
||||
why := fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(r.previous.Commit), strings.Join(r.g.Machines, ", "), err, firstOf(r.g.Machines))
|
||||
r.g.Rollback = inventory.NotRolledBack
|
||||
if err := inv.SetRollback(ctx, r.state.Build, inventory.NotRolledBack, r.g.Why+"; not put back: "+why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
|
||||
}
|
||||
sayRollback(ctx, open, r.module, r.g, why)
|
||||
continue
|
||||
}
|
||||
r.g.Rollback = inventory.RolledBack
|
||||
back = append(back, r.module+" to "+short(r.previous.Commit))
|
||||
if err := inv.SetRollback(ctx, r.state.Build, inventory.RolledBack, r.g.Why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
|
||||
}
|
||||
sayRollback(ctx, open, r.module, r.g, "")
|
||||
}
|
||||
if err != nil {
|
||||
p.Note += fmt.Sprintf("; NOT put back: sending %s was refused: %v", strings.Join(b.machines, ", "), err)
|
||||
return
|
||||
}
|
||||
p.Note += fmt.Sprintf("; put back %s on %s, in one send", strings.Join(back, ", "), strings.Join(sent, ", "))
|
||||
}
|
||||
|
||||
// gateUnchanged is a failed gate's word on a module its send changed nothing of (novox/hq issue 281):
|
||||
// left as it was, never marked failed, and so no verdict on its build — `plans retry` asks it again.
|
||||
const gateUnchanged = "unchanged"
|
||||
|
||||
// unchangedBy says whether a send moving a module from one build to another changed nothing of it: the
|
||||
// same commit, builds made from the same source, or builds that made the same artifacts from the same
|
||||
// manifest. Not known is changed.
|
||||
func unchangedBy(ctx context.Context, inv *inventory.Inventory, module, from, to string) bool {
|
||||
if from == "" || to == "" {
|
||||
return false
|
||||
}
|
||||
if sameCommit(from, to) {
|
||||
return true
|
||||
}
|
||||
// Made from the same source (issue 280), or the same artifacts from the same manifest.
|
||||
f := moveFacts{}
|
||||
var err error
|
||||
if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
|
||||
return false
|
||||
}
|
||||
if f.fps, err = inv.Fingerprints(ctx); err != nil {
|
||||
return false
|
||||
}
|
||||
return f.identical(module, from, to)
|
||||
}
|
||||
|
||||
// rolledBackEvent is the body of `rolled-back` (ADR 0236): a contract, like a condition's events.
|
||||
type rolledBackEvent struct {
|
||||
Event string `json:"event"`
|
||||
At time.Time `json:"at"`
|
||||
Module string `json:"module"`
|
||||
Component string `json:"component,omitempty"`
|
||||
Machines []string `json:"machines"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to,omitempty"`
|
||||
Why string `json:"why"`
|
||||
// Rollback is rolled-back, or not-rolled-back with NotWhy.
|
||||
Rollback string `json:"rollback"`
|
||||
NotWhy string `json:"not_why,omitempty"`
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// sayRollback raises the gate's condition at once — the probe keeps it from then — and says the event.
|
||||
func sayRollback(ctx context.Context, open *stores, module string, g *inventory.PlanGate, notWhy string) {
|
||||
o := gateObservation(module, g.Component, g.Machines, g.Rollback, g.Why, notWhy, g.To, g.From)
|
||||
fmt.Println(o.Summary)
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
if d.keeper != nil {
|
||||
o.Source = gateProbe
|
||||
if _, err := d.keeper.Observe(ctx, o); err != nil {
|
||||
fmt.Printf("the gate's condition %s could not be kept: %v\n", o.Key(), err)
|
||||
}
|
||||
}
|
||||
if d.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(rolledBackEvent{Event: link.KeyRolledBack, At: time.Now().UTC(), Module: module,
|
||||
Component: g.Component, Machines: g.Machines, From: g.To, To: g.From, Why: g.Why, Rollback: g.Rollback,
|
||||
NotWhy: notWhy, Show: conditions.Condition{Key: o.Key()}.Show()})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyRolledBack, body); err != nil {
|
||||
fmt.Printf("%s's rollback could NOT be said on the bus: %v\n", module, err)
|
||||
}
|
||||
}
|
||||
|
||||
// gateObservation is a failed gate as a condition: `core.<component>.<machine>.rolled-back` for the
|
||||
// core (to-be 45 §2), `build.<module>.<machine>.rolled-back` for any other module; `rollback-failed`
|
||||
// when it could not be put back. Urgent for the core and for a build left in place; a warning for a
|
||||
// module put back, which runs what it ran before.
|
||||
func gateObservation(module, component string, machines []string, rollback, why, notWhy, failed, previous string) conditions.Observation {
|
||||
machine := strings.Join(machines, ",")
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: module + "." + machine, Kind: kindRolledBack,
|
||||
Token: kindRolledBack, Machine: firstOf(machines), Severity: conditions.Warning, Source: gateProbe,
|
||||
Summary: fmt.Sprintf("%s's build %s failed its gate on %s and was put back to %s: %s", module, short(failed),
|
||||
machine, short(previous), why)}
|
||||
if component != "" {
|
||||
o.Scope, o.ID, o.Severity = conditions.ScopeCore, component+"."+machine, conditions.Urgent
|
||||
}
|
||||
if len(machines) > 1 {
|
||||
o.Also = machines[1:]
|
||||
}
|
||||
if rollback != inventory.RolledBack && rollback != inventory.RollingBack {
|
||||
o.Kind, o.Token, o.Severity = kindRollbackFailed, kindRollbackFailed, conditions.Urgent
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
o.Summary = fmt.Sprintf("%s's build %s failed its gate on %s and was NOT put back: %s — %s", module, short(failed),
|
||||
machine, why, orNone(notWhy))
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// probeGates is DG: no build that failed its gate is left without its condition, and no witness's
|
||||
// rollback goes unsaid. Each module whose newest verdict is a failure keeps its condition; a newer build
|
||||
// that passes clears it. Each core component a machine's witness says it put back is `core.<component>.
|
||||
// <machine>.rolled-back`, urgent, until the machine's reports stop saying it.
|
||||
func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
latest, err := d.open.inventory.LatestGates(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, v := range latest {
|
||||
if v.Verdict != inventory.GateFailed {
|
||||
continue
|
||||
}
|
||||
notWhy := ""
|
||||
if v.Rollback == inventory.NotRolledBack {
|
||||
notWhy = v.Why
|
||||
}
|
||||
out = append(out, gateObservation(v.Module, v.Component, v.Machines, v.Rollback, v.Why, notWhy, v.Commit, v.Previous))
|
||||
}
|
||||
for node, list := range witnessed.all() {
|
||||
for _, r := range list {
|
||||
out = append(out, witnessObservation(node, r))
|
||||
}
|
||||
}
|
||||
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
||||
out = append(out, backlogObservation()...)
|
||||
return sortedFound(dedupeObservations(out)), nil
|
||||
}
|
||||
|
||||
// dedupeObservations keeps one observation per key, the first.
|
||||
func dedupeObservations(list []conditions.Observation) []conditions.Observation {
|
||||
seen := map[string]bool{}
|
||||
var out []conditions.Observation
|
||||
for _, o := range list {
|
||||
if seen[o.Key()] {
|
||||
continue
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules'
|
||||
// tools are served where. A variable so a test needs no runtime.
|
||||
var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) {
|
||||
inbox := conn.NewRespInbox()
|
||||
sub, err := conn.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||
}
|
||||
out := map[string]served{}
|
||||
add := func(node string) served {
|
||||
s, ok := out[node]
|
||||
if !ok {
|
||||
s = served{tools: map[string]bool{}}
|
||||
}
|
||||
return s
|
||||
}
|
||||
deadline := time.Now().Add(discoveryPatience)
|
||||
for time.Now().Before(deadline) {
|
||||
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
break
|
||||
}
|
||||
var info micro.Info
|
||||
if json.Unmarshal(msg.Data, &info) != nil {
|
||||
continue
|
||||
}
|
||||
if info.Name == broker.RuntimeModule {
|
||||
node := info.Metadata["node"]
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
s := add(node)
|
||||
s.runtime = true
|
||||
out[node] = s
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" {
|
||||
continue
|
||||
}
|
||||
node := e.Metadata["node"]
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
s := add(node)
|
||||
s.tools[e.Metadata["module"]] = true
|
||||
out[node] = s
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// gateLines is a plan's gates as `plans <id>` says them: the rollout's record.
|
||||
func gateLine(g *inventory.PlanGate) string {
|
||||
if g == nil {
|
||||
return ""
|
||||
}
|
||||
what := "judging"
|
||||
switch g.Verdict {
|
||||
case inventory.GatePassed:
|
||||
what = "passed"
|
||||
case inventory.GateFailed:
|
||||
what = "FAILED"
|
||||
}
|
||||
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
||||
if g.Component != "" {
|
||||
line += " (core: " + g.Component + ")"
|
||||
}
|
||||
if g.From != "" || g.To != "" {
|
||||
line += fmt.Sprintf(", %s → %s", short(orNone(g.From)), short(g.To))
|
||||
}
|
||||
if g.Took != "" {
|
||||
line += ", after " + g.Took
|
||||
}
|
||||
if g.Why != "" {
|
||||
line += ": " + g.Why
|
||||
} else if g.Last != "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
|
||||
} else if g.Verdict == "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||
}
|
||||
if g.Rollback != "" {
|
||||
line += "; " + g.Rollback
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// witnessObservation is a witness's verdict as a condition (ADR 0236, the host's contract):
|
||||
// `core.<component>.<node>.<outcome>`, urgent for rolled-back, not-reversible, restore-failed and
|
||||
// halted, a warning for nothing-to-restore and unwitnessed.
|
||||
func witnessObservation(node string, r lease.Rollback) conditions.Observation {
|
||||
severity := conditions.Warning
|
||||
if lease.Urgent(r.Outcome) {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
outcome := r.Outcome
|
||||
if outcome == "" {
|
||||
outcome = lease.OutcomeRolledBack
|
||||
}
|
||||
what := map[string]string{
|
||||
lease.OutcomeRolledBack: "and put back " + short(orNone(r.To)),
|
||||
lease.OutcomeNotReversible: "and left it: it is not reversible",
|
||||
lease.OutcomeNothingToRestore: "and had nothing to put back",
|
||||
lease.OutcomeRestoreFailed: "and could not put the previous build back",
|
||||
lease.OutcomeUnwitnessed: "and could not judge it at all",
|
||||
lease.OutcomeHalted: "and gave up: the build before it fails too",
|
||||
}[outcome]
|
||||
return conditions.Observation{Scope: conditions.ScopeCore, ID: r.Component + "." + node, Kind: kindRolledBack,
|
||||
Token: outcome, Machine: node, Severity: severity,
|
||||
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
||||
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).
|
||||
|
||||
// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
|
||||
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
|
||||
// sent and reports it — with the gate's bounds shortened to judge in three steps.
|
||||
type gateMesh struct {
|
||||
open *stores
|
||||
sent [][]string
|
||||
health map[string]health // per machine, what a judging finds; healthy when unsaid
|
||||
keeper *conditions.Keeper
|
||||
told *conditions.Told
|
||||
}
|
||||
|
||||
func aGateMesh(t *testing.T) *gateMesh {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register := func(commit string, asked time.Time) {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
||||
Head: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c1", time.Now().Add(-2*time.Hour))
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register("c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// Every send: recorded with the build the module is at, applied and reported by the machine.
|
||||
n := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
g.sent = append(g.sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
// What a judging reads: the store's reports, and a health the test says per machine.
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if g.health[r.Node] == healthBroken {
|
||||
r.Outcome = inventory.OutcomeFailed
|
||||
}
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
for node, h := range g.health {
|
||||
if h == healthNotYet {
|
||||
f.rolledBack[node] = nil
|
||||
r := f.reports[node]
|
||||
r.Current = false
|
||||
f.reports[node] = r
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||
gateSettle, gateEvery = 0, time.Hour
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// A module's build that fails its gate on the first machine is put back there — the previous build
|
||||
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
||||
// condition and an event, never registered or rolled back again.
|
||||
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v, not the first machine alone", g.sent)
|
||||
}
|
||||
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
|
||||
}
|
||||
|
||||
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
p := g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
|
||||
gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
|
||||
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
|
||||
t.Fatalf("anchor was last sent %v, not the previous build", sent)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
|
||||
t.Fatalf("laptop was sent the failed build: %v", sent)
|
||||
}
|
||||
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
|
||||
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
|
||||
}
|
||||
|
||||
// Said: a condition for the operator, and the event.
|
||||
open, err := g.keeper.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var key string
|
||||
for _, c := range open {
|
||||
if c.Kind == kindRolledBack {
|
||||
key = c.Key
|
||||
}
|
||||
}
|
||||
if key != "build.app.anchor.rolled-back" {
|
||||
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
|
||||
}
|
||||
saidIt := false
|
||||
for _, e := range g.told.Said() {
|
||||
saidIt = saidIt || e.Event == link.KeyRolledBack
|
||||
}
|
||||
if !saidIt {
|
||||
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
|
||||
}
|
||||
|
||||
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
|
||||
// heard again is not registered.
|
||||
advancePlans(ctx, g.open)
|
||||
state := p.Modules["app"]
|
||||
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
|
||||
if len(g.sent) != 2 {
|
||||
t.Fatalf("sent again after the rollback: %v", g.sent)
|
||||
}
|
||||
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"})})
|
||||
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("the failed build was registered again: %v", err)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module moved to %s", current["app"].Commit)
|
||||
}
|
||||
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
||||
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
|
||||
}
|
||||
|
||||
// The probe keeps the condition while the newest verdict is the failure.
|
||||
obs, err := probeGates(ctx, doctorFrom)
|
||||
if err != nil || len(obs) != 1 || obs[0].Key() != key {
|
||||
t.Fatalf("the gate's probe found %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
|
||||
// the rest are sent, the verdict kept, and the plan done.
|
||||
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v", g.sent)
|
||||
}
|
||||
gate := p.Modules["app"].Gate
|
||||
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
|
||||
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
||||
}
|
||||
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
advancePlans(ctx, g.open)
|
||||
g.health["anchor"] = healthNotYet
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, g.open)
|
||||
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
|
||||
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
|
||||
}
|
||||
gateBound = -time.Second
|
||||
advancePlans(ctx, g.open)
|
||||
p := g.plan(t)
|
||||
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
|
||||
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
|
||||
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
|
||||
}
|
||||
}
|
||||
|
||||
// The health a judging finds, per core component and for a module.
|
||||
func TestTheHealthDefinitions(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
applied := func(node string) gateFacts {
|
||||
at := now
|
||||
return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
||||
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
|
||||
}
|
||||
m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}
|
||||
|
||||
f := applied("anchor")
|
||||
f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
|
||||
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
|
||||
t.Errorf("a module whose tools are not served is %v (%s)", h, why)
|
||||
}
|
||||
f.served["anchor"].tools["app"] = true
|
||||
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("a module applied with its tools served is %v (%s)", h, why)
|
||||
}
|
||||
// A condition raised about it on that machine since it was sent: not yet healthy.
|
||||
f.judged = true
|
||||
f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
|
||||
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("a module with a new condition about it is %v", h)
|
||||
}
|
||||
f.open[0].Raised = since.Add(-time.Hour)
|
||||
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("a condition older than the send counted against it: %v", h)
|
||||
}
|
||||
// A witness that put it back: broken.
|
||||
f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
|
||||
Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
||||
t.Errorf("a component a witness put back is %v", h)
|
||||
}
|
||||
// The node tools answer, or not.
|
||||
f = applied("anchor")
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("node tools not answering are %v", h)
|
||||
}
|
||||
f.served["anchor"] = served{runtime: true}
|
||||
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("node tools answering are %v", h)
|
||||
}
|
||||
// The controller: the lease held since the send, by a controller that says it is ready.
|
||||
f = applied("control")
|
||||
f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
|
||||
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
|
||||
t.Errorf("a lease held by a controller older than the build is %v", h)
|
||||
}
|
||||
f.holder.Taken = now
|
||||
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
|
||||
t.Errorf("a new controller holding the lease and ready is %v", h)
|
||||
}
|
||||
f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
|
||||
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
|
||||
!strings.Contains(why, "first run") {
|
||||
t.Errorf("a controller not ready is %v (%s)", h, why)
|
||||
}
|
||||
// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
|
||||
// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
|
||||
f = applied("anchor")
|
||||
stated := func(state, reason string, heard time.Time) {
|
||||
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
|
||||
Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
|
||||
{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
|
||||
}
|
||||
plain := catalogue.Manifest{Module: "app"}
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
|
||||
}
|
||||
stated(link.StateStarting, "", now)
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
|
||||
t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
|
||||
}
|
||||
stated(link.StateUnhealthy, "restarting", now)
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
|
||||
t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
|
||||
}
|
||||
stated(link.StateHealthy, "", since.Add(-time.Minute))
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
|
||||
}
|
||||
stated(link.StateHealthy, "", now)
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
||||
t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
|
||||
}
|
||||
f.healthErr = errors.New("the store is away")
|
||||
if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
||||
t.Errorf("health that cannot be read is never read as healthy: %v", h)
|
||||
}
|
||||
// And the condition it raises after the send holds it, as every condition about it does.
|
||||
f.healthErr = nil
|
||||
f.judged = true
|
||||
f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
|
||||
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
|
||||
t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
|
||||
}
|
||||
|
||||
// A machine that refused what it was sent: broken.
|
||||
f = applied("anchor")
|
||||
r := f.reports["anchor"]
|
||||
r.Outcome = inventory.OutcomeRefused
|
||||
f.reports["anchor"] = r
|
||||
if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
||||
t.Errorf("a refusal is %v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
|
||||
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
|
||||
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
|
||||
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
|
||||
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := inv.UpgradeOf(ctx, "nats")
|
||||
if err != nil || u.RollOut || u.From != catalogue.FromBus {
|
||||
t.Fatalf("the bus's policy is %+v %v", u, err)
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
|
||||
t.Fatalf("a person rolled the bus out: %v", err)
|
||||
}
|
||||
var sent [][]string
|
||||
was := sendRollout
|
||||
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
now := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
|
||||
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
|
||||
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
|
||||
}
|
||||
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
|
||||
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
|
||||
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
|
||||
}
|
||||
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
|
||||
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
|
||||
t.Fatalf("a send to the bus's machine was not refused: %v", err)
|
||||
}
|
||||
// A rebuild that made the same artifacts is no move.
|
||||
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
|
||||
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
|
||||
b.Asked, b.At = time.Now(), time.Now()
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
|
||||
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
|
||||
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: "registry/nats@sha256:new"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
|
||||
// first by the bus machine's backup holder.
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
|
||||
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
|
||||
}
|
||||
wasSnapshot := takeBusSnapshot
|
||||
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
|
||||
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
|
||||
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
|
||||
}
|
||||
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
|
||||
return node + "'s restore point of " + module, nil
|
||||
}
|
||||
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("the step sent %v, not the bus's machine", sent)
|
||||
}
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
|
||||
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
|
||||
t.Fatalf("the step is %+v %v %v", s, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
|
||||
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
|
||||
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
for _, name := range []string{"gone", "kept"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
|
||||
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
|
||||
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a deleted module was asked to build: %v", *asked)
|
||||
}
|
||||
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
|
||||
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := shelf["gone"]; still {
|
||||
t.Fatal("a deleted module nothing holds was not forgotten")
|
||||
}
|
||||
|
||||
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
|
||||
asked0 := time.Now().UTC().Add(-time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
|
||||
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
|
||||
"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
|
||||
p, _ := inv.PlanByID(ctx, "plan-deleted")
|
||||
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
|
||||
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func nodeID(t *testing.T, open *stores, name string) string {
|
||||
t.Helper()
|
||||
n, err := open.inventory.NodeByName(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n.ID
|
||||
}
|
||||
|
||||
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
|
||||
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
|
||||
// gone with the first report that no longer carries it.
|
||||
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
d := &doctor{open: open}
|
||||
at := time.Now().UTC()
|
||||
witnessed.heard("control", []lease.Rollback{
|
||||
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
|
||||
Why: "the new controller did not take the lease within 60s", At: at},
|
||||
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
|
||||
}, at)
|
||||
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
|
||||
obs, err := probeGates(t.Context(), d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]conditions.Severity{}
|
||||
for _, o := range obs {
|
||||
got[o.Key()] = o.Severity
|
||||
}
|
||||
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
|
||||
"core.node-tools.control.unwitnessed": conditions.Warning}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("the witness's verdicts are %v", got)
|
||||
}
|
||||
witnessed.heard("control", nil, time.Now())
|
||||
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
|
||||
//
|
||||
// The serving controller hears every report; a clean one about the declaration a machine was last
|
||||
// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is
|
||||
// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
|
||||
// so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log
|
||||
// is read as the count of repairs a healer is wanted for.
|
||||
|
||||
// SecretReplaced is the controller seat's fact that a value given by hand was replaced
|
||||
// (link.KeySecretReplaced). Never the value: neither the old one, which the mesh cannot read,
|
||||
// nor the new one, sealed to the machine as it was made.
|
||||
type SecretReplaced struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
Given time.Time `json:"given"`
|
||||
// Sent are the machines sent so the module starts again on the new value; Unsent says why
|
||||
// they could not be, in which case the next push carries it.
|
||||
Sent []string `json:"sent"`
|
||||
Unsent string `json:"unsent,omitempty"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// givenEvents is where the serving controller states it; nil in a command.
|
||||
var givenEvents link.Bus
|
||||
|
||||
// replacing keeps one replacement per machine at a time: two reports arriving together find the
|
||||
// same rows, and the store's claim makes one of them the replacer, but the sends need not race.
|
||||
var replacing sync.Map
|
||||
|
||||
// startedWell says a report is a machine's clean account of a declaration: everything applied,
|
||||
// nothing failed or refused. Whether it is the declaration last sent is the store's to answer.
|
||||
func startedWell(report link.Report) bool {
|
||||
return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil
|
||||
}
|
||||
|
||||
const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
|
||||
|
||||
// replaceGiven replaces what the report makes due, sends the machines, and says so.
|
||||
func replaceGiven(ctx context.Context, open *stores, report link.Report) {
|
||||
if _, busy := replacing.LoadOrStore(report.Node, true); busy {
|
||||
return
|
||||
}
|
||||
defer replacing.Delete(report.Node)
|
||||
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared)
|
||||
if err != nil {
|
||||
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err)
|
||||
}
|
||||
for _, r := range replaced {
|
||||
said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
|
||||
Sent: r.Machines, Why: givenWhy}
|
||||
log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; sending %s",
|
||||
r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
|
||||
if err := sendTo(ctx, open, r.Machines); err != nil {
|
||||
said.Sent, said.Unsent = nil, err.Error()
|
||||
log.Printf("the new %q of %s is sealed and not yet delivered to %s — the next push carries it: %v",
|
||||
r.Name, r.Module, strings.Join(r.Machines, ", "), err)
|
||||
}
|
||||
stateReplaced(ctx, said)
|
||||
}
|
||||
}
|
||||
|
||||
func stateReplaced(ctx context.Context, said SecretReplaced) {
|
||||
if givenEvents == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(said)
|
||||
if err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
return
|
||||
}
|
||||
stating, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := givenEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySecretReplaced, body); err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A rotation asked through the seat carries why to the command, which records it in the hand-act
|
||||
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
|
||||
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
|
||||
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password --why leaked into logs --cause leaked" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
|
||||
t.Fatalf("without why: %v %v", argv, err)
|
||||
}
|
||||
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
|
||||
t.Fatalf("why beside a provision was passed over: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that
|
||||
// the machine is there is not (novox/hq ADR 0228).
|
||||
func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
report link.Report
|
||||
good bool
|
||||
}{
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true},
|
||||
{link.Report{Node: "anchor"}, false},
|
||||
{link.Report{Node: "anchor", Applied: []string{"x"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false},
|
||||
} {
|
||||
if got := startedWell(c.report); got != c.good {
|
||||
t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 274: a provider is granted exactly the consumers whose own resolution binds them to
|
||||
// it — not every consumer a pair credential from it was ever made for.
|
||||
|
||||
// grantOf is the grant of one provision to one consuming module, and whether there is one at all.
|
||||
func grantOf(grants []catalogue.Grant, provision, consumer, module string) (catalogue.Grant, bool) {
|
||||
for _, g := range grants {
|
||||
if g.Provision == provision && g.Consumer == consumer && (g.From == module || g.From == "") {
|
||||
return g, true
|
||||
}
|
||||
}
|
||||
return catalogue.Grant{}, false
|
||||
}
|
||||
|
||||
// The morning after issue 273, through the stores: a consumer was bound to the store on another
|
||||
// machine, a credential from there was made, and a person pinned it back to the store beside it. Both
|
||||
// credentials are on record. The store it left is no longer granted it — so it retires it and keeps
|
||||
// its data (ADR 0230) — and says so; the store it is bound to keeps its grant; and the credential from
|
||||
// the store it left stays on record.
|
||||
func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Fatalf("pinned to the anchor and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// Pinned back beside its data, as the operator did.
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" {
|
||||
t.Fatalf("pinned back to the laptop and bound to %s", n.From)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(holders) != 2 {
|
||||
t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders)
|
||||
}
|
||||
|
||||
// The store it left: no longer granted, and said.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" {
|
||||
t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g)
|
||||
}
|
||||
if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" ||
|
||||
strings.Join(unbound[0].BoundTo, ",") != "laptop" {
|
||||
t.Fatalf("the consumer that moved is not the one said: %+v", unbound)
|
||||
}
|
||||
planned, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", planned, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := declared.Received["store"]["postgres-database"]; len(got) != 0 {
|
||||
t.Fatalf("the anchor's store is still told about %+v", got)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") ||
|
||||
!strings.Contains(said, "cleanup delete") {
|
||||
t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said)
|
||||
}
|
||||
|
||||
// The store it is bound to: granted.
|
||||
grants, _, unbound, err = grantsFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
|
||||
t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
|
||||
// And the credential from the store it left is kept: the key to the login and data held there.
|
||||
if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 {
|
||||
t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer whose resolution cannot be read is an error, never a consumer bound nowhere — withdrawing
|
||||
// a grant on that reading would take its access away (issue 152).
|
||||
func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, m := range storeManifests() {
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and
|
||||
// that is not its set failing to compose.
|
||||
laptop, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) {
|
||||
t.Fatalf("the seam this test relies on moved: %v", err)
|
||||
}
|
||||
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
|
||||
if err == nil {
|
||||
t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") {
|
||||
t.Fatalf("the error does not say whose resolution could not be read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The rule itself, on a resolution: bound is the provider a need for exactly that credential is
|
||||
// answered by, never one answered by a record, and a different local name is a different credential.
|
||||
func TestBindsFromIsTheProviderOfThatCredential(t *testing.T) {
|
||||
r := catalogue.Resolution{Needs: []catalogue.Needed{
|
||||
{Name: "postgres-database", For: "board", From: "laptop"},
|
||||
{Name: "postgres-database", For: "board", From: "laptop", Local: "reports"},
|
||||
{Name: "postgres-database", For: "wiki", From: "anchor"},
|
||||
{Name: "a-licence", For: "board", From: "the-licence", ByRecord: true},
|
||||
}}
|
||||
s := inventory.Secret{Name: "postgres-database", ConsumerModule: "board"}
|
||||
if got := r.BindsFrom(s.Name, s.ConsumerModule, s.Local); strings.Join(got, ",") != "laptop" {
|
||||
t.Fatalf("board's credential is bound to %v", got)
|
||||
}
|
||||
if got := r.BindsFrom("postgres-database", "board", "archive"); len(got) != 0 {
|
||||
t.Fatalf("a local name nothing asks for is bound to %v", got)
|
||||
}
|
||||
if got := r.BindsFrom("a-licence", "board", ""); len(got) != 0 {
|
||||
t.Fatalf("a need answered by a record is bound to %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
//
|
||||
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||
// (Phase 1).
|
||||
|
||||
// handActVerb is one verb that writes the hand-act log, and whether what it records is a repair.
|
||||
type handActVerb struct {
|
||||
Verb string
|
||||
// Decision says why an act of this verb is a person's decision by design rather than a repair a
|
||||
// healer could take over; empty for a repair.
|
||||
Decision string
|
||||
// DecidedFor limits Decision to these causes; empty, it holds for every act of the verb.
|
||||
DecidedFor []string
|
||||
}
|
||||
|
||||
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
|
||||
const causeLeakedInLogs = "leaked-in-logs"
|
||||
|
||||
// causeDrill is the cause of every act `hand-act drill` records, and the one cause `hand-act record`
|
||||
// refuses: a drill has its own verb, so whether an act was a drill is said by the verb a person chose,
|
||||
// never by a word typed into a repair's cause (novox/hq issue 292).
|
||||
const causeDrill = "drill"
|
||||
|
||||
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
|
||||
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
|
||||
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
|
||||
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
|
||||
var handActVerbs = []handActVerb{
|
||||
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
|
||||
// default (ADR 0236) exists to end.
|
||||
{Verb: "push"},
|
||||
{Verb: "plans stop"},
|
||||
{Verb: "plans close"},
|
||||
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
|
||||
// not trusted with it — either is a repair the owner should have made.
|
||||
{Verb: "plans go"},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending.
|
||||
{Verb: "conditions silence"},
|
||||
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts —
|
||||
// except a drill recorded through it before `hand-act drill` existed (2026-10-07). It refuses the
|
||||
// cause since, so no act recorded through it now carries it.
|
||||
{Verb: "hand-act record", Decision: "a drill recorded before `hand-act drill` existed: a person's " +
|
||||
"deliberate test, never a repair", DecidedFor: []string{causeDrill}},
|
||||
// A drill: something broken on purpose to see the mesh raise and clear it. A person's test, never a
|
||||
// repair, however often it is run.
|
||||
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||
// A person's decisions by design.
|
||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
||||
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
|
||||
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
|
||||
"person starts (ADR 0236)"},
|
||||
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
|
||||
"person releases it (ADR 0236)"},
|
||||
// A leak is judged by a person — which value was exposed, to whom — and its rotation is the answer
|
||||
// to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the
|
||||
// module that prints them, an issue against it, not a healer that rotates. A rotation for any other
|
||||
// cause — a credential that stopped working — counts: a schedule or a healer could take it over.
|
||||
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||
DecidedFor: []string{causeLeakedInLogs}},
|
||||
}
|
||||
|
||||
// personsDecision is whether an act in the log is a person's decision by design, by the verb that
|
||||
// recorded it (handActVerbs).
|
||||
func personsDecision(a link.HandAct) bool {
|
||||
for _, v := range handActVerbs {
|
||||
if v.Verb != a.Verb {
|
||||
continue
|
||||
}
|
||||
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// handActFlags are the flags every repairing verb takes.
|
||||
type handActFlags struct {
|
||||
why, cause, condition *string
|
||||
}
|
||||
|
||||
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||
return handActFlags{
|
||||
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||
}
|
||||
}
|
||||
|
||||
// given is whether a reason was given.
|
||||
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||
|
||||
// require refuses an act without a reason, before anything is done.
|
||||
func (f handActFlags) require(verb string) error {
|
||||
if f.given() {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||
}
|
||||
|
||||
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||
// command dials its own.
|
||||
var handActConn *nats.Conn
|
||||
|
||||
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||
func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
|
||||
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
if !f.given() {
|
||||
return
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
act = written
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||
}
|
||||
|
||||
// handActCommand is `hand-act record` and `hand-acts`.
|
||||
func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "record" {
|
||||
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||
if what == "" {
|
||||
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||
}
|
||||
if err := f.require("hand-act record"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||
"act for the same reason will use — it is how a repair done twice is found")
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(*f.cause), causeDrill) {
|
||||
return errors.New("a drill is not recorded as a repair: hand-act drill <what was done> --why <text> " +
|
||||
"records it as the person's deliberate test it is, which no healer is wanted for. Nothing was recorded")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||
}
|
||||
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||
written.Why, written.Cause)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if len(args) > 0 && args[0] == "drill" {
|
||||
return handActDrill(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-act drill <what> --why <text> " +
|
||||
"| hand-acts [--days N] [--json]")
|
||||
}
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
now := time.Now()
|
||||
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeated := link.RepeatedCauses(repairs(acts), now)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Println(")")
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
causes := make([]string, 0, len(repeated))
|
||||
for c, n := range repeated {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// handActDrill is `hand-act drill`: an act done on purpose to test the mesh — a module stopped, a
|
||||
// process killed — recorded so the conditions it raises are read as the drill they are. Its cause is
|
||||
// always causeDrill and S15 never counts it (handActVerbs).
|
||||
func handActDrill(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("hand-act drill", flag.ContinueOnError)
|
||||
why := set.String("why", "", "what the drill tests — recorded in the hand-act log (novox/hq to-be 45 §7)")
|
||||
condition := set.String("condition", "", "the key of the condition the drill is meant to raise, if any")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||
if what == "" {
|
||||
return errors.New("hand-act drill <what was done on purpose> --why <what it tests> [--condition <key>]")
|
||||
}
|
||||
if strings.TrimSpace(*why) == "" {
|
||||
return errors.New("a drill says what it tests: --why <text> (recorded in the hand-act log, novox/hq " +
|
||||
"to-be 45 §7). Nothing was recorded")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act drill", Args: []string{what}, Why: strings.TrimSpace(*why),
|
||||
Cause: causeDrill, Condition: strings.TrimSpace(*condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the drill could not be recorded: %w", err)
|
||||
}
|
||||
fmt.Printf("recorded as %s: %s drilled %q, because %q — a drill, which no healer is wanted for\n",
|
||||
written.ID, written.By, what, written.Why)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// repairs are the acts that are not a person's decision by design: what S15 counts.
|
||||
func repairs(acts []link.HandAct) []link.HandAct {
|
||||
out := make([]link.HandAct, 0, len(acts))
|
||||
for _, a := range acts {
|
||||
if !personsDecision(a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||
// the log could not be read, which status says rather than reading as none.
|
||||
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||
n := -1
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||
n = len(acts)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return -1, err.Error()
|
||||
}
|
||||
return n, ""
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||
// the verb.
|
||||
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor"}},
|
||||
{"push", map[string]any{}},
|
||||
{"plans", map[string]any{"close": "plan-1"}},
|
||||
{"plans", map[string]any{"stop": "plan-1"}},
|
||||
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||
{"command", map[string]any{"command": "push anchor"}},
|
||||
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if c.verb == "plans" && err == nil {
|
||||
// The seat composes the command line; the command refuses it, before opening anything.
|
||||
err = plansCommand(context.Background(), argv[1:])
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("consumer-reset without why: %v", err)
|
||||
}
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("hand-act record without why: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--cause") {
|
||||
t.Errorf("hand-act record without a cause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||
"plans close plan-1 --why the report will not come"},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
var ds []inventory.Duration
|
||||
for i := 1; i <= 10; i++ {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||
got := summarise(ds)
|
||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
|
||||
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||
// the cause, so a repair cannot pass for a drill by the word it gives.
|
||||
func TestADrillIsRecordedThroughItsOwnVerb(t *testing.T) {
|
||||
err := handActCommand(context.Background(), []string{"record", "stopped searxng", "--why", "a test", "--cause", "drill"})
|
||||
if err == nil || !strings.Contains(err.Error(), "hand-act drill") {
|
||||
t.Errorf("hand-act record --cause drill was not sent to the drill verb: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"drill", "stopped searxng"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("a drill without what it tests: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"drill", "--why", "a test"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "hand-act drill <what") {
|
||||
t.Errorf("a drill without what was done: %v", err)
|
||||
}
|
||||
argv, err := argvFor("drill", map[string]any{"what": "stopped searxng", "why": "ADR 0240 phase A",
|
||||
"condition": "machine.ace.module.searxng.unhealthy"})
|
||||
if want := "hand-act drill stopped searxng --why ADR 0240 phase A --condition machine.ace.module.searxng.unhealthy"; err != nil ||
|
||||
strings.Join(argv, " ") != want {
|
||||
t.Errorf("the seat's drill: %v %v, want %q", argv, err, want)
|
||||
}
|
||||
if _, err := argvFor("drill", map[string]any{"what": "stopped searxng"}); err == nil {
|
||||
t.Error("the seat's drill without why was not refused")
|
||||
}
|
||||
if repairingCommand([]string{"hand-act", "drill", "x"}) != "hand-act drill" {
|
||||
t.Error("a drill through `command` is not held to why")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,876 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The healers (novox/hq to-be 45 §7, ADR 0227 rule 7, Phase 3).
|
||||
//
|
||||
// **A known failure heals itself, under a brake, and every repair is said.** Research 031 counted the
|
||||
// repairs people made by hand in six days: a push to unstick a plan waiting on a report (four times),
|
||||
// a controller restarted to make an object again (twice), a plan closed (twice), a consumer re-made from
|
||||
// now (once). Each was the ordinary path, taken again by a person who noticed. A healer is that act,
|
||||
// registered against the one condition kind it answers, so the mesh takes it itself:
|
||||
//
|
||||
// - **its repair is the ordinary path again** — the send a push makes, the plan's own close, the
|
||||
// assertion every send makes, the consumer reset the verb makes — never a withdrawal, a deletion of
|
||||
// data or a recreation of it;
|
||||
// - **its budget** is how many acts it may take against one thing in a window, and **its settle** how
|
||||
// long after an act it leaves the observation to say whether it worked;
|
||||
// - **success is never the healer's to say.** The condition clears when the watchdog or the probe that
|
||||
// raised it no longer sees it. A healer marking its own work done would be the second opinion of a
|
||||
// fact that rule 1 forbids;
|
||||
// - **a spent budget stops it**: the condition is the operator's, urgent, with every attempt in
|
||||
// `tried`, and no healer touches it again until observation clears it;
|
||||
// - **every act is said**: begun in the store before it is made (so a controller dying mid-act has
|
||||
// still spent it), kept in the condition's `tried` as `healer Hn`, and said on the bus as the
|
||||
// controller seat's `healer-acted`. A heal is not a hand act and is never in the hand-act log —
|
||||
// which is how S15 can tell a repair the mesh made from one a person had to.
|
||||
//
|
||||
// **And the mesh-wide brake**: more than healBrakeLimit acts in an hour, all healers together, and every
|
||||
// healer stops — an urgent condition says so — until an hour has passed with none. A healer looping is
|
||||
// then at most a dozen acts, said, and never the incident itself.
|
||||
//
|
||||
// Only the controller holding the lease heals, under its epoch: a controller serving without the lease
|
||||
// (S12) heals nothing, because a repair is the one act that can always wait for the lease.
|
||||
|
||||
// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour.
|
||||
const (
|
||||
healBrakeLimit = 12
|
||||
healBrakeWindow = time.Hour
|
||||
)
|
||||
|
||||
// healEvery is how often the healers look at what is open.
|
||||
var healEvery = 30 * time.Second
|
||||
|
||||
// What raises the healers' own conditions, and their kinds.
|
||||
const (
|
||||
sourceHealers = "healers"
|
||||
kindHealersBraked = "healers-braked"
|
||||
kindConsumerBehind = "consumer-behind"
|
||||
kindHealersBlind = "probe-failed"
|
||||
)
|
||||
|
||||
// healerRow is one row of the registry.
|
||||
type healerRow struct {
|
||||
ID string
|
||||
// Kinds are the condition kinds it answers: from the signals table, the probe registry, or an event.
|
||||
Kinds []string
|
||||
// Condition, Repair, Then, From are the row in words, as to-be 45 §7 and `healers` say it.
|
||||
Condition string
|
||||
Repair string
|
||||
Then string
|
||||
From string
|
||||
// Budget acts against one budget key within Window; Settle after an act before the next, or the
|
||||
// escalation, so the observation has had its turn to say whether it worked.
|
||||
Budget int
|
||||
Window time.Duration
|
||||
Settle time.Duration
|
||||
// ActsIn is where the repair runs: the controller, or a module that repairs its own (H5).
|
||||
ActsIn string
|
||||
// Event is what each act is said as.
|
||||
Event string
|
||||
// applies says whether this condition is one the healer may act on, and what its budget is
|
||||
// counted against; why when it is not. Reads, never acts. Nil where the repair is a module's.
|
||||
applies func(ctx context.Context, h *healing, c conditions.Condition) (budget string, ok bool, why string, err error)
|
||||
// repair is the act: what it did in words, what came of it, or an error when it could not be done.
|
||||
repair func(ctx context.Context, h *healing, c conditions.Condition) (act, said string, err error)
|
||||
}
|
||||
|
||||
// actsInController is a healer whose repair the controller makes.
|
||||
const actsInController = "controller"
|
||||
|
||||
// healerRegistry is to-be 45 §7's table, compiled in. **The registry is the design's live form**: a
|
||||
// test generated from it holds every row to a condition kind the mesh raises, a budget, a brake and its
|
||||
// event (healers_test.go).
|
||||
var healerRegistry = []healerRow{
|
||||
{ID: "H1", Kinds: []string{"sent-not-reported"},
|
||||
Condition: "a machine was sent a declaration and has not reported it (S2)",
|
||||
Repair: "ask the machine's node-engine to say again what it last applied (the `report` verb); if what " +
|
||||
"comes back is not the declaration it was sent, or nothing comes, send it its current declaration " +
|
||||
"again — what a push of that one machine does, never moving a build a policy or a plan holds back",
|
||||
Then: "resolver operator, urgent", From: "a push by hand to unstick a plan waiting on a report (230, 257, 264, 267)",
|
||||
Budget: 2, Window: 6 * time.Hour, Settle: 3 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAMachine, repair: repairReport},
|
||||
{ID: "H2", Kinds: []string{"stalled"},
|
||||
Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " +
|
||||
"exists — or already finished — every module of every tier built or failed, and every one that " +
|
||||
"rolls out sent (S3); or a delivery held past its state's bound where mesh-delivery's table lets H2 " +
|
||||
"take a transition (D14, novox/hq ADR 0239)",
|
||||
Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " +
|
||||
"done; what it asked still builds and registers — or, for a delivery, mesh-delivery's `close`, which " +
|
||||
"reads its walk again and takes only the transition its table names",
|
||||
Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)",
|
||||
Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAStalePlan, repair: repairPlan},
|
||||
{ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"},
|
||||
Condition: "a seat's holder that does not answer (D3), or a durable consumer the mesh expects and the " +
|
||||
"bus does not hold (D6, S9)",
|
||||
Repair: "assert the bus's streams, consumers and seat workers again — the assertion every send makes " +
|
||||
"(issue 208)",
|
||||
Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)",
|
||||
Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAnObject, repair: repairObjects},
|
||||
{ID: "H4", Kinds: []string{kindConsumerBehind},
|
||||
Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable",
|
||||
Repair: "re-make the consumer to deliver from now — `broker consumer-reset` (issue 248); what it drops " +
|
||||
"is caught up where the table says",
|
||||
Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)",
|
||||
Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAResettableConsumer, repair: repairConsumer},
|
||||
{ID: "H5", Kinds: []string{kindProviderFailing},
|
||||
Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing, " +
|
||||
"credentials-rejected)",
|
||||
Repair: "the provider repairs it itself through the server's own bootstrap command, checks again and " +
|
||||
"says what it did (ADR 0224 §5); the controller keeps its word as the condition",
|
||||
Then: "announced failing by the provider, braked from ten minutes doubling to six hours (ADR 0224 §5)",
|
||||
From: "the identity provider's admin reset through its bootstrap command (179)",
|
||||
// Its budget and brake are the module's own: ten minutes, doubling, to six hours.
|
||||
Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute,
|
||||
ActsIn: "the provider module (keycloak), ADR 0224 §5", Event: "provisioner.failing, provisioner.recovered"},
|
||||
}
|
||||
|
||||
// healerFor is the healer registered for a kind, and false when none acts in the controller.
|
||||
func healerFor(kind string) (healerRow, bool) {
|
||||
for _, r := range healerRegistry {
|
||||
if r.repair != nil && slices.Contains(r.Kinds, kind) {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return healerRow{}, false
|
||||
}
|
||||
|
||||
// healerNamedFor is any healer registered for a kind, wherever it acts: what S15 says beside a cause.
|
||||
func healerNamedFor(kind string) string {
|
||||
for _, r := range healerRegistry {
|
||||
if slices.Contains(r.Kinds, kind) {
|
||||
return r.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// healing is the healers' runner and what their repairs reach.
|
||||
type healing struct {
|
||||
open *stores
|
||||
keeper *conditions.Keeper
|
||||
teller conditions.Teller
|
||||
// js is the bus, for the repairs that assert or reset its objects; nil where there is none.
|
||||
js *broker.JetStream
|
||||
// epoch is the lease's gate; acting says this controller is the one acting, not one standing by.
|
||||
epoch func(ctx context.Context) (uint64, error)
|
||||
acting func() bool
|
||||
now func() time.Time
|
||||
say func(format string, args ...any)
|
||||
|
||||
// The acts, as seams a test replaces: asking a machine to report, sending it again, asserting the
|
||||
// bus's objects, resetting a consumer.
|
||||
askReport func(ctx context.Context, node string) error
|
||||
sendAgain func(ctx context.Context, node string) error
|
||||
assertObjects func(ctx context.Context) error
|
||||
resetConsumer func(stream, name string) (string, error)
|
||||
// reportWait is how long H1 waits for the machine's report after asking.
|
||||
reportWait time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
// declined is why each condition was last passed over, so it is said once and `healers` can show it.
|
||||
declined map[string]string
|
||||
paused string
|
||||
}
|
||||
|
||||
// newHealing is the serving controller's runner, its acts the real ones.
|
||||
func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Teller, js *broker.JetStream) *healing {
|
||||
h := &healing{open: open, keeper: keeper, teller: teller, js: js, acting: link.Holding,
|
||||
epoch: func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) },
|
||||
now: time.Now, say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
reportWait: 45 * time.Second, declined: map[string]string{}}
|
||||
h.askReport = func(ctx context.Context, node string) error {
|
||||
if h.js == nil {
|
||||
return errors.New("this controller is not on the bus")
|
||||
}
|
||||
return askToReport(ctx, h.js.Conn(), node)
|
||||
}
|
||||
h.sendAgain = func(ctx context.Context, node string) error {
|
||||
// **Never what a policy or a plan holds back** (ADR 0221): a send by the mesh itself is a push
|
||||
// that did not name the machine — a person's word sends a held build, a healer's does not.
|
||||
held, err := heldMachines(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if why := held[node]; len(why) > 0 {
|
||||
return fmt.Errorf("not sent again: it would move what a policy or a plan holds back (ADR 0221) — %s; "+
|
||||
"`push %s` sends it, on a person's word", strings.Join(why, "; "), node)
|
||||
}
|
||||
return sendTo(ctx, open, []string{node})
|
||||
}
|
||||
h.assertObjects = func(ctx context.Context) error {
|
||||
if h.js == nil {
|
||||
return errors.New("this controller is not on the bus")
|
||||
}
|
||||
return assertOnSend(ctx, open.inventory, h.js, " ")
|
||||
}
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
if h.js == nil {
|
||||
return "", errors.New("this controller is not on the bus")
|
||||
}
|
||||
before, after, err := h.js.ResetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("it was %d behind with %d unacknowledged; it delivers from now, %d pending", before.Pending,
|
||||
before.AckPending, after.Pending), nil
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// askToReport asks one machine's node-engine to say again what it last applied (to-be 45 §6). On core
|
||||
// NATS, fired and flushed: the answer is the machine's ordinary report, read from the store.
|
||||
func askToReport(ctx context.Context, conn *nats.Conn, node string) error {
|
||||
body, err := json.Marshal(map[string]any{"asked": time.Now().UTC(), "by": "the controller's healer H1"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := conn.Publish(broker.AskReportSubject(node), body); err != nil {
|
||||
return err
|
||||
}
|
||||
flushing, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return conn.FlushWithContext(flushing)
|
||||
}
|
||||
|
||||
// keep runs the healers until ctx ends.
|
||||
func (h *healing) keep(ctx context.Context) {
|
||||
tick := time.NewTicker(healEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
h.tick(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick is one look at what is open, and every act it calls for.
|
||||
func (h *healing) tick(ctx context.Context) {
|
||||
if h.acting != nil && !h.acting() {
|
||||
return
|
||||
}
|
||||
epoch, err := h.epoch(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
h.pause(fmt.Sprintf("this controller may not act: %v", err))
|
||||
return
|
||||
case epoch == 0:
|
||||
h.pause("this controller serves without the lease (S12): a repair waits for it")
|
||||
return
|
||||
}
|
||||
inv := h.open.inventory
|
||||
now := h.now()
|
||||
heals, err := inv.HealsSince(ctx, now.Add(-24*time.Hour))
|
||||
if err != nil {
|
||||
// Blind: nothing is done, and that is said — never read as "no heals, budgets whole".
|
||||
h.reconcile(ctx, []conditions.Observation{{Scope: conditions.ScopeProbe, ID: "healers", Token: "failed",
|
||||
Kind: kindHealersBlind, Severity: conditions.Warning,
|
||||
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
|
||||
Said: firstLine(err.Error())}})
|
||||
return
|
||||
}
|
||||
open, err := h.keeper.Open(ctx)
|
||||
if err != nil {
|
||||
h.say("the healers cannot read the open conditions, and act on nothing: %v", err)
|
||||
return
|
||||
}
|
||||
acts := actsWithin(heals, now.Add(-healBrakeWindow))
|
||||
if braked, said := brakeHolds(acts, open, now); braked {
|
||||
h.reconcile(ctx, []conditions.Observation{brakeObservation(acts, said)})
|
||||
h.pause("the mesh-wide brake holds: " + said)
|
||||
return
|
||||
}
|
||||
h.reconcile(ctx, nil)
|
||||
h.resume()
|
||||
for _, c := range open {
|
||||
row, ok := healerFor(c.Kind)
|
||||
if !ok || c.Escalated() {
|
||||
continue
|
||||
}
|
||||
budget, applies, why, err := row.applies(ctx, h, c)
|
||||
if err != nil {
|
||||
h.decline(c.Key, row.ID, "could not tell whether it applies: "+err.Error())
|
||||
continue
|
||||
}
|
||||
if !applies {
|
||||
h.decline(c.Key, row.ID, why)
|
||||
continue
|
||||
}
|
||||
h.forget(c.Key)
|
||||
spent := spentOn(heals, row, budget, now)
|
||||
if n := len(spent); n > 0 && now.Sub(spent[n-1].At) < row.Settle {
|
||||
continue // its last act is still the observation's to judge
|
||||
}
|
||||
if len(spent) >= row.Budget {
|
||||
h.escalate(ctx, row, c, budget, spent)
|
||||
continue
|
||||
}
|
||||
if len(acts) >= healBrakeLimit {
|
||||
return // the brake takes hold on the next look, said there
|
||||
}
|
||||
if h.act(ctx, row, c, budget, len(spent)) {
|
||||
acts = append(acts, inventory.Heal{At: now})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// act is one healer's act on one condition: begun in the store, made, finished, kept in the
|
||||
// condition's tried and said. False when it could not even be begun.
|
||||
func (h *healing) act(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent int) bool {
|
||||
inv := h.open.inventory
|
||||
begun, err := inv.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
|
||||
BudgetKey: budget, Act: row.Repair, At: h.now()})
|
||||
if err != nil {
|
||||
h.say("healer %s did not act on %s: %v", row.ID, c.Key, err)
|
||||
return false
|
||||
}
|
||||
act, said, err := row.repair(ctx, h, c)
|
||||
outcome := inventory.HealActed
|
||||
if err != nil {
|
||||
outcome, said = inventory.HealFailed, err.Error()
|
||||
}
|
||||
if act == "" {
|
||||
act = row.Repair
|
||||
}
|
||||
finishing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if ferr := inv.FinishHeal(finishing, begun.ID, outcome, act+" — "+said); ferr != nil {
|
||||
h.say("healer %s acted on %s and could not record what came of it: %v", row.ID, c.Key, ferr)
|
||||
}
|
||||
budgetWords := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window)
|
||||
attempt := conditions.Attempt{What: act, Outcome: outcome + ": " + said + " (" + budgetWords + ")",
|
||||
By: "healer " + row.ID}
|
||||
if _, _, terr := h.keeper.Tried(finishing, c.Key, attempt, conditions.ResolverHealer(row.ID)); terr != nil {
|
||||
h.say("healer %s acted on %s and could not keep it in the condition: %v", row.ID, c.Key, terr)
|
||||
}
|
||||
h.tell(finishing, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: act, Outcome: outcome,
|
||||
Said: said, Budget: budgetWords, Epoch: begun.Epoch})
|
||||
h.say("healer %s %s %s: %s — %s (%s)", row.ID, outcome, c.Key, act, said, budgetWords)
|
||||
return true
|
||||
}
|
||||
|
||||
// escalate is a spent budget: the condition is the operator's now, urgent, with what was tried. Said
|
||||
// once: an escalated condition is passed over by every healer until observation clears it.
|
||||
func (h *healing) escalate(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent []inventory.Heal) {
|
||||
var tried []string
|
||||
for _, s := range spent {
|
||||
tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.At.UTC().Format("15:04 MST"), firstLine(s.Said)))
|
||||
}
|
||||
said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window,
|
||||
c.Key, strings.Join(tried, "; "))
|
||||
if _, err := h.open.inventory.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
|
||||
BudgetKey: budget, Act: "handed the condition to the operator", Outcome: inventory.HealEscalated, Said: said,
|
||||
At: h.now()}); err != nil {
|
||||
h.say("healer %s could not record handing %s to the operator, and does not: %v", row.ID, c.Key, err)
|
||||
return
|
||||
}
|
||||
attempt := conditions.Attempt{What: "handed to the operator: nothing in the mesh will repair it now",
|
||||
Outcome: inventory.HealEscalated + ": " + said, By: "healer " + row.ID}
|
||||
if _, _, err := h.keeper.Escalate(ctx, c.Key, attempt); err != nil {
|
||||
h.say("healer %s could not hand %s to the operator: %v", row.ID, c.Key, err)
|
||||
}
|
||||
h.tell(ctx, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator",
|
||||
Outcome: inventory.HealEscalated, Said: said, Budget: fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window)})
|
||||
h.say("healer %s handed %s to the operator: %s", row.ID, c.Key, said)
|
||||
}
|
||||
|
||||
// healerActed is the body of `healer-acted` (to-be 45 §7): the healer, the condition, the act and its
|
||||
// outcome, and where the budget stands. **A contract**, like a condition's events: the operator-channel's
|
||||
// holder may say it.
|
||||
type healerActed struct {
|
||||
Event string `json:"event"`
|
||||
At time.Time `json:"at"`
|
||||
Healer string `json:"healer"`
|
||||
By string `json:"by"`
|
||||
Condition string `json:"condition"`
|
||||
Kind string `json:"kind"`
|
||||
Act string `json:"act"`
|
||||
// Outcome is acted, failed or escalated. Acted says the act was made, not that it worked: the
|
||||
// condition clearing says that.
|
||||
Outcome string `json:"outcome"`
|
||||
Said string `json:"said"`
|
||||
Budget string `json:"budget"`
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// tell says a heal on the bus. Not said is said in the log: the act and the condition's tried still
|
||||
// hold it.
|
||||
func (h *healing) tell(ctx context.Context, e healerActed) {
|
||||
e.Event, e.At, e.By = link.KeyHealerActed, h.now().UTC(), "healer "+e.Healer
|
||||
e.Show = "mesh-controller.conditions key=" + e.Condition
|
||||
if h.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := h.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyHealerActed, body); err != nil {
|
||||
h.say("healer %s %s %s, and that could NOT be said on the bus: %v", e.Healer, e.Outcome, e.Condition, err)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcile keeps the healers' own conditions: the brake, and their being blind.
|
||||
func (h *healing) reconcile(ctx context.Context, observed []conditions.Observation) {
|
||||
if err := h.keeper.Reconcile(ctx, sourceHealers, observed); err != nil {
|
||||
h.say("the healers' own conditions could not be kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *healing) pause(why string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.paused != why {
|
||||
h.say("the healers act on nothing: %s", why)
|
||||
}
|
||||
h.paused = why
|
||||
}
|
||||
|
||||
func (h *healing) resume() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.paused != "" {
|
||||
h.say("the healers act again")
|
||||
}
|
||||
h.paused = ""
|
||||
}
|
||||
|
||||
// decline remembers why a healer passed a condition over, said once.
|
||||
func (h *healing) decline(key, healer, why string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.declined[key] != why {
|
||||
h.say("healer %s leaves %s alone: %s", healer, key, why)
|
||||
}
|
||||
h.declined[key] = why
|
||||
}
|
||||
|
||||
func (h *healing) forget(key string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
delete(h.declined, key)
|
||||
}
|
||||
|
||||
// actsWithin are the heals since a moment that were acts — begun, made or failed; an escalation is a
|
||||
// saying, not an act, and the brake does not count it.
|
||||
func actsWithin(heals []inventory.Heal, since time.Time) []inventory.Heal {
|
||||
var out []inventory.Heal
|
||||
for _, h := range heals {
|
||||
if !h.At.Before(since) && h.Outcome != inventory.HealEscalated {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// spentOn is one healer's acts against one budget key within its window, oldest first.
|
||||
func spentOn(heals []inventory.Heal, row healerRow, budget string, now time.Time) []inventory.Heal {
|
||||
var out []inventory.Heal
|
||||
for _, h := range actsWithin(heals, now.Add(-row.Window)) {
|
||||
if h.Healer == row.ID && h.BudgetKey == budget {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake
|
||||
// already said and an act within the hour still — it lets go an hour after the last act, not the first.
|
||||
func brakeHolds(acts []inventory.Heal, open []conditions.Condition, now time.Time) (bool, string) {
|
||||
said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), healBrakeWindow, healBrakeLimit)
|
||||
if len(acts) >= healBrakeLimit {
|
||||
return true, said
|
||||
}
|
||||
held := slices.ContainsFunc(open, func(c conditions.Condition) bool { return c.Kind == kindHealersBraked })
|
||||
if held && len(acts) > 0 {
|
||||
last := acts[len(acts)-1].At
|
||||
return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said,
|
||||
last.Add(healBrakeWindow).UTC().Format("15:04 MST"))
|
||||
}
|
||||
return false, said
|
||||
}
|
||||
|
||||
// brakeObservation is the brake, as the urgent condition that says it.
|
||||
func brakeObservation(acts []inventory.Heal, said string) conditions.Observation {
|
||||
counts := map[string]int{}
|
||||
for _, a := range acts {
|
||||
if a.Healer != "" {
|
||||
counts[a.Healer+" on "+a.ConditionKey]++
|
||||
}
|
||||
}
|
||||
var most []string
|
||||
for what, n := range counts {
|
||||
most = append(most, fmt.Sprintf("%s ×%d", what, n))
|
||||
}
|
||||
sort.Strings(most)
|
||||
return conditions.Observation{Scope: conditions.ScopeMesh, ID: "healers", Token: "braked", Kind: kindHealersBraked,
|
||||
Severity: conditions.Urgent,
|
||||
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
|
||||
"healer looping, not repairing — " + said,
|
||||
Said: strings.Join(most, ", ")}
|
||||
}
|
||||
|
||||
// --- H1 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// appliesToAMachine is H1's: a machine named, its budget counted against the condition.
|
||||
func appliesToAMachine(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if c.Subject.Machine == "" {
|
||||
return "", false, "it names no machine", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairReport is H1: ask the machine to report; if what comes back is not what it was sent, or nothing
|
||||
// comes, send it again.
|
||||
func repairReport(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
node := c.Subject.Machine
|
||||
inv := h.open.inventory
|
||||
// The store's clock, as the report's time is: not the runner's, which a test moves by hand.
|
||||
asked := time.Now()
|
||||
askErr := h.askReport(ctx, node)
|
||||
current, heard := false, false
|
||||
if askErr == nil {
|
||||
deadline := time.Now().Add(h.reportWait)
|
||||
for {
|
||||
r, found, err := lastReportOf(ctx, inv, node)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if found && r.At != nil && r.At.After(asked) {
|
||||
heard, current = true, r.Current
|
||||
if current {
|
||||
break
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", "", ctx.Err()
|
||||
case <-time.After(min(2*time.Second, h.reportWait/4+time.Millisecond)):
|
||||
}
|
||||
}
|
||||
}
|
||||
if current {
|
||||
return "asked " + node + "'s node-engine to say again what it last applied",
|
||||
"it reported the declaration it was sent: its report had not reached the mesh", nil
|
||||
}
|
||||
why := "it said nothing within " + h.reportWait.String() + " — its node-engine may be older than the report verb"
|
||||
switch {
|
||||
case askErr != nil:
|
||||
why = "it could not be asked: " + askErr.Error()
|
||||
case heard:
|
||||
why = "it reported a declaration other than the one it was sent"
|
||||
}
|
||||
if err := h.sendAgain(ctx, node); err != nil {
|
||||
return "asked " + node + " to report, then sent it its current declaration again", why + "; the send failed",
|
||||
err
|
||||
}
|
||||
return "asked " + node + " to report, then sent it its current declaration again", why + "; sent again", nil
|
||||
}
|
||||
|
||||
// lastReportOf is one machine's last report beside its last send.
|
||||
func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (inventory.Reported, bool, error) {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return inventory.Reported{}, false, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r, true, nil
|
||||
}
|
||||
}
|
||||
return inventory.Reported{}, false, nil
|
||||
}
|
||||
|
||||
// --- H2 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty
|
||||
// when it is neither, which is not H2's to repair.
|
||||
func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) {
|
||||
if p.Release != nil {
|
||||
return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, newer := range recent {
|
||||
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
|
||||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded {
|
||||
continue
|
||||
}
|
||||
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
|
||||
"and branch", newer.ID, newer.Repository, short(newer.Commit)), nil
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil || (s.State != "built" && s.State != "failed") {
|
||||
return "", "", nil
|
||||
}
|
||||
if s.State == "built" && s.SentAt == nil {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if u.RollOut {
|
||||
return "", "", nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return inventory.PlanDone, "finished: every module of every tier is built or failed, and every one that rolls " +
|
||||
"out was sent — nothing is left to wait on", nil
|
||||
}
|
||||
|
||||
// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished.
|
||||
func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if c.Subject.Scope == conditions.ScopeDelivery {
|
||||
return appliesToAStalledDelivery(ctx, h, c)
|
||||
}
|
||||
p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
if !p.Open() {
|
||||
return "", false, "the plan is " + p.State + " already: its condition clears on the next look", nil
|
||||
}
|
||||
state, _, err := planStale(ctx, h.open.inventory, p)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
if state == "" {
|
||||
return "", false, "its wait is neither superseded nor finished: it waits on something still to come, " +
|
||||
"which its own signal says", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set.
|
||||
func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
if c.Subject.Scope == conditions.ScopeDelivery {
|
||||
return repairDelivery(ctx, h, c)
|
||||
}
|
||||
inv := h.open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, c.Subject.ID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if !p.Open() {
|
||||
return "closed nothing", "the plan is " + p.State + " already", nil
|
||||
}
|
||||
state, why, err := planStale(ctx, inv, p)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if state == "" {
|
||||
return "closed nothing", "its wait is no longer superseded or finished", nil
|
||||
}
|
||||
p.State = state
|
||||
p.Note = fmt.Sprintf("closed by healer H2 at tier %d: %s", p.Tier, why)
|
||||
if state != inventory.PlanDone {
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return "closed the plan " + p.ID + " (" + state + ")", why, nil
|
||||
}
|
||||
|
||||
// --- H3 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// appliesToAnObject is H3's: every holder or consumer the probe or the bus names, its budget per object.
|
||||
func appliesToAnObject(_ context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if h.assertObjects == nil {
|
||||
return "", false, "this controller is not on the bus", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairObjects is H3: the send's own assertion of every stream, consumer and seat worker.
|
||||
func repairObjects(ctx context.Context, h *healing, _ conditions.Condition) (string, string, error) {
|
||||
if err := h.assertObjects(ctx); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return "asserted the bus's streams, consumers and seat workers again",
|
||||
"every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there", nil
|
||||
}
|
||||
|
||||
// --- H4 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// resettable is why a consumer may be reset by the mesh itself, from the stream table; empty when not.
|
||||
func resettable(stream, name string) string {
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
if c.Stream == stream && c.Name == name {
|
||||
return c.Resettable
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// consumerOf is the stream and consumer a bus condition names: `<stream>.<consumer>`.
|
||||
func consumerOf(c conditions.Condition) (string, string, bool) {
|
||||
stream, name, found := strings.Cut(c.Subject.ID, ".")
|
||||
return stream, name, found && stream != "" && name != ""
|
||||
}
|
||||
|
||||
// appliesToAResettableConsumer is H4's: only a consumer the stream table marks resettable.
|
||||
func appliesToAResettableConsumer(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
stream, name, ok := consumerOf(c)
|
||||
if !ok {
|
||||
return "", false, "it names no consumer", nil
|
||||
}
|
||||
if resettable(stream, name) == "" {
|
||||
return "", false, fmt.Sprintf("%s on %s is not marked resettable in the stream table: what a reset drops, "+
|
||||
"nothing would catch up", name, stream), nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairConsumer is H4: `broker consumer-reset`, made by the mesh.
|
||||
func repairConsumer(_ context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
stream, name, _ := consumerOf(c)
|
||||
said, err := h.resetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return fmt.Sprintf("re-made %s on %s to deliver from now", name, stream),
|
||||
said + "; " + resettable(stream, name), nil
|
||||
}
|
||||
|
||||
// --- the verb ----------------------------------------------------------------------------------------
|
||||
|
||||
// healersCommand is `healers`: the registry, what the healers did lately, and the brake.
|
||||
func healersCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("healers", flag.ContinueOnError)
|
||||
daysFlag := set.Int("days", 7, "how many days of acts back")
|
||||
jsonFlag := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("healers [--days N] [--json]")
|
||||
}
|
||||
days, asJSON := *daysFlag, *jsonFlag
|
||||
if days <= 0 {
|
||||
return fmt.Errorf("healers --days takes a number of days, not %d", days)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
now := time.Now()
|
||||
heals, err := open.inventory.HealsSince(ctx, now.Add(-time.Duration(days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return fmt.Errorf("what the healers did cannot be read: %w", err)
|
||||
}
|
||||
conds, condErr := openConditions(ctx)
|
||||
braked, said := brakeHolds(actsWithin(heals, now.Add(-healBrakeWindow)), conds, now)
|
||||
brake := map[string]any{"holds": braked, "said": said, "limit": healBrakeLimit, "window": healBrakeWindow.String()}
|
||||
if condErr != nil {
|
||||
brake["conditions"] = "the open conditions could not be read, so whether the brake was said is not known: " +
|
||||
condErr.Error()
|
||||
}
|
||||
var rows []map[string]any
|
||||
for _, r := range healerRegistry {
|
||||
rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair,
|
||||
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
|
||||
"then": r.Then, "acts-in": r.ActsIn, "event": r.Event, "from": r.From})
|
||||
}
|
||||
if heals == nil {
|
||||
heals = []inventory.Heal{}
|
||||
}
|
||||
if asJSON {
|
||||
return printJSON(map[string]any{"healers": rows, "heals": heals, "brake": brake, "days": days,
|
||||
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"})
|
||||
}
|
||||
for _, r := range healerRegistry {
|
||||
fmt.Printf("%s %s\n → %s\n budget %d within %s; then %s (in %s)\n", r.ID, r.Condition, r.Repair, r.Budget,
|
||||
r.Window, r.Then, r.ActsIn)
|
||||
}
|
||||
fmt.Printf("\nthe brake: %s — %s\n\n", map[bool]string{true: "HOLDS, every healer has stopped", false: "off"}[braked], said)
|
||||
if len(heals) == 0 {
|
||||
fmt.Printf("no healer acted in the last %d day(s)\n", days)
|
||||
return nil
|
||||
}
|
||||
for i := len(heals) - 1; i >= 0; i-- {
|
||||
x := heals[i]
|
||||
fmt.Printf("%s %s %-9s %s\n %s\n", x.At.Local().Format("2006-01-02 15:04"), x.Healer, x.Outcome, x.ConditionKey,
|
||||
firstLine(x.Said))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldHeals removes heals past their keeping once a day, by the controller acting only.
|
||||
func forgettingOldHeals(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if link.Holding() {
|
||||
if n, err := inv.ForgetOldHeals(ctx); err != nil {
|
||||
fmt.Printf("heals older than %s could not be removed: %v\n", inventory.HealsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d heal(s) older than %s\n", n, inventory.HealsKeptFor)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// healsCount is what the healers did, counted for `status`.
|
||||
type healsCount struct {
|
||||
Acts int `json:"acts"`
|
||||
Escalated int `json:"escalated"`
|
||||
}
|
||||
|
||||
// countHeals counts acts and escalations.
|
||||
func countHeals(heals []inventory.Heal) *healsCount {
|
||||
out := &healsCount{}
|
||||
for _, h := range heals {
|
||||
if h.Outcome == inventory.HealEscalated {
|
||||
out.Escalated++
|
||||
} else {
|
||||
out.Acts++
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,640 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
|
||||
// checked"): **every row is walked.** Its kinds are ones the mesh raises — a row of the signals table, a
|
||||
// probe of the self-check, or a provider's event — it has a budget, a window and a settle inside it,
|
||||
// what happens when the budget is spent, and the event each act is said as; a healer the controller runs
|
||||
// has its applies and its repair, and an induced failure below that sees it act, say so and brake. A
|
||||
// row added without one fails, so the registry cannot grow a healer nobody has seen act.
|
||||
|
||||
// raisedKinds is every condition kind the mesh raises, and what raises it.
|
||||
func raisedKinds() map[string]string {
|
||||
out := map[string]string{kindProviderFailing: "the provisioner.failing event (ADR 0224)"}
|
||||
for _, r := range signalsTable {
|
||||
for _, k := range kindsOf(r) {
|
||||
out[k] = r.Row
|
||||
}
|
||||
}
|
||||
for _, p := range probeRegistry {
|
||||
out[p.Kind] = p.ID
|
||||
for _, k := range p.Raises {
|
||||
out[k] = p.ID
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inducedFailures are the healers seen acting in this file, by id: a row without one fails.
|
||||
var inducedFailures = map[string]string{
|
||||
"H1": "TestH1AsksAMachineToReportAndSendsItAgain",
|
||||
"H2": "TestH2ClosesAPlanAnotherHasTakenOver",
|
||||
"H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget",
|
||||
"H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers",
|
||||
}
|
||||
|
||||
func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) {
|
||||
kinds := raisedKinds()
|
||||
source, err := os.ReadFile("healers_test.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
answered := map[string]string{}
|
||||
for _, r := range healerRegistry {
|
||||
t.Run(r.ID, func(t *testing.T) {
|
||||
if seen[r.ID] {
|
||||
t.Fatalf("%s is in the registry twice", r.ID)
|
||||
}
|
||||
seen[r.ID] = true
|
||||
if len(r.Kinds) == 0 || r.Condition == "" || r.Repair == "" || r.Then == "" || r.From == "" || r.ActsIn == "" {
|
||||
t.Fatalf("%s does not say what it answers, what it does, what then, and which hand act it replaces: %+v", r.ID, r)
|
||||
}
|
||||
for _, k := range r.Kinds {
|
||||
if _, ok := kinds[k]; !ok {
|
||||
t.Errorf("%s answers %q, which nothing in the mesh raises", r.ID, k)
|
||||
}
|
||||
if other, twice := answered[k]; twice {
|
||||
t.Errorf("%q is answered by %s and %s: one healer per kind", k, other, r.ID)
|
||||
}
|
||||
answered[k] = r.ID
|
||||
}
|
||||
if r.Budget <= 0 || r.Window <= 0 || r.Settle <= 0 || r.Settle > r.Window {
|
||||
t.Errorf("%s has no budget it can spend: %d within %s, settled after %s", r.ID, r.Budget, r.Window, r.Settle)
|
||||
}
|
||||
if r.Event == "" {
|
||||
t.Errorf("%s says nothing when it acts", r.ID)
|
||||
}
|
||||
if r.ActsIn != actsInController {
|
||||
if r.repair != nil || r.applies != nil {
|
||||
t.Errorf("%s acts in %s and the controller would act for it too", r.ID, r.ActsIn)
|
||||
}
|
||||
return
|
||||
}
|
||||
if r.repair == nil || r.applies == nil {
|
||||
t.Fatalf("%s acts in the controller and has no repair or no applies", r.ID)
|
||||
}
|
||||
if r.Event != link.KeyHealerActed || !slices.Contains(broker.ControllerStates, r.Event) {
|
||||
t.Errorf("%s is said as %q, which the controller's grant does not permit", r.ID, r.Event)
|
||||
}
|
||||
if inducedFailures[r.ID] == "" {
|
||||
t.Errorf("%s has no induced failure: a healer nobody has seen act", r.ID)
|
||||
} else if !strings.Contains(string(source), "func "+inducedFailures[r.ID]+"(t *testing.T)") {
|
||||
t.Errorf("%s's induced failure %s is not a test in this file", r.ID, inducedFailures[r.ID])
|
||||
}
|
||||
})
|
||||
}
|
||||
for id := range inducedFailures {
|
||||
if !seen[id] {
|
||||
t.Errorf("an induced failure for %s, which the registry does not have", id)
|
||||
}
|
||||
}
|
||||
if healBrakeLimit <= 0 || healBrakeWindow <= 0 {
|
||||
t.Error("the mesh-wide brake holds nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// heard keeps the healer-acted events said.
|
||||
type heard struct {
|
||||
mu sync.Mutex
|
||||
acts []healerActed
|
||||
}
|
||||
|
||||
func (h *heard) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||
if seat != conditions.Seat || event != link.KeyHealerActed {
|
||||
return errors.New("said under the wrong seat or name: " + seat + " " + event)
|
||||
}
|
||||
var e healerActed
|
||||
if err := json.Unmarshal(body, &e); err != nil {
|
||||
return err
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.acts = append(h.acts, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *heard) said() []healerActed {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]healerActed(nil), h.acts...)
|
||||
}
|
||||
|
||||
// testClock is a moment a test moves by hand.
|
||||
type testClock struct {
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func (c *testClock) now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.at
|
||||
}
|
||||
|
||||
func (c *testClock) pass(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.at = c.at.Add(d)
|
||||
}
|
||||
|
||||
// healingOn is a runner over a mesh's stores and its condition store, under epoch 57, every act a
|
||||
// fake that fails the test unless the test gives it.
|
||||
func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) {
|
||||
t.Helper()
|
||||
if conditionsFrom == nil {
|
||||
t.Fatal("the mesh has no condition store")
|
||||
}
|
||||
told, clock := &heard{}, &testClock{at: time.Now()}
|
||||
// The store's gate, as the serving controller's is the lease's (stores.go).
|
||||
open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
|
||||
h := &healing{open: open, keeper: conditionsFrom, teller: told,
|
||||
epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now,
|
||||
say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond,
|
||||
declined: map[string]string{}}
|
||||
h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil }
|
||||
h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil }
|
||||
h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil }
|
||||
h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil }
|
||||
return h, told, clock
|
||||
}
|
||||
|
||||
// sentNotReported raises S2 for a machine, as the watchdog does.
|
||||
func sentNotReported(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "sent-not-reported", Machine: node,
|
||||
Severity: conditions.Warning, Summary: node + " was sent a declaration and has not reported it", Source: "S2"}
|
||||
if _, err := conditionsFrom.Observe(t.Context(), o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return o.Key()
|
||||
}
|
||||
|
||||
// **H1, the commonest hand act** (031/01 §f: a push by hand to unstick a plan waiting on a report, four
|
||||
// times): the machine is asked to report; a report that names what it was sent is all it takes, and one
|
||||
// that does not — or none — is a send of its current declaration again. Each act kept in `tried` as
|
||||
// `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more.
|
||||
func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := sentNotReported(t, "laptop")
|
||||
|
||||
// First: the report had not reached the mesh, and asking for it brings it.
|
||||
asked := 0
|
||||
h.askReport = func(ctx context.Context, node string) error {
|
||||
asked++
|
||||
if node != "laptop" {
|
||||
t.Errorf("asked %s", node)
|
||||
}
|
||||
_, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied,
|
||||
At: time.Now().Add(time.Second), Declared: "d2"})
|
||||
return err
|
||||
}
|
||||
h.tick(ctx)
|
||||
c, found, err := conditionsFrom.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err)
|
||||
}
|
||||
if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") ||
|
||||
c.Resolver != "healer:H1" {
|
||||
t.Fatalf("after the first act: asked %d, %+v", asked, c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed ||
|
||||
acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 {
|
||||
t.Fatalf("the act was not said as healer-acted: %+v", acts)
|
||||
}
|
||||
|
||||
// Within its settle, nothing more: the observation has its turn.
|
||||
clock.pass(time.Minute)
|
||||
h.tick(ctx)
|
||||
if asked != 1 {
|
||||
t.Fatalf("acted again inside the settle: asked %d", asked)
|
||||
}
|
||||
|
||||
// Second: the machine says nothing, so it is sent again.
|
||||
clock.pass(3 * time.Minute)
|
||||
sent := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(_ context.Context, node string) error { sent++; return nil }
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 {
|
||||
t.Fatalf("the second act: asked %d, sent %d", asked, sent)
|
||||
}
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") {
|
||||
t.Fatalf("tried %+v", c.Tried)
|
||||
}
|
||||
|
||||
// The budget is spent: the operator's, urgent, said; and no healer touches it again.
|
||||
clock.pass(4 * time.Minute)
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 ||
|
||||
!strings.Contains(c.Tried[2].Outcome, "budget of 2") {
|
||||
t.Fatalf("not handed to the operator: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("the escalation was not said: %+v", acts)
|
||||
}
|
||||
clock.pass(time.Hour)
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 || len(told.said()) != 3 {
|
||||
t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said()))
|
||||
}
|
||||
heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed ||
|
||||
heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 {
|
||||
t.Fatalf("the heals kept: %+v %v", heals, err)
|
||||
}
|
||||
// And a heal is not a hand act: what S15 counts never sees it.
|
||||
for _, x := range heals {
|
||||
if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") {
|
||||
t.Errorf("a heal reads as a hand act: %+v", x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing.
|
||||
func TestNoHealerActsWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
sentNotReported(t, "laptop")
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, nil }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 57, nil }
|
||||
h.acting = func() bool { return false }
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted without the lease: %+v", told.said())
|
||||
}
|
||||
}
|
||||
|
||||
// **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour
|
||||
// after the last.
|
||||
func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
for i := 0; i < healBrakeLimit; i++ {
|
||||
if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent",
|
||||
Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed,
|
||||
At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sentNotReported(t, "laptop")
|
||||
h.tick(ctx) // the fakes fail the test if anything acts
|
||||
braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked")
|
||||
if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") {
|
||||
t.Fatalf("the brake was not said: %+v %v", braked, err)
|
||||
}
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted under the brake: %+v", told.said())
|
||||
}
|
||||
// Past the hour of the first act, still held: it lets go an hour after the last.
|
||||
clock.pass(30 * time.Minute)
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held {
|
||||
t.Fatal("the brake let go before an hour had passed since the last act")
|
||||
}
|
||||
clock.pass(31 * time.Minute)
|
||||
asked := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(context.Context, string) error { return nil }
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held {
|
||||
t.Fatal("the brake held an hour after the last act")
|
||||
}
|
||||
if asked != 1 {
|
||||
t.Fatalf("the healers did not act again after the brake let go: asked %d", asked)
|
||||
}
|
||||
}
|
||||
|
||||
// **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is
|
||||
// still to come: that one is its own signal's, not a healer's.
|
||||
func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
created := time.Now().UTC().Add(-time.Hour)
|
||||
older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}}
|
||||
waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}
|
||||
for _, p := range []*inventory.Plan{&older, &waiting, &newer} {
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"plan-old", "plan-other"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled",
|
||||
Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
closed, err := open.inventory.PlanByID(ctx, "plan-old")
|
||||
if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") ||
|
||||
!strings.Contains(closed.Note, "plan-new") {
|
||||
t.Fatalf("the superseded plan: %+v %v", closed, err)
|
||||
}
|
||||
if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding {
|
||||
t.Fatalf("a plan still waiting was closed: %+v", other)
|
||||
}
|
||||
if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf {
|
||||
t.Fatalf("a plan H2 does not repair was touched: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// Finished: every module built, none rolled out unsent — closed as done.
|
||||
done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created,
|
||||
State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") {
|
||||
t.Fatalf("a finished plan reads %q %q %v", state, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said
|
||||
// and left — what a reset drops, nothing would catch up for it.
|
||||
func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, _, _ := healingOn(t, open)
|
||||
marked := 0
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
if c.Resettable != "" {
|
||||
marked++
|
||||
if c.Stream != broker.EventsStream || c.Name != broker.ControllerName {
|
||||
t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
if marked != 1 {
|
||||
t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked)
|
||||
}
|
||||
for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||
Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx) // the fake reset fails the test if it is called
|
||||
reset := ""
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
reset = stream + "." + name
|
||||
return "it was 1500 behind", nil
|
||||
}
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller",
|
||||
Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if reset != "EVENTS.controller" {
|
||||
t.Fatalf("reset %q", reset)
|
||||
}
|
||||
}
|
||||
|
||||
// **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3
|
||||
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
|
||||
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
|
||||
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h, told, clock := healingOn(t, open)
|
||||
h.js = js
|
||||
h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") }
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const key = "bus.NODES.laptop.missing"
|
||||
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" {
|
||||
t.Fatalf("the deleted consumer was not raised: %+v", c)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil {
|
||||
t.Fatalf("H3 did not make the consumer again: %v", err)
|
||||
}
|
||||
c, still, _ := conditionsFrom.Get(ctx, key)
|
||||
if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" {
|
||||
t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c)
|
||||
}
|
||||
probe()
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H3 repaired")
|
||||
}
|
||||
// Kept in the history as the keeper says it, a moment later.
|
||||
var cleared *conditions.Event
|
||||
for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) {
|
||||
history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range history {
|
||||
if history[i].Key == key && history[i].Change == conditions.ChangeCleared {
|
||||
cleared = &history[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 {
|
||||
t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared)
|
||||
}
|
||||
|
||||
// Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops.
|
||||
clock.pass(10 * time.Minute)
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil }
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 {
|
||||
t.Fatalf("the spent budget was not handed to the operator: %+v", c)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
clock.pass(2 * time.Hour)
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 2 {
|
||||
t.Fatal("a healer acted on what the operator holds")
|
||||
}
|
||||
}
|
||||
|
||||
// **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it
|
||||
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
|
||||
// receives what comes next: a reset that left the controller deaf would be the incident.
|
||||
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none.
|
||||
events := make(chan *nats.Msg, 64)
|
||||
sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
followed := broker.ControllerFollows[0]
|
||||
for i := 0; i < consumerFarBehind+200; i++ {
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
probe()
|
||||
const key = "bus.EVENTS.controller.behind"
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind {
|
||||
t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c)
|
||||
}
|
||||
h, told, _ := healingOn(t, open)
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
before, after, err := js.ResetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) +
|
||||
" pending now", nil
|
||||
}
|
||||
h.tick(ctx)
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
if found := probe(); len(found) != 0 {
|
||||
t.Fatalf("after the reset the probe still finds %+v", found)
|
||||
}
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H4 repaired")
|
||||
}
|
||||
// What comes next still reaches the controller.
|
||||
for len(events) > 0 {
|
||||
<-events
|
||||
}
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case m := <-events:
|
||||
if strings.Contains(string(m.Data), "after") {
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
case <-deadline:
|
||||
t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
|
||||
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := askToReport(t.Context(), conn, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err := asked.NextMsg(5 * time.Second)
|
||||
if err != nil || !strings.Contains(string(msg.Data), "healer H1") {
|
||||
t.Fatalf("the machine heard %v, %v", msg, err)
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
|
||||
if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") ||
|
||||
slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") {
|
||||
t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every catalogue module that runs something long-lived declares how it is ready (novox/hq ADR 0240 rule
|
||||
// 8): `module check` warns and counts the undeclared before the date, and refuses them from it.
|
||||
func TestModuleCheckCountsTheUndeclaredAndRefusesThemFromTheDate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
digest := "@sha256:" + strings.Repeat("a", 64)
|
||||
path := filepath.Join(dir, "module.json")
|
||||
os.WriteFile(path, []byte(`{"module":"web","listens":[{"name":"web","port":80,"from":"mesh"}],"resources":[
|
||||
{"id":"server","type":"container","name":"web","image":"registry.example/web`+digest+`","ports":["80"],
|
||||
"health":{"kind":"http","endpoint":"web"}},
|
||||
{"id":"worker","type":"container","name":"web-worker","image":"registry.example/web`+digest+`"},
|
||||
{"id":"seed","type":"container","name":"web-seed","image":"registry.example/web`+digest+`","run-once":true}]}`), 0o600)
|
||||
defer func() { checkNow = time.Now }()
|
||||
|
||||
checkNow = func() time.Time { return catalogue.HealthRequiredFrom.Add(-time.Hour) }
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||
t.Fatalf("refused before the date: %v\n%s", err, out.String())
|
||||
}
|
||||
for _, want := range []string{"ready: server by http / on web every 30s", "WARNING: worker stay(s) up",
|
||||
catalogue.HealthRequiredFrom.Format("2006-01-02"), UndeclaredHealthLine + " 1"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the check does not say %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
checkNow = func() time.Time { return catalogue.HealthRequiredFrom }
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{path}, &out); err == nil {
|
||||
t.Fatalf("a long-running resource without health passed after the date:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "web: worker stays up and does not say how it is ready") {
|
||||
t.Errorf("the refusal does not name the resource:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// The core components' health definitions, as probes in the self-check's registry (novox/hq to-be 45
|
||||
// §8, §4 `H-*`). The same definitions judge a core component's new build on its first machine (the
|
||||
// gate, gate.go); here they are run against every machine on the self-check's schedule, so a core
|
||||
// component that stops being healthy between upgrades is said too.
|
||||
//
|
||||
// controller holds the lease, and says it is ready: its self-check ran, `status` answered in bound
|
||||
// node-engine has reported its current declaration, under a build the mesh delivered
|
||||
// node tools announced, and answer the bus's discovery
|
||||
// bus every stream and durable consumer the mesh defines is there, and a request crosses the
|
||||
// bus to every machine's node tools and back
|
||||
const kindCoreUnhealthy = "core-unhealthy"
|
||||
|
||||
// probeControllerHealth is H-controller: the lease is held, fresh, by a controller that says it is
|
||||
// ready — or one that started less than the witness's bound ago.
|
||||
func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
h, found, err := theLease.holder(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
unhealthy := func(why string) []conditions.Observation {
|
||||
node := d.host
|
||||
if found && h.Host != "" {
|
||||
node = h.Host
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: lease.ComponentController + "." + node,
|
||||
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Urgent,
|
||||
Summary: "the controller is not healthy: " + why}}
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
return unhealthy("nobody holds the controller lease"), nil
|
||||
case time.Since(h.Renewed) > lease.FreshWithin:
|
||||
return unhealthy(fmt.Sprintf("the lease was last renewed %s ago", time.Since(h.Renewed).Round(time.Second))), nil
|
||||
case (h.Health == nil || !h.Health.Ready) && time.Since(h.Taken) > lease.ReadyWithin:
|
||||
why := "the controller holding the lease does not say it is ready"
|
||||
if h.Health != nil && h.Health.Why != "" {
|
||||
why += ": " + h.Health.Why
|
||||
}
|
||||
return unhealthy(why), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the
|
||||
// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send.
|
||||
func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
return machinesHealth(ctx, d, hostModule)
|
||||
}
|
||||
|
||||
// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them
|
||||
// announced, answering the bus's discovery.
|
||||
func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
return machinesHealth(ctx, d, broker.RuntimeModule)
|
||||
}
|
||||
|
||||
// machinesHealth judges a component on every machine running it and heard from, by its health
|
||||
// definition, as the gate does — with no condition taken as the build's doing.
|
||||
func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
running, err := inv.Running(ctx, component)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := gatherGateFacts(ctx, d.open, coreComponent(component))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.judged = false
|
||||
heard := heardMachines(d)
|
||||
var out []conditions.Observation
|
||||
for _, node := range running {
|
||||
if !heard[node] {
|
||||
continue // a machine not heard from is S1's
|
||||
}
|
||||
if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound {
|
||||
continue // inside the bound of a send: S2's, and the gate's
|
||||
}
|
||||
// What the machine did with what it was sent is not the component's health: the node-engine's is
|
||||
// that it reported its current declaration at all, the node tools' that they answer.
|
||||
if r := f.reports[node]; r.Current || component == broker.RuntimeModule {
|
||||
now := time.Now()
|
||||
r.Current, r.Outcome = true, inventory.OutcomeApplied
|
||||
if r.At == nil {
|
||||
r.At = &now
|
||||
}
|
||||
f.reports[node] = r
|
||||
}
|
||||
if component == hostModule {
|
||||
// A node-engine reporting a build the mesh delivered, current or previous, is the version
|
||||
// split's (D10), not ill health; a build the mesh did not deliver is.
|
||||
f.engines[node] = deliveredOr(shelf[component], f.engines[node])
|
||||
}
|
||||
h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f)
|
||||
if h == healthGood {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node,
|
||||
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)})
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// deliveredOr is the reported engine build, or the current delivered one when the report names any
|
||||
// build at all: what H-engine judges is that it reports, not which.
|
||||
func deliveredOr(m catalogue.Manifest, reported string) string {
|
||||
if reported == "" {
|
||||
return reported
|
||||
}
|
||||
if v := deliveredVersions(m); len(v) > 0 {
|
||||
return v[0]
|
||||
}
|
||||
return reported
|
||||
}
|
||||
|
||||
// componentWords is a core component as a sentence names it.
|
||||
func componentWords(component string) string {
|
||||
switch component {
|
||||
case hostModule:
|
||||
return "node-engine"
|
||||
case broker.RuntimeModule:
|
||||
return "node tools"
|
||||
case catalogue.ControllerSeatName:
|
||||
return "controller"
|
||||
}
|
||||
return component
|
||||
}
|
||||
|
||||
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a
|
||||
// request crosses it to every machine's node tools and back.
|
||||
func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
problems, err := busHealth(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: "mesh", Token: "unhealthy",
|
||||
Kind: kindCoreUnhealthy, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the bus is not healthy: %s", strings.Join(problems, "; ")),
|
||||
Said: strings.Join(problems, "; ")}}, nil
|
||||
}
|
||||
|
||||
// busHealth is the bus's health definition, as what is wanting: nothing when healthy. What the bus's
|
||||
// planned step checks after the bus is replaced, too.
|
||||
var busHealth = func(ctx context.Context, d *doctor) ([]string, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("this controller has no bus to ask")
|
||||
}
|
||||
streams, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var problems []string
|
||||
for _, s := range streams {
|
||||
if _, err := js.StreamInfo(s.Name, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||
problems = append(problems, "the stream "+s.Name+" is missing")
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
for _, c := range consumers {
|
||||
_, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound) {
|
||||
problems = append(problems, consumerWords(c)+" is missing")
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||
}
|
||||
}
|
||||
// The round trip: every machine heard from that runs the node tools answers across the bus.
|
||||
running, err := d.open.inventory.Running(ctx, broker.RuntimeModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var expected []string
|
||||
for _, n := range running {
|
||||
if heard[n] {
|
||||
expected = append(expected, n)
|
||||
}
|
||||
}
|
||||
if len(expected) > 0 {
|
||||
answered, err := servedOnTheBus(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var silent []string
|
||||
for _, n := range expected {
|
||||
if !answered[n].runtime {
|
||||
silent = append(silent, n)
|
||||
}
|
||||
}
|
||||
// One machine's tools silent is that machine's (H-tools); none answering is the bus.
|
||||
if len(silent) == len(expected) {
|
||||
slices.Sort(silent)
|
||||
problems = append(problems, "no request crossed the bus and came back: no machine's node tools answered ("+
|
||||
strings.Join(silent, ", ")+")")
|
||||
}
|
||||
}
|
||||
return problems, nil
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
|
||||
// issue 259, ADR 0221).
|
||||
//
|
||||
// A named push ends by sending every other machine whose declaration differs from what it was last
|
||||
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
|
||||
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
|
||||
// out makes every machine running it differ from the merge on, and so did a module whose plan was
|
||||
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
|
||||
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
|
||||
// everywhere at once.
|
||||
//
|
||||
// What tells the two apart is which build of each module the machine was last sent, kept with every
|
||||
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
|
||||
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
|
||||
|
||||
// heldBack is why a push that did not name a machine must not send it, empty when it may.
|
||||
//
|
||||
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
|
||||
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
|
||||
// machine was last sent — including a module the machine was never sent at all. A move is held when
|
||||
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
|
||||
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
|
||||
// is not known, so a held upgrade cannot be told from anything else.
|
||||
func heldBack(node string, modules []string, sent map[string]string, known bool,
|
||||
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
|
||||
if !known {
|
||||
return []string{"which builds it was last sent is not known — it was last sent before the " +
|
||||
"mesh kept them, or sent a declaration by hand"}
|
||||
}
|
||||
var why []string
|
||||
for _, m := range modules {
|
||||
now := current[m]
|
||||
was, carried := sent[m]
|
||||
if carried && was == now.Commit {
|
||||
continue
|
||||
}
|
||||
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
|
||||
if !now.RollOut {
|
||||
why = append(why, move+", which its upgrade policy records rather than rolls out")
|
||||
continue
|
||||
}
|
||||
if id := planStillToSend(plans, m, node); id != "" {
|
||||
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
|
||||
}
|
||||
}
|
||||
sort.Strings(why)
|
||||
return why
|
||||
}
|
||||
|
||||
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
|
||||
// one holding the module that has neither finished sending it nor sent it here first, and has not
|
||||
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
|
||||
// per machine.
|
||||
func planStillToSend(plans []inventory.Plan, module, node string) string {
|
||||
for _, p := range plans {
|
||||
s, holds := p.Modules[module]
|
||||
if !p.Open() || !holds {
|
||||
continue
|
||||
}
|
||||
if s == nil {
|
||||
return p.ID
|
||||
}
|
||||
if s.SentAt != nil || s.State == "failed" {
|
||||
continue
|
||||
}
|
||||
first := false
|
||||
for _, n := range s.First {
|
||||
if n == node {
|
||||
first = true
|
||||
}
|
||||
}
|
||||
if !first {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func fromBuild(was string, carried bool) string {
|
||||
if !carried {
|
||||
return "(never sent it) "
|
||||
}
|
||||
return "from " + buildName(was) + " "
|
||||
}
|
||||
|
||||
func buildName(commit string) string {
|
||||
if commit == "" {
|
||||
return "a build with no source"
|
||||
}
|
||||
return shortCommit(commit)
|
||||
}
|
||||
|
||||
// heldMachines reads, for each machine named, why a push that did not name it must not send it
|
||||
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
|
||||
// to the send, which says why.
|
||||
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
|
||||
out := map[string][]string{}
|
||||
if len(names) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
inv := open.inventory
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
modules := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it
|
||||
// runs.
|
||||
same := map[string]string{}
|
||||
for m, was := range sent {
|
||||
same[m] = was
|
||||
if f.identical(m, was, current[m].Commit) {
|
||||
same[m] = current[m].Commit
|
||||
}
|
||||
}
|
||||
why := heldBack(node, modules, same, known, current, plans)
|
||||
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
|
||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||
if planStillToSend(plans, mv.Module, node) == "" {
|
||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
}
|
||||
}
|
||||
if len(why) > 0 {
|
||||
sort.Strings(why)
|
||||
out[node] = why
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
|
||||
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
|
||||
func sayHeld(w io.Writer, node string, why []string) {
|
||||
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
|
||||
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
|
||||
"grant from this push among it — waits with it. `push %s` sends it\n",
|
||||
node, strings.Join(why, "; "), node)
|
||||
}
|
||||
|
||||
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
|
||||
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
|
||||
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
|
||||
//
|
||||
// `handled` is every machine already sent or already said; it is not considered again. Answers the
|
||||
// machines that could not be composed, as refusals.
|
||||
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
||||
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
||||
w io.Writer) ([]string, error) {
|
||||
inv := open.inventory
|
||||
var refusals []string
|
||||
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
||||
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
|
||||
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
|
||||
// cascade legitimately still converging, so it is said rather than passed over in silence.
|
||||
rounds := 0
|
||||
for {
|
||||
would, err := wouldSend(ctx, open, nodes)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
behind, err := inv.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var also []string
|
||||
for _, m := range behind {
|
||||
if !handled[m.Node] {
|
||||
also = append(also, m.Node)
|
||||
}
|
||||
}
|
||||
if len(also) == 0 {
|
||||
return refusals, nil
|
||||
}
|
||||
if rounds++; rounds > len(nodes) {
|
||||
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
|
||||
"should not happen; run `push --behind` to finish\n",
|
||||
rounds-1, strings.Join(also, ", "))
|
||||
return refusals, nil
|
||||
}
|
||||
sort.Strings(also)
|
||||
held, err := heldMachines(ctx, open, also)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var sending []string
|
||||
for _, name := range also {
|
||||
// Every candidate this round is marked handled — the sent ones so they are not
|
||||
// re-listed, the held ones because they stay held, and the refused ones so a machine
|
||||
// that cannot be composed does not make the loop spin on it for ever.
|
||||
handled[name] = true
|
||||
if why, isHeld := held[name]; isHeld {
|
||||
sayHeld(w, name, why)
|
||||
continue
|
||||
}
|
||||
sending = append(sending, name)
|
||||
}
|
||||
if len(sending) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
|
||||
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
|
||||
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
|
||||
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
|
||||
// is left out of it said, and its declaration allocated.
|
||||
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
|
||||
return func(held context.Context, node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
reportKept(held, plan)
|
||||
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
if err == nil {
|
||||
reportLeftOut(node, declared)
|
||||
}
|
||||
return declared, err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
|
||||
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
|
||||
// else that moved is still a consequence the push sends (ADR 0083).
|
||||
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{
|
||||
"resolver": {Commit: "c2c2c2c2c2"},
|
||||
"agent": {Commit: "a2", RollOut: true},
|
||||
"network": {},
|
||||
}
|
||||
modules := []string{"network", "resolver", "agent"}
|
||||
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
|
||||
|
||||
// A module whose policy records moved: held, naming it, both builds and why.
|
||||
why := heldBack("laptop", modules, sent, true, current, nil)
|
||||
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
|
||||
!strings.Contains(why[0], "upgrade policy records") {
|
||||
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
|
||||
}
|
||||
|
||||
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
|
||||
sent["resolver"] = "c2c2c2c2c2"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||
t.Fatalf("a machine whose builds are all current was held: %v", why)
|
||||
}
|
||||
|
||||
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
|
||||
sent["agent"] = "a1"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
|
||||
}
|
||||
|
||||
// The last send's builds are not known: held whole.
|
||||
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
|
||||
!strings.Contains(why[0], "not known") {
|
||||
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
|
||||
}
|
||||
|
||||
// A module the machine was never sent, under a recording policy: held, and said so.
|
||||
delete(sent, "resolver")
|
||||
sent["agent"] = "a2"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
|
||||
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
|
||||
t.Fatalf("a module never sent under a recording policy: %v", why)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
|
||||
// naming some other machine must not send them for it.
|
||||
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
|
||||
at := time.Now()
|
||||
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
|
||||
sent := map[string]string{"agent": "a1"}
|
||||
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
|
||||
|
||||
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
|
||||
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
|
||||
t.Fatalf("a machine the plan has not reached was not held: %v", why)
|
||||
}
|
||||
// The first machine itself was sent by the plan: not held by it.
|
||||
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
|
||||
t.Fatalf("the plan's first machine was held: %v", why)
|
||||
}
|
||||
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
|
||||
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
|
||||
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
|
||||
Modules: map[string]*inventory.PlanModule{"agent": s}}}
|
||||
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
|
||||
t.Errorf("%s: not held: %v", what, why)
|
||||
}
|
||||
}
|
||||
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
|
||||
for what, plans := range map[string][]inventory.Plan{
|
||||
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
|
||||
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "failed"}}}},
|
||||
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built"}}}},
|
||||
} {
|
||||
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
|
||||
t.Errorf("%s: held: %v", what, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A send records the build of each module it carried; a module left out of it keeps the build it
|
||||
// was last sent, since the machine keeps that one.
|
||||
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
|
||||
got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
|
||||
current, map[string]string{"a": "a1", "b": "b1"})
|
||||
if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("carried %v, wanted %v", got, want)
|
||||
}
|
||||
// Not known before: the left-out module is not recorded at all, so it reads as never sent.
|
||||
got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
|
||||
if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("carried %v, wanted %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
|
||||
// reads the machine as current — and writes down which machines it declared.
|
||||
type recordedDelivery struct {
|
||||
inv *inventory.Inventory
|
||||
declared []string
|
||||
}
|
||||
|
||||
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
|
||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
r.declared = append(r.declared, s.node)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
}
|
||||
|
||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||
// says which build it is.
|
||||
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
|
||||
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
|
||||
}}
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// A third machine on the private network: once it is sent, every other machine's peers change with
|
||||
// it, which is a consequence a push must still send — no build moved.
|
||||
func aThirdMachine(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.AddNode(ctx, "spare")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
|
||||
{"name": "systemd", "present": true}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var profile map[string]any
|
||||
if err := json.Unmarshal(reported, &profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
|
||||
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
|
||||
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
aResolver(t, open, "c1c1c1c1c1", asked)
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
|
||||
}
|
||||
digestOfLaptop := func() string {
|
||||
sent, err := inv.Outstanding(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sent
|
||||
}
|
||||
before := digestOfLaptop()
|
||||
|
||||
// The change merges; the policy is record. `push anchor` sends the anchor...
|
||||
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ...and its cascade leaves the laptop, saying so.
|
||||
var said bytes.Buffer
|
||||
d.declared = nil
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
|
||||
if err != nil || len(refused) != 0 {
|
||||
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||
}
|
||||
if len(d.declared) != 0 {
|
||||
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
|
||||
}
|
||||
if digestOfLaptop() != before {
|
||||
t.Fatal("the laptop's last send moved: it was sent the held build")
|
||||
}
|
||||
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
|
||||
"upgrade policy records", "`push laptop` sends it"} {
|
||||
if !strings.Contains(said.String(), want) {
|
||||
t.Errorf("the push did not say %q:\n%s", want, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
|
||||
// is held, and that what else it is owed waits with it.
|
||||
aThirdMachine(t, open)
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
|
||||
}
|
||||
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
|
||||
t.Fatalf("the push did not say both:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
|
||||
// (novox/hq ADR 0236).
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
|
||||
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
|
||||
t.Fatalf("the new send did not record the new build: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
|
||||
// for another reason is sent by a push that names someone else.
|
||||
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
|
||||
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// `push spare`, the machine just placed: the others' peers change with it.
|
||||
aThirdMachine(t, open)
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
var said bytes.Buffer
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if strings.Contains(said.String(), "was not sent") {
|
||||
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
|
||||
record, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||
// question.
|
||||
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
|
||||
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
}
|
||||
@@ -18,16 +18,16 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||
plain := func(context.Context, string) (sendable, error) {
|
||||
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
||||
}
|
||||
failing := func(readyNode, []byte) error { return errors.New("the broker went away") }
|
||||
fine := func(readyNode, []byte) error { return nil }
|
||||
failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
|
||||
fine := &recordingDelivery{}
|
||||
|
||||
for name, round := range map[string]func() error{
|
||||
"a body that cannot be marshalled": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
|
||||
return err
|
||||
},
|
||||
"a send that fails": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
|
||||
return err
|
||||
},
|
||||
} {
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
||||
// as holding.
|
||||
//
|
||||
// **A seat held by derivation is a seat held by accident of being alone** (novox/hq
|
||||
// 04-ISSUES/170). ADR 0131 lets a holder on record settle a seat, and lets any other assignment
|
||||
// whose module could hold it stand beside the holder, eligible and silent. But a seat nobody
|
||||
// ever handed over has no record, so its holder is whichever assignment happened to be the sole
|
||||
// claimant — and the day a second one is assigned, both claim, both are refused, and the first
|
||||
// one's whole machine stops resolving. That is what assigning a second postgres did to the
|
||||
// control plane's own store.
|
||||
//
|
||||
// So the mesh writes the derived answer down before it acts on an assignment: for every
|
||||
// mesh-scoped seat with exactly one resolved holder and nothing on record, that holder is
|
||||
// recorded as the standing one — the same record `seat <name> --to <node>/<module>` makes by
|
||||
// hand, made from what the mesh already resolved. A seat with two derived claimants is left
|
||||
// alone: that is the ambiguity a person settles, and recording either would be guessing.
|
||||
//
|
||||
// Node-scoped seats are untouched: a record is one holder per seat, and a node-scoped seat has
|
||||
// one holder per machine (ADR 0121), so there is nothing for a record to settle there.
|
||||
func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// exclude nobody: every node's claims, resolved with the holdings on record.
|
||||
world, err := theRestOfTheMesh(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A record is a row against a seat the store knows. A seat it does not — a mesh whose seats
|
||||
// were never seeded, a seat a module declares for itself — stays held by derivation, as it
|
||||
// always was; a missing row is not a reason an assignment fails.
|
||||
known, err := inv.Seats(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recordable := map[string]bool{}
|
||||
for _, s := range known {
|
||||
recordable[s.Name] = true
|
||||
}
|
||||
onRecord := map[string]bool{}
|
||||
for _, h := range recorded {
|
||||
if s, ok := catalogue.SeatNamed(h.Claim); ok {
|
||||
onRecord[s.Name] = true
|
||||
}
|
||||
}
|
||||
holders := map[string][]catalogue.Held{}
|
||||
for _, h := range world.Held {
|
||||
if h.Scope != catalogue.ScopeMesh {
|
||||
continue
|
||||
}
|
||||
s, ok := catalogue.SeatNamed(h.Claim)
|
||||
if !ok || onRecord[s.Name] || !recordable[s.Name] {
|
||||
continue
|
||||
}
|
||||
holders[s.Name] = append(holders[s.Name], h)
|
||||
}
|
||||
names := make([]string, 0, len(holders))
|
||||
for name := range holders {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var said []string
|
||||
for _, name := range names {
|
||||
if len(holders[name]) != 1 {
|
||||
continue
|
||||
}
|
||||
h := holders[name][0]
|
||||
if err := inv.HoldSeat(ctx, name, catalogue.ScopeMesh, h.Node, h.Module); err != nil {
|
||||
return said, err
|
||||
}
|
||||
said = append(said, fmt.Sprintf(
|
||||
"recorded %s on %s as the standing holder of %s, which it held only by being alone",
|
||||
h.Module, h.Node, name))
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
|
||||
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
|
||||
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
|
||||
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
|
||||
//
|
||||
// **The holders on record, and only the sole claimant when there are none** — the same answer the
|
||||
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
|
||||
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
|
||||
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
|
||||
// listed here. A holder off the private network is left out: a resolver named at an address nothing
|
||||
// answers is a lookup that waits out its timeout on every name.
|
||||
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
|
||||
var replicated []catalogue.Seat
|
||||
for _, s := range catalogue.Seats() {
|
||||
if s.Replicated && s.Scope == catalogue.ScopeMesh {
|
||||
replicated = append(replicated, s)
|
||||
}
|
||||
}
|
||||
if len(replicated) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address := map[string]string{}
|
||||
for _, p := range places {
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]map[string]string{}
|
||||
for _, seat := range replicated {
|
||||
var nodes []string
|
||||
for _, h := range recorded {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
|
||||
nodes = append(nodes, h.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
var derived []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||
derived = append(derived, e.On...)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(derived) == 1 {
|
||||
nodes = derived
|
||||
}
|
||||
}
|
||||
at := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
if address[n] != "" {
|
||||
at[overlay.InternalName(n)] = address[n]
|
||||
}
|
||||
}
|
||||
out[seat.Name] = at
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// A seat nobody ever handed over is held by whichever assignment happened to be alone — and the
|
||||
// day a second module able to hold it is assigned, both claimed, both were refused, and the first
|
||||
// one's machine stopped resolving (novox/hq 04-ISSUES/170). The mesh now writes the derived holder
|
||||
// down before it acts, so the second assignment stands beside the holder on record.
|
||||
|
||||
func aSeatedStore() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "store", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}}
|
||||
}
|
||||
|
||||
func TestASecondEligibleHolderStandsBesideTheOneHeldByBeingAlone(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
// Every deploy seeds the mesh's own seats; a record is a row against one of them.
|
||||
if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, aSeatedStore())
|
||||
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := assign(ctx, open, "laptop", "store")
|
||||
if err != nil {
|
||||
t.Fatalf("a second store, eligible for the seat, was refused:\n%s\n%v", said, err)
|
||||
}
|
||||
if strings.Contains(said, "cannot be worked out") {
|
||||
t.Fatalf("assigning a second store unsettled the first one's machine:\n%s", said)
|
||||
}
|
||||
if !strings.Contains(said, "recorded store on anchor as the standing holder of mesh-store") {
|
||||
t.Fatalf("the holder by derivation was not written down:\n%s", said)
|
||||
}
|
||||
|
||||
holdings, err := open.inventory.Holdings(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, h := range holdings {
|
||||
if h.Claim == "mesh-store" {
|
||||
found = true
|
||||
if h.Node != "anchor" || h.Module != "store" {
|
||||
t.Fatalf("mesh-store is recorded on %s/%s, not on the one that held it", h.Node, h.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("mesh-store has no holder on record after assigning: %v", holdings)
|
||||
}
|
||||
|
||||
// And the record decides from here: the anchor's plan holds the seat, the laptop's does not.
|
||||
for node, holds := range map[string]bool{"anchor": true, "laptop": false} {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatalf("%s no longer resolves: %v", node, err)
|
||||
}
|
||||
var claimed bool
|
||||
for _, c := range plan.Claims {
|
||||
if c.Claim == "mesh-store" {
|
||||
claimed = true
|
||||
}
|
||||
}
|
||||
if claimed != holds {
|
||||
t.Fatalf("%s holds mesh-store: %v, want %v", node, claimed, holds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHolderOnRecordIsNotRewrittenByDerivation(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, aSeatedStore())
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := assign(ctx, open, node, "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A person hands the seat to the laptop. From here the record decides, and what the mesh
|
||||
// derives must never write over it.
|
||||
if err := open.inventory.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := recordDerivedHolders(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("a seat on record was written again from derivation: %v", said)
|
||||
}
|
||||
holdings, _ := open.inventory.Holdings(ctx)
|
||||
for _, h := range holdings {
|
||||
if h.Claim == "mesh-store" && h.Node != "laptop" {
|
||||
t.Fatalf("the record moved to %s", h.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A host refuses a declaration carrying a field it does not know, and refuses it whole — so every new
|
||||
// field is a flag day, and the mesh had no record of which host any machine ran (novox/hq
|
||||
// 04-ISSUES/087). The order was kept by somebody remembering it.
|
||||
|
||||
func TestTheMeshNamesWhichMachinesRunWhichHost(t *testing.T) {
|
||||
split := hostSplit([]inventory.Node{
|
||||
{Name: "anchor", HostVersion: "04a27ca"},
|
||||
{Name: "laptop", HostVersion: "ced54d4"},
|
||||
{Name: "spare", HostVersion: "04a27ca"},
|
||||
})
|
||||
if len(split) != 2 {
|
||||
t.Fatalf("two versions were reported and the split has %d: %v", len(split), split)
|
||||
}
|
||||
if got := strings.Join(split["04a27ca"], ","); got != "anchor,spare" && got != "spare,anchor" {
|
||||
t.Fatalf("04a27ca is held by %q", got)
|
||||
}
|
||||
if got := strings.Join(split["ced54d4"], ","); got != "laptop" {
|
||||
t.Fatalf("ced54d4 is held by %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMeshDoesNotClaimWhichHostIsNewer(t *testing.T) {
|
||||
// **The fault this replaced.** A host reports its version as a commit, and commits have no order.
|
||||
// The first version compared them as strings and, on the live mesh, named the three machines
|
||||
// running the NEWER host as the ones behind: `ced54d4` sorts above `04a27ca` and means nothing.
|
||||
//
|
||||
// There is no assertion to make about which is newer, and that is the point — the type says so.
|
||||
// hostSplit returns who runs what, and nothing that could be read as an ordering.
|
||||
split := hostSplit([]inventory.Node{
|
||||
{Name: "old-but-sorts-high", HostVersion: "ced54d4"},
|
||||
{Name: "new-but-sorts-low", HostVersion: "04a27ca"},
|
||||
})
|
||||
for version, machines := range split {
|
||||
if len(machines) != 1 {
|
||||
t.Fatalf("%s is held by %v", version, machines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineThatHasNotSaidIsNotAVersion(t *testing.T) {
|
||||
// It may be running anything. Counting it as a version would invent a disagreement; `node show`
|
||||
// says per machine that it has not said.
|
||||
split := hostSplit([]inventory.Node{
|
||||
{Name: "anchor", HostVersion: "04a27ca"},
|
||||
{Name: "quiet"},
|
||||
})
|
||||
if split != nil {
|
||||
t.Fatalf("one reported version and one silence read as a disagreement: %v", split)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMachinesAgreeingOnTheirHostAreNotADisagreement(t *testing.T) {
|
||||
if split := hostSplit([]inventory.Node{
|
||||
{Name: "anchor", HostVersion: "v2"},
|
||||
{Name: "laptop", HostVersion: "v2"},
|
||||
}); split != nil {
|
||||
t.Fatalf("machines agreeing reported a split: %v", split)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMeshWhereNothingReportedAHostStatesNoDisagreement(t *testing.T) {
|
||||
if split := hostSplit([]inventory.Node{{Name: "anchor"}, {Name: "laptop"}}); split != nil {
|
||||
t.Fatalf("a mesh told no host version reported a split: %v", split)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) {
|
||||
// The machine has sent this since ADR 0141 and the controller's own copy of the report did not
|
||||
// have the field, so it was unmarshalled into nothing. End to end through the store, because the
|
||||
// fault was a field that existed on one side of the wire only.
|
||||
open := aMesh(t)
|
||||
record, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.HostVersion != "" {
|
||||
t.Fatalf("a machine that never reported one has host version %q", record.HostVersion)
|
||||
}
|
||||
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.HostVersion != "ced54d4" {
|
||||
t.Fatalf("the reported host version read back as %q", again.HostVersion)
|
||||
}
|
||||
// An empty report never clears what a machine last said: a bare word that the node is there says
|
||||
// nothing about its host.
|
||||
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept.HostVersion != "ced54d4" {
|
||||
t.Fatalf("a report carrying no host version cleared what the machine had said: %q",
|
||||
kept.HostVersion)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
||||
// one consumer's identity.
|
||||
|
||||
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
||||
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, body string) string {
|
||||
p := filepath.Join(dir, name+".json")
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
objects := write("objects", `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
||||
resolver := write("resolver", `{"module":"resolver","version":"1",
|
||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
||||
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
||||
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
||||
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
||||
}
|
||||
out.Reset()
|
||||
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
||||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
||||
strings.Contains(out.String(), "networkmanager wants") {
|
||||
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
||||
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||
Requires: []string{"wildcard-resolution"}})
|
||||
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||
consumer, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over := consumer.Overflowing()
|
||||
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||
}
|
||||
|
||||
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||
// networkmanager, and no push to the provider could go through.
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||
}
|
||||
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||
}
|
||||
grants, _, _, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keyless bool
|
||||
for _, g := range grants {
|
||||
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||
}
|
||||
if !keyless {
|
||||
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||
}
|
||||
var granted []string
|
||||
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||
granted = append(granted, c.From)
|
||||
}
|
||||
if strings.Join(granted, ",") != "files" {
|
||||
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||
!strings.Contains(said, "left out of anchor's grants") {
|
||||
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||
}
|
||||
|
||||
// And `status` names it, and does not call the mesh well while it stands.
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||
t.Fatalf("status does not say it:\n%s", shown)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||
doc.Overflowing[0].Bound.Max != 20 {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
|
||||
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
|
||||
// declaration composed before an assignment changed and sent after a newer one carried the higher
|
||||
// number — and the machine, which refuses a lower number, took the older content as the mesh's
|
||||
// newest word. Taken before the composition reads anything, the order of numbers is the order of
|
||||
// compositions, and the host's refusal does what it is for.
|
||||
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
|
||||
allot := numbered()
|
||||
var composed []string
|
||||
compose := func(stamp string) func(string) (sendable, error) {
|
||||
return func(node string) (sendable, error) {
|
||||
composed = append(composed, stamp)
|
||||
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
|
||||
}
|
||||
}
|
||||
// Composed first — before an assignment changed — and sent last.
|
||||
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
|
||||
// Composed after the change, sent first.
|
||||
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
|
||||
|
||||
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
|
||||
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
|
||||
stale[0].declared.Sequence, fresh[0].declared.Sequence)
|
||||
}
|
||||
// Sent in the other order, the numbers do not change — so the machine that has applied the
|
||||
// fresh one (2) refuses the stale one (1) when it arrives late.
|
||||
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
|
||||
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
|
||||
}
|
||||
if len(composed) != 2 || composed[0] != "before" {
|
||||
t.Fatalf("compositions happened in an unexpected order: %v", composed)
|
||||
}
|
||||
}
|
||||
|
||||
// The number is taken before the first read of the composition, not after it: an allotter that
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (order, error) {
|
||||
if node == "anchor" {
|
||||
return order{}, context.DeadlineExceeded
|
||||
}
|
||||
return order{sequence: 7}, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
if node == "anchor" {
|
||||
t.Fatal("anchor was composed although its number could not be taken")
|
||||
}
|
||||
return sendable{}, nil
|
||||
})
|
||||
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
|
||||
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
|
||||
}
|
||||
if len(refusals) != 1 {
|
||||
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
|
||||
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
|
||||
// current" over a machine that had just been sent something else.
|
||||
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
t.Fatalf("recording a send after cancellation failed: %v", err)
|
||||
}
|
||||
outstanding, err := inv.Outstanding(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outstanding != digest || digest != digestOf(body) {
|
||||
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// A command run at a shell (novox/hq to-be 45 §6): under the holder's epoch while a controller holds the
|
||||
// lease, read at the moment it acts; under a lease of its own while none does, given back as it ends.
|
||||
func TestACommandActsUnderTheHoldersEpochOrItsOwn(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
ctx := t.Context()
|
||||
|
||||
// Nobody holds it: the command takes it, and gives it back.
|
||||
cmd := &actor{}
|
||||
own, err := cmd.epoch(ctx)
|
||||
if err != nil || own == 0 {
|
||||
t.Fatalf("a command with nobody holding the lease acts as %d (%v)", own, err)
|
||||
}
|
||||
if h, found, _ := lease.Current(ctx, kv); !found || h.Epoch != own {
|
||||
t.Fatalf("the command's lease is not on the bus: %+v", h)
|
||||
}
|
||||
cmd.release()
|
||||
if _, found, _ := lease.Current(ctx, kv); found {
|
||||
t.Fatal("the command did not give its lease back as it ended")
|
||||
}
|
||||
|
||||
// A controller holds it: a command acts under that epoch.
|
||||
l, err := lease.Open(ctx, mustJetStream(t, url), broker.LeaseBucket, lease.Options{Holder: lease.Holder{Instance: "serving"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := l.TryTake(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
borrower := &actor{}
|
||||
defer borrower.release()
|
||||
if got, err := borrower.epoch(ctx); err != nil || got != held {
|
||||
t.Fatalf("a command acts as %d (%v), want the holder's %d", got, err, held)
|
||||
}
|
||||
// The holder lets go: the command does not go on under an epoch nobody holds.
|
||||
l.Release(ctx)
|
||||
if _, err := borrower.epoch(ctx); err == nil {
|
||||
t.Fatal("a command acted under an epoch nobody holds any more")
|
||||
}
|
||||
}
|
||||
|
||||
// mustJetStream is a connection of its own to the test bus.
|
||||
func mustJetStream(t *testing.T, url string) jetstream.JetStream {
|
||||
t.Helper()
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
|
||||
// serving controllers over one store and one bus. The second waits while the first holds the lease; on
|
||||
// a handover it takes it at a higher epoch, the record says which held what and how each ended; and the
|
||||
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
|
||||
// moment it lost it.
|
||||
//
|
||||
// MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus)
|
||||
|
||||
// aBusForTheLease is the test bus with the controller's lease bucket new.
|
||||
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.KeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) })
|
||||
return url, kv
|
||||
}
|
||||
|
||||
func TestTwoControllersOneActs(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Controller A takes the lease.
|
||||
a := &actor{}
|
||||
aCtx, stopA := context.WithCancel(ctx)
|
||||
defer stopA()
|
||||
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epochA, err := a.epoch(ctx)
|
||||
if err != nil || epochA == 0 {
|
||||
t.Fatalf("A holds no epoch: %d, %v", epochA, err)
|
||||
}
|
||||
|
||||
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
|
||||
b := &actor{}
|
||||
bCtx, stopB := context.WithCancel(ctx)
|
||||
defer stopB()
|
||||
tookB := make(chan (<-chan struct{}), 1)
|
||||
go func() {
|
||||
lost, err := b.serveUnderTheLease(bCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Errorf("B: %v", err)
|
||||
close(tookB)
|
||||
return
|
||||
}
|
||||
tookB <- lost
|
||||
}()
|
||||
select {
|
||||
case <-tookB:
|
||||
t.Fatal("B took the lease while A held it")
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B, waiting, may act: %v", err)
|
||||
}
|
||||
|
||||
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
|
||||
stopA()
|
||||
a.release()
|
||||
var lostB <-chan struct{}
|
||||
select {
|
||||
case lostB = <-tookB:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("B did not take the lease A gave back")
|
||||
}
|
||||
select {
|
||||
case <-lostA:
|
||||
default:
|
||||
t.Fatal("A, having given the lease back, is not told it no longer holds it")
|
||||
}
|
||||
epochB, err := b.epoch(ctx)
|
||||
if err != nil || epochB <= epochA {
|
||||
t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err)
|
||||
}
|
||||
if _, err := a.epoch(ctx); err == nil {
|
||||
t.Fatal("A acts after giving the lease back")
|
||||
}
|
||||
ea, _, _ := inv.EpochOf(ctx, epochA)
|
||||
eb, _, _ := inv.EpochOf(ctx, epochB)
|
||||
if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance {
|
||||
t.Fatalf("the record of the handover reads %+v then %+v", ea, eb)
|
||||
}
|
||||
|
||||
// Something else writes the lease's key — a third controller on a clock that read it as expired:
|
||||
// B's next renewal is refused, and B stops acting at once.
|
||||
if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-lostB:
|
||||
case <-time.After(2 * lease.RenewEvery):
|
||||
t.Fatal("B was not told it lost the lease")
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B acts after losing the lease: %v", err)
|
||||
}
|
||||
// Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition
|
||||
// is not raised.
|
||||
inv.ActsUnder(b.epoch)
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
|
||||
State: inventory.PlanBuilding}
|
||||
if err := inv.SavePlan(ctx, &saved); err == nil {
|
||||
t.Fatal("B wrote a plan after losing the lease")
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
|
||||
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
|
||||
defer keeper.Close(context.Background())
|
||||
if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x",
|
||||
Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil {
|
||||
t.Fatal("B raised a condition after losing the lease")
|
||||
}
|
||||
if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost {
|
||||
t.Fatalf("B's epoch does not say it was lost: %+v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease:
|
||||
// it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can.
|
||||
func TestAControllerTheBusRefusesTheLeaseServesUnleasedAndSaysSo(t *testing.T) {
|
||||
a := &actor{serving: true, unleased: "the bus refused the lease's key"}
|
||||
if epoch, err := a.epoch(context.Background()); err != nil || epoch != 0 {
|
||||
t.Fatalf("an unleased controller answers %d, %v: it acts, claiming no epoch", epoch, err)
|
||||
}
|
||||
if st := a.standing(); st.Unleased == "" || st.Held {
|
||||
t.Fatalf("its standing says %+v", st)
|
||||
}
|
||||
// One that neither holds nor is unleased — still waiting — acts on nothing.
|
||||
waiting := &actor{serving: true}
|
||||
if _, err := waiting.epoch(context.Background()); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("a controller waiting for the lease may act: %v", err)
|
||||
}
|
||||
}
|
||||
+167
-9
@@ -8,6 +8,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -54,6 +55,9 @@ func run() error {
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
||||
// 45 §6), so the next controller takes it at once rather than after its age.
|
||||
defer theLease.release()
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
@@ -72,11 +76,36 @@ func run() error {
|
||||
return askCommand(ctx, args[1:])
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return queueCommand(ctx, args[1:])
|
||||
case "cancel":
|
||||
return cancelCommand(ctx, args[1:])
|
||||
case "clear":
|
||||
return clearCommand(ctx, args[1:])
|
||||
case "rebuild":
|
||||
return rebuildCommand(ctx, args[1:])
|
||||
case "replay":
|
||||
return replayCommand(ctx, args[1:])
|
||||
case "kill":
|
||||
return killCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
return collectionCommand(ctx, args[1:])
|
||||
case "plans":
|
||||
return plansCommand(ctx, args[1:])
|
||||
case "delivery":
|
||||
// The verbs the delivery's owner asks with (novox/hq ADR 0239).
|
||||
return deliveryCommand(ctx, args[1:])
|
||||
case "pin":
|
||||
return pinCommand(ctx, args[1:], true)
|
||||
case "unpin":
|
||||
return pinCommand(ctx, args[1:], false)
|
||||
case "migrate":
|
||||
// `prepare` is how the mesh asks any module to bring its state to the shape this version needs
|
||||
// (novox/hq ADR 0135), and the control plane answers it the same way as everything else — its
|
||||
// own schema is not a special case. `migrate` remains the word a person types.
|
||||
case "prepare", "migrate":
|
||||
return migrate(ctx)
|
||||
case "node":
|
||||
return nodeCommand(ctx, args[1:])
|
||||
@@ -85,11 +114,23 @@ func run() error {
|
||||
case "identity":
|
||||
return identityCommand(ctx, args[1:])
|
||||
case "broker":
|
||||
return brokerCommand(args[1:])
|
||||
return brokerCommand(ctx, args[1:])
|
||||
case "serve":
|
||||
return serve(ctx)
|
||||
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9).
|
||||
case "facts":
|
||||
return factsCommand(ctx, args[1:])
|
||||
// The merge gate: every machine of the snapshot composed with a change (novox/hq to-be 45 §9).
|
||||
case "merge-gate":
|
||||
return mergeGateCommand(ctx, args[1:])
|
||||
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 283).
|
||||
case "check-here":
|
||||
return checkHereCommand(ctx, args[1:])
|
||||
case "upgrade":
|
||||
return upgradeCommand(ctx, args[1:])
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
|
||||
case "bus":
|
||||
return busCommand(ctx, args[1:])
|
||||
case "declare":
|
||||
return declare(ctx, args[1:])
|
||||
case "overlay":
|
||||
@@ -122,6 +163,31 @@ func run() error {
|
||||
return seatCommand(ctx, args[1:])
|
||||
case "status":
|
||||
return statusCommand(ctx, args[1:])
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
case "hand-act":
|
||||
return handActCommand(ctx, args[1:])
|
||||
case "hand-acts":
|
||||
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||
case "durations":
|
||||
return durationsCommand(ctx, args[1:])
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
case "conditions":
|
||||
return conditionsCommand(ctx, args[1:])
|
||||
case "doctor":
|
||||
return doctorCommand(ctx, args[1:])
|
||||
// What the healers did, and their brake (novox/hq to-be 45 §7).
|
||||
case "healers":
|
||||
return healersCommand(ctx, args[1:])
|
||||
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
||||
// (novox/hq ADR 0230).
|
||||
case "retire":
|
||||
return retireCommand(ctx, args[1:])
|
||||
case "cleanup":
|
||||
return cleanupCommand(ctx, args[1:])
|
||||
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
|
||||
case "data":
|
||||
return dataCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
@@ -138,12 +204,16 @@ func usage() {
|
||||
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
||||
|
||||
migrate bring each context's schema up to date
|
||||
prepare the same, asked the way the mesh asks any module (ADR 0135)
|
||||
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
||||
node list the nodes this mesh knows about
|
||||
node show <name> what one machine reported it can do, and why
|
||||
node public-domain <name> the domain it composes its routed names under
|
||||
node public-domain <name> <d> ...set it to d
|
||||
node public-domain <name> --clear ...it faces the outside no longer
|
||||
node networks <name> the networks it routes for what it hosts
|
||||
node networks <name> <cidr>... ...set them; its filter forwards these too
|
||||
node networks <name> --clear ...only the container runtime's own
|
||||
token issue --node <name> a one-time right to join, for an existing record
|
||||
token issue --new <name> create the record and issue for it
|
||||
token issue ... --adopted ...for a machine in use, which joins adopted
|
||||
@@ -154,6 +224,7 @@ func usage() {
|
||||
overlay place <node> [flags] say where a node is and how it is reached
|
||||
overlay show the private network, as the mesh computes it
|
||||
module add <file> register a module from its manifest
|
||||
module check <file|dir>... judge manifests where they are written, with no mesh (exit 1 on any problem)
|
||||
module list what modules this mesh knows about
|
||||
module moved <name> <commit> the source has a newer commit than the mesh built
|
||||
module forget <name> remove one, unless a node runs it or the mesh holds things for it
|
||||
@@ -163,12 +234,22 @@ func usage() {
|
||||
upgrade <name> roll-out [--together] ...send it to the machines running it
|
||||
upgrade <name> record ...record that they are behind, and send nothing
|
||||
status [--json] what is wrong, what is quiet, and what is out of date
|
||||
retire [--json] every provider waiting for a person to approve a retirement (ADR 0230)
|
||||
retire approve <node> <module> --why <text> retire what it waits with: access off, data kept
|
||||
retire reject <node> <module> --why <text> keep them active; a warning stays open
|
||||
cleanup [list] [--json] every retired consumer per provider: age, size, why
|
||||
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
|
||||
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||
assign <node> <module> put a module on a node
|
||||
unassign <node> <module> take it off
|
||||
take <node> <module> cut a module over on an adopted node, once its data has moved
|
||||
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||
unassign <node> <module>... take them off
|
||||
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
||||
what it declares; --yes <digest> cuts it over as previewed
|
||||
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
||||
adopt <node> return a converged node to adopted; what was taken stays taken
|
||||
settings set <module> <file> what a module's config should say, for the whole mesh
|
||||
@@ -186,17 +267,42 @@ func usage() {
|
||||
build --behind build every module the mesh holds older than its source
|
||||
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
||||
builds [<module>] what has been built lately, and what came of it
|
||||
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
|
||||
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
|
||||
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
|
||||
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
|
||||
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||
record an act done by hand outside the mesh (to-be 45 §7)
|
||||
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||
conditions [--scope S] [--severity S] [--machine M] [--json]
|
||||
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
||||
conditions show <key> one condition whole, with its evidence
|
||||
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
||||
conditions history [--days N] [--key K] every raising, change and clearing lately
|
||||
doctor [run|probes|signals] [--json]
|
||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
||||
durations [--kind K] [--days N] [--json]
|
||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||
licence refresh <name> mint a new access token and seal it to every holder
|
||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
||||
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||
pin <node> <provision> <from> which node this one gets a provision from
|
||||
pin <node> <provision> <from-node> <module>
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
unpin <node> <provision> put that question back
|
||||
plan <node> [--files|--json] what that node would run, and why
|
||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
||||
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||
that need it; --why records it in the hand-act log
|
||||
version what this binary is
|
||||
|
||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||
@@ -233,6 +339,58 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Built is the daemon hearing a build's outcome on the bus — its own asking, an announcement's, or
|
||||
// a tool's that did not wait (novox/hq issue 176) — and taking it in: recorded, and the module
|
||||
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
||||
// became of a build nobody was watching.
|
||||
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
return b.inv.RecordBuild(ctx, buildFrom(result))
|
||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||
// recorded or registered.
|
||||
if result.Check != nil || result.Checked != nil {
|
||||
checked(ctx, result)
|
||||
return nil
|
||||
}
|
||||
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
|
||||
// unmerged branch was heard here like any build, registered, and its definition reached a machine
|
||||
// before anyone had reviewed it.
|
||||
if result.DryRun {
|
||||
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||
recordBuildDuration(ctx, b.inv, result, asked)
|
||||
switch {
|
||||
case err != nil && result.Failed != "":
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
if result.Module != "" {
|
||||
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
|
||||
} else {
|
||||
planFailedBuild(ctx, b.open, result)
|
||||
}
|
||||
return nil
|
||||
case errors.Is(err, inventory.ErrSuperseded):
|
||||
// Not a failure: the module is already at what a later request built. A plan that asked
|
||||
// before that later request is answered by it; one that asked after it ignores this.
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
return nil
|
||||
case err != nil:
|
||||
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
|
||||
if manifest.Module != "" {
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,414 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The merge gate (novox/hq to-be 45 §9): a change judged against every machine of the snapshot, by the
|
||||
// incidents it is written from. Each case raises a mesh, takes its snapshot, and judges a pull request's
|
||||
// tree against it in throwaway stores of its own.
|
||||
|
||||
// catalogueMesh is two machines and a catalogue repository: a resolver and an object store on the
|
||||
// anchor, and on the laptop a network manager requiring the resolver, an album requiring the object
|
||||
// store, and a login manager. Every module is registered from novox/mesh-catalog, as the mesh's are.
|
||||
func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifests := map[string]string{
|
||||
"objects": `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
|
||||
"resolver": `{"module":"resolver","version":"1",
|
||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`,
|
||||
"networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`,
|
||||
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
|
||||
"lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"],
|
||||
"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`,
|
||||
}
|
||||
for name, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"},
|
||||
{"laptop", "album"}, {"laptop", "lemurs"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f, manifests
|
||||
}
|
||||
|
||||
// aTree is a checkout of the catalogue repository holding these manifests.
|
||||
func aTree(t *testing.T, manifests map[string]string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for name, raw := range manifests {
|
||||
at := filepath.Join(dir, "modules", name)
|
||||
if err := os.MkdirAll(at, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(raw), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict {
|
||||
t.Helper()
|
||||
admin := os.Getenv("MESH_TEST_POSTGRES")
|
||||
in := mergeCheckInput{facts: f, admin: admin, changed: changed}
|
||||
if tree != "" {
|
||||
in.repository, in.tree = "novox/mesh-catalog", tree
|
||||
}
|
||||
v, err := judgeChange(t.Context(), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func withEdit(manifests map[string]string, name, raw string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for k, v := range manifests {
|
||||
out[k] = v
|
||||
}
|
||||
if raw == "" {
|
||||
delete(out, name)
|
||||
} else {
|
||||
out[name] = raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The mesh as it is passes: every machine composes in the gate's store as the controller composed it.
|
||||
func TestTheMeshAsItIsPassesTheGate(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
for _, m := range f.Machines {
|
||||
if !m.Declaration.Composes {
|
||||
t.Fatalf("the mesh itself does not compose: %+v", m.Declaration)
|
||||
}
|
||||
}
|
||||
v := gateJudged(t, f, aTree(t, manifests))
|
||||
if v.Verdict != "pass" {
|
||||
t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report())
|
||||
}
|
||||
for _, m := range v.Machines {
|
||||
if !m.Base.Composes || !m.Change.Composes {
|
||||
t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 263**: a change makes the network manager require the object store's provision; on a machine
|
||||
// whose name is six characters its identity is 26 against a bound of 20. Refused in the pull request,
|
||||
// naming the module, and not on the anchor after it merged.
|
||||
func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
tree := aTree(t, withEdit(manifests, "networkmanager",
|
||||
`{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`))
|
||||
v := gateJudged(t, f, tree)
|
||||
if v.Verdict != "fail" {
|
||||
t.Fatalf("the overflow passed the gate:\n%s", v.Report())
|
||||
}
|
||||
report := v.Report()
|
||||
if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") {
|
||||
t.Errorf("the refusal does not name the module and the provision:\n%s", report)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 236**: a login manager's service that omits its state passed the catalogue check and was
|
||||
// refused whole by the node-engine on the first machine. A change to a module a machine runs, and a
|
||||
// module nobody runs yet, are both refused before merge, naming the machine and the module.
|
||||
func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
broken := `{"module":"lemurs","version":"1","capabilities":["systemd"],
|
||||
"resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}`
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") {
|
||||
t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report())
|
||||
}
|
||||
laptop := snapshot.Pseudonym("machine", "laptop")
|
||||
if !strings.Contains(v.Report(), laptop) {
|
||||
t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report())
|
||||
}
|
||||
|
||||
// And as a new module, which no machine runs: tried on one that could.
|
||||
newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`)
|
||||
newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service")
|
||||
v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") {
|
||||
t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// **Version skew**: a manifest the controller judging it cannot read — the one the mesh runs, for a
|
||||
// catalogue change — fails here, not at registration after the merge.
|
||||
func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album",
|
||||
`{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`)))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") {
|
||||
t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// A module a machine runs, removed from its source, fails until it is unassigned (ADR 0236).
|
||||
func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", "")))
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") {
|
||||
t.Fatalf("removing a module a machine runs passed:\n%s", v.Report())
|
||||
}
|
||||
}
|
||||
|
||||
// **Issues 278 and 280**: a file in no held module's directory read as shared code, and a merge rebuilt the
|
||||
// catalogue. A file is a module's only by being inside it — no build reads one outside — so it rebuilds
|
||||
// nothing, and the gate says why; a merge that does rebuild widely is warned of.
|
||||
func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) {
|
||||
f, manifests := catalogueMesh(t)
|
||||
was := wideRebuild
|
||||
wideRebuild = 2
|
||||
t.Cleanup(func() { wideRebuild = was })
|
||||
for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} {
|
||||
v := gateJudged(t, f, aTree(t, manifests), file)
|
||||
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" {
|
||||
t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict)
|
||||
}
|
||||
if !strings.Contains(v.Report(), "read by no module's build") {
|
||||
t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report())
|
||||
}
|
||||
}
|
||||
v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go")
|
||||
if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" {
|
||||
t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict)
|
||||
}
|
||||
// With the reference module's definition in the tree, its directory is a module, held or not.
|
||||
withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`)
|
||||
v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts")
|
||||
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 {
|
||||
t.Fatalf("a file of a module nobody holds reads %+v", v.Width)
|
||||
}
|
||||
v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json")
|
||||
if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" {
|
||||
t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
// **A delivery group is judged as one future state** (novox/hq ADR 0239): a catalogue change that needs a
|
||||
// provision only another repository's change adds fails alone and passes composed with it; and a group
|
||||
// whose other head breaks what the first needs fails, naming it.
|
||||
func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) {
|
||||
f, manifests := catalogueMeshWithAnApp(t)
|
||||
needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`)
|
||||
catTree := aTree(t, needsTheApp)
|
||||
alone := gateJudged(t, f, catTree, "modules/album/module.json")
|
||||
if alone.Verdict != "fail" {
|
||||
t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report())
|
||||
}
|
||||
app := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1",
|
||||
"provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog",
|
||||
tree: catTree, changed: []string{"modules/album/module.json"},
|
||||
group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}}
|
||||
together, err := judgeChange(t.Context(), in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if together.Verdict == "fail" {
|
||||
t.Fatalf("the group composed together fails:\n%s", together.Report())
|
||||
}
|
||||
if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" {
|
||||
t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
// catalogueMeshWithAnApp is catalogueMesh with an application built from a repository of its own, at its
|
||||
// root, on the anchor.
|
||||
func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifests := map[string]string{
|
||||
"objects": `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
|
||||
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
|
||||
}
|
||||
for name, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f, manifests
|
||||
}
|
||||
|
||||
// busMesh is aMesh with a module on the bus on the laptop, its account issued as `module issue` issues
|
||||
// one: minted, and its credential held as the module's own secret named broker.
|
||||
func busMesh(t *testing.T) snapshot.Facts {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1",
|
||||
"own-secrets":{"broker":"/var/lib/speaker/broker"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "speaker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username()
|
||||
password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule,
|
||||
Node: "laptop", Module: "speaker"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker",
|
||||
`{"user":"`+user+`","password":"`+password+`"}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mc := range f.Machines {
|
||||
if !mc.Declaration.Composes {
|
||||
t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration)
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// **Issue 282**: every machine running a module on the bus failed to compose in the gate's store, with
|
||||
// the change and without — the store held the module's credential and no account for it, which
|
||||
// composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the
|
||||
// mesh issued is raised with its credential, so the machine composes in the gate as on the mesh.
|
||||
func TestIssue282AModuleOnTheBusComposesInTheGate(t *testing.T) {
|
||||
f := busMesh(t)
|
||||
v := gateJudged(t, f, "")
|
||||
if v.Verdict != "pass" {
|
||||
t.Fatalf("the mesh as it is does not pass:\n%s", v.Report())
|
||||
}
|
||||
for _, m := range v.Machines {
|
||||
if !m.Base.Composes {
|
||||
t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(v.Summary, "2 of 2 compose") {
|
||||
t.Errorf("the summary does not say every machine composes: %s", v.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 282**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
|
||||
// against a machine that is not the mesh's — broken against broken, which passes whatever the change does.
|
||||
// That is an error, never a pass.
|
||||
func TestIssue282AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
|
||||
f := busMesh(t)
|
||||
for i := range f.Machines {
|
||||
// A fact the snapshot does not carry: the module's credential, gone from the laptop's.
|
||||
var kept []snapshot.Accepted
|
||||
for _, a := range f.Machines[i].Accepted {
|
||||
if a.Name != "broker" {
|
||||
kept = append(kept, a)
|
||||
}
|
||||
}
|
||||
f.Machines[i].Accepted = kept
|
||||
}
|
||||
v := gateJudged(t, f, "")
|
||||
if v.Verdict != "error" {
|
||||
t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report())
|
||||
}
|
||||
if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") {
|
||||
t.Errorf("the error does not name what could not be raised: %v", v.Errors)
|
||||
}
|
||||
}
|
||||
|
||||
// A path the snapshot withheld is given a path of its own where an access or a place needs one: a bare
|
||||
// "withheld" is no absolute path, and a machine whose setting composes on the mesh would not in the gate.
|
||||
func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
|
||||
got := standInPaths(map[string]any{
|
||||
"accesses": map[string]any{"races": "withheld", "films": "/media/films", "shows": "/withheld"},
|
||||
"places": map[string]any{"config": map[string]any{"path": "withheld", "owner": "1000:1000"}},
|
||||
"puid": 1000,
|
||||
})
|
||||
accesses := got["accesses"].(map[string]any)
|
||||
if accesses["races"] == accesses["shows"] || !strings.HasPrefix(accesses["races"].(string), "/") ||
|
||||
!strings.HasPrefix(accesses["shows"].(string), "/") || accesses["films"] != "/media/films" {
|
||||
t.Errorf("accesses: %v", accesses)
|
||||
}
|
||||
place := got["places"].(map[string]any)["config"].(map[string]any)
|
||||
if !strings.HasPrefix(place["path"].(string), "/") || place["owner"] != "1000:1000" || got["puid"] != 1000 {
|
||||
t.Errorf("places: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 283**: a check run by hand clones beside a change what the seat clones — one rule, read by the
|
||||
// controller's ask and by the facts — and finds the repository it checks from its origin.
|
||||
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
|
||||
for dir, refs := range map[string]map[string]string{
|
||||
"mesh-catalog": {"mesh-catalog": "main"},
|
||||
"mesh-host": {"mesh-host": "c0ffee"},
|
||||
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
|
||||
} {
|
||||
got := besideRefs(dir, "c0ffee")
|
||||
for d, ref := range refs {
|
||||
if got[d] != ref {
|
||||
t.Errorf("beside %s, %s is cloned at %q, not %q", dir, d, got[d], ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
for owner, want := range map[string][3]string{
|
||||
"ssh://git@git.example:222/novox/mesh-host.git": {"novox", "mesh-host", "ssh://git@git.example:222/novox"},
|
||||
"git@git.example:novox/mesh-tools.git": {"novox", "mesh-tools", "git@git.example:novox"},
|
||||
"http://git.example/novox/hq": {"novox", "hq", "http://git.example/novox"},
|
||||
} {
|
||||
o, r, p := ownerRepoOf(owner)
|
||||
if [3]string{o, r, p} != want {
|
||||
t.Errorf("%s reads as %s %s %s", owner, o, r, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"crypto/ecdh"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -37,6 +40,9 @@ func aMesh(t *testing.T) *stores {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
|
||||
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
|
||||
withConditionsInMemory(t)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -106,3 +112,17 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
||||
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||
}
|
||||
|
||||
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
|
||||
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
|
||||
t.Helper()
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
before := conditionsFrom
|
||||
conditionsFrom = k
|
||||
t.Cleanup(func() {
|
||||
conditionsFrom = before
|
||||
k.Close(context.Background())
|
||||
})
|
||||
return k, store
|
||||
}
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
|
||||
//
|
||||
// The forge's poll announces every merge on the events stream, and the controller acts on what its
|
||||
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
|
||||
// server moved the consumer past it — a fault of consumers with several filters in the server the
|
||||
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
|
||||
// built from that repository stayed behind and were built by hand.
|
||||
//
|
||||
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
|
||||
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
|
||||
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
|
||||
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
|
||||
// would still move something was never acted on — it is said, and acted on now.
|
||||
const (
|
||||
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
|
||||
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
|
||||
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
|
||||
mergeGrace = 10 * time.Minute
|
||||
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
|
||||
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
|
||||
mergeLookBack = 24 * time.Hour
|
||||
// mergeCatchUpEvery is how often the stream is read back.
|
||||
mergeCatchUpEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
// merges is what reads back the forge's announcements; the link server, or a test's list.
|
||||
type merges interface {
|
||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||
}
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := open.inventory.ReadRepositories(ctx)
|
||||
return entries, read, err
|
||||
}
|
||||
failing := ""
|
||||
tick := time.NewTicker(mergeCatchUpEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
watchedMerges.begin()
|
||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||
// says that instead, and leaves what the last one found standing.
|
||||
watchedMerges.end(time.Now(), err)
|
||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("merges the bus may not have handed over are looked for again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
|
||||
// move something is said and acted on, oldest first.
|
||||
//
|
||||
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
|
||||
// because acting on an earlier missed merge of the same repository may have moved what a later one
|
||||
// would have.
|
||||
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
|
||||
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
|
||||
|
||||
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, read, err := catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range all {
|
||||
if now.Sub(a.At) < mergeGrace {
|
||||
continue
|
||||
}
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
moves := wouldMove(a.SourceMoved, entries, read)
|
||||
if len(moves) == 0 {
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moves {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
||||
At: a.At, Modules: names})
|
||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||
isAre(len(names)))
|
||||
if err := act(ctx, a.SourceMoved); err != nil {
|
||||
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
|
||||
a.Owner, a.Repo, a.Commit, err)
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
||||
type missedMerge struct {
|
||||
Owner, Repo, Base, Commit string
|
||||
// At is when the bus took the announcement.
|
||||
At time.Time
|
||||
Modules []string
|
||||
}
|
||||
|
||||
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
||||
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
||||
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
||||
// the fault. The next pass, finding it acted on, clears it.
|
||||
type mergeWatch struct {
|
||||
mu sync.Mutex
|
||||
passed time.Time
|
||||
err error
|
||||
finding []missedMerge
|
||||
missed []missedMerge
|
||||
}
|
||||
|
||||
var watchedMerges = &mergeWatch{}
|
||||
|
||||
func (w *mergeWatch) begin() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = nil
|
||||
}
|
||||
|
||||
func (w *mergeWatch) found(m missedMerge) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = append(w.finding, m)
|
||||
}
|
||||
|
||||
func (w *mergeWatch) end(at time.Time, err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.passed, w.err = at, err
|
||||
if err == nil {
|
||||
w.missed = w.finding
|
||||
}
|
||||
}
|
||||
|
||||
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
||||
// could not read.
|
||||
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// announcedList is the events stream's merges as a test gives them.
|
||||
type announcedList []link.AnnouncedMerge
|
||||
|
||||
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
var out []link.AnnouncedMerge
|
||||
for _, m := range a {
|
||||
if !m.At.Before(since) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
|
||||
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
|
||||
type aCatalogue struct {
|
||||
entries []inventory.Entry
|
||||
acted []string
|
||||
fail error
|
||||
}
|
||||
|
||||
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
return append([]inventory.Entry(nil), c.entries...), nil, nil
|
||||
}
|
||||
|
||||
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
|
||||
return func(_ context.Context, m link.SourceMoved) error {
|
||||
if c.fail != nil {
|
||||
return c.fail
|
||||
}
|
||||
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
|
||||
for _, moved := range wouldMove(m, c.entries, nil) {
|
||||
for i := range c.entries {
|
||||
if c.entries[i].Manifest.Module == moved.Manifest.Module {
|
||||
c.entries[i].Source.Head = m.Commit
|
||||
c.entries[i].Source.Seen = now()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func at(s string) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
|
||||
return link.AnnouncedMerge{
|
||||
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||
MergedAt: mergedAt, Paths: paths,
|
||||
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
|
||||
At: at(onTheBus),
|
||||
}
|
||||
}
|
||||
|
||||
func built(module, repo, path, commit, seen string) inventory.Entry {
|
||||
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
|
||||
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
|
||||
return e
|
||||
}
|
||||
|
||||
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
|
||||
// events stream; the bus never handed it to the controller, which acted on the merges around it and
|
||||
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
|
||||
// move something — so it is said and acted on, once, and only after the controller's own consumer
|
||||
// has had its time with it.
|
||||
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{
|
||||
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
// Acted on when it was announced: looked at after it was merged.
|
||||
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
|
||||
}}
|
||||
stream := announcedList{
|
||||
// Nothing the mesh holds is built from the records repository.
|
||||
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
|
||||
// Acted on: its module was looked at since.
|
||||
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
|
||||
// Never handed over.
|
||||
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
|
||||
"node-tools/internal/console/console.go"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:50:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
pass()
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T22:58:00Z")
|
||||
pass()
|
||||
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
|
||||
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
|
||||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
|
||||
t.Fatalf("the missed merge was not said as one: %q", said)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T23:03:00Z")
|
||||
pass()
|
||||
if len(cat.acted) != 1 || len(said) != 1 {
|
||||
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
|
||||
// that could not be acted on is said and tried again on the next pass.
|
||||
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
|
||||
cat := &aCatalogue{
|
||||
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
|
||||
fail: errors.New("the store is restarting"),
|
||||
}
|
||||
stream := announcedList{
|
||||
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
|
||||
"modules/plex/module.json", "modules/plex/x.ts"),
|
||||
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
|
||||
t.Fatalf("a failed catch-up was not said: %q", said)
|
||||
}
|
||||
cat.fail = nil
|
||||
clock = at("2026-10-05T22:05:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
|
||||
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge older than the look-back is left to the operator: a controller that did not run this
|
||||
// missed it, and acting on it days later would be a surprise rebuild.
|
||||
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
|
||||
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
|
||||
func(string, ...any) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
||||
// Phase A).
|
||||
//
|
||||
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
||||
// machine states, in every report and as an event between reports, the state of every long-running
|
||||
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
||||
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
||||
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
||||
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
||||
// the stated health: a judging passes a module only when every long-running resource of it on that
|
||||
// machine is stated healthy, so a resource still starting is not yet a pass.
|
||||
//
|
||||
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
||||
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
||||
|
||||
// The condition a module's health raises.
|
||||
const (
|
||||
kindModuleUnhealthy = "module-unhealthy"
|
||||
// sourceHealth is what raised it: the machine's own statement.
|
||||
sourceHealth = "health"
|
||||
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
||||
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
||||
// moduleUnhealthyAfter is how many statements in a row raise it.
|
||||
moduleUnhealthyAfter = 2
|
||||
)
|
||||
|
||||
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
||||
var healthRefused atomic.Int64
|
||||
|
||||
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
||||
type moduleHealth struct {
|
||||
inv *inventory.Inventory
|
||||
keeper func() *conditions.Keeper
|
||||
}
|
||||
|
||||
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
||||
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
||||
}
|
||||
|
||||
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
||||
// older statement than the one kept is refused, by when the engine looked.
|
||||
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
||||
now time.Time) error {
|
||||
if h.Contract == 0 {
|
||||
return nil
|
||||
}
|
||||
prev, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if had && h.At.Before(prev.SaidAt) {
|
||||
healthRefused.Add(1)
|
||||
return nil
|
||||
}
|
||||
unhealthy := map[string][]inventory.ResourceHealth{}
|
||||
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
||||
for _, r := range h.Resources {
|
||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||
Check: r.Check, Needs: r.Needs}
|
||||
resources = append(resources, kept)
|
||||
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||
}
|
||||
}
|
||||
streaks := map[string]int{}
|
||||
for module := range unhealthy {
|
||||
streaks[module] = prev.Streaks[module] + 1
|
||||
}
|
||||
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
||||
HeardAt: now, Resources: resources, Streaks: streaks})
|
||||
if err != nil || !stored {
|
||||
if err == nil {
|
||||
healthRefused.Add(1)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
||||
}
|
||||
|
||||
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
||||
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
||||
// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
|
||||
// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
|
||||
// waits on it.
|
||||
func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
|
||||
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
standing := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
var hold *holding
|
||||
if inv != nil {
|
||||
if hold, err = readHolding(ctx, inv, open); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
modules := make([]string, 0, len(unhealthy))
|
||||
for m := range unhealthy {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
seen := map[string]bool{}
|
||||
heldOn := map[string]string{}
|
||||
providers := map[catalogue.Chosen]bool{}
|
||||
for _, m := range modules {
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
if hold != nil {
|
||||
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node
|
||||
providers[p] = true
|
||||
continue
|
||||
}
|
||||
if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
}
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
c, isOpen := standing[o.Key()]
|
||||
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
||||
}
|
||||
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
for key, c := range standing {
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
||||
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
||||
if on, held := heldOn[key]; held {
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||
}
|
||||
if _, err := k.Clear(ctx, key, why); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
// And each provider a consumer here now waits on, when its own condition is open: said again with who
|
||||
// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
|
||||
for p := range providers {
|
||||
if err := sayWaiters(ctx, k, hold, p, now); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||
var raisedAt *conditions.Condition
|
||||
for i, c := range hold.open {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
||||
raisedAt = &hold.open[i]
|
||||
}
|
||||
}
|
||||
if raisedAt == nil {
|
||||
return nil
|
||||
}
|
||||
var rs []inventory.ResourceHealth
|
||||
for _, r := range hold.healths[p.Node].Resources {
|
||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||
rs = append(rs, r)
|
||||
}
|
||||
}
|
||||
if len(rs) == 0 {
|
||||
return nil
|
||||
}
|
||||
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||
if waiters := hold.waitersOn(p); len(waiters) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
}
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
var words, said []string
|
||||
for _, r := range rs {
|
||||
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
||||
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
||||
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
||||
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||
Said: strings.Join(said, "; ")}
|
||||
}
|
||||
|
||||
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||
func reasonWords(r inventory.ResourceHealth) string {
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
||||
case "down":
|
||||
return "is not running"
|
||||
case "":
|
||||
return "is unhealthy"
|
||||
}
|
||||
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||
if r.Check != "" {
|
||||
return "fails its " + r.Check + " check"
|
||||
}
|
||||
return "is unhealthy: " + r.Reason
|
||||
}
|
||||
|
||||
func orNotSaid(s string) string {
|
||||
if s == "" {
|
||||
return "no reason said"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
||||
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
||||
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
||||
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
||||
if f.healthErr != nil {
|
||||
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
||||
}
|
||||
h, states := f.health[machine]
|
||||
if !states {
|
||||
return healthGood, ""
|
||||
}
|
||||
if h.HeardAt.Before(since) {
|
||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||
}
|
||||
for _, r := range h.Resources {
|
||||
if r.Module != module {
|
||||
continue
|
||||
}
|
||||
switch r.State {
|
||||
case link.StateHealthy:
|
||||
case link.StateStarting:
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||
case link.StateUnhealthy:
|
||||
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||
r.Resource, machine, on)
|
||||
}
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||
default:
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
||||
reasonAfter(r.Reason))
|
||||
}
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
func reasonAfter(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
return ": " + s
|
||||
}
|
||||
|
||||
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
||||
// since when, an unhealthy one said once marked unconfirmed.
|
||||
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
if !had {
|
||||
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
||||
}
|
||||
if len(h.Resources) == 0 {
|
||||
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
||||
}
|
||||
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
||||
for _, r := range h.Resources {
|
||||
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
||||
r.Since.Local().Format("2006-01-02 15:04"))
|
||||
if r.Reason != "" {
|
||||
line += " — " + r.Reason
|
||||
}
|
||||
if r.Restarts > 0 {
|
||||
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
||||
}
|
||||
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
||||
line += " (unconfirmed: said once)"
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module's stated health, as the controller keeps it and raises from it (novox/hq ADR 0240, "how it is
|
||||
// checked", rule 4): one unhealthy statement raises nothing and is listed unconfirmed; two raise; a
|
||||
// healthy one clears; a condition from before the send clears at the new build's start; an older
|
||||
// statement is refused; and an engine that states nothing raises nothing.
|
||||
|
||||
var h0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func aStatement(at time.Time, states ...string) link.Health {
|
||||
h := link.Health{Contract: link.LivenessContract, At: at}
|
||||
for i, s := range states {
|
||||
r := link.ResourceHealth{Module: "letta", Resource: "letta.server", Kind: "container", Target: "letta-server",
|
||||
State: s, Since: at}
|
||||
if i > 0 {
|
||||
r.Module, r.Resource, r.Target = "mqtt", "mqtt.broker", "mosquitto.service"
|
||||
}
|
||||
if s == link.StateUnhealthy {
|
||||
r.Reason, r.Restarts, r.Streak = "restarting", 4, 2
|
||||
}
|
||||
h.Resources = append(h.Resources, r)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func TestTwoUnhealthyStatementsRaiseTheModulesConditionAndAHealthyOneClearsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
k := conditionsFrom
|
||||
const key = "module.letta.anchor.unhealthy"
|
||||
openKeys := func() []string {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range list {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0, link.StateHealthy, link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// One statement: nothing raised, and `node show` lists it as unconfirmed.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||
h0.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("one statement raised %v", keys)
|
||||
}
|
||||
kept, had, err := inv.HealthOf(ctx, "anchor")
|
||||
if err != nil || !had {
|
||||
t.Fatalf("the statement was not kept: %v %v", had, err)
|
||||
}
|
||||
if lines := strings.Join(healthLines(kept, had, h0.Add(time.Minute)), "\n"); !strings.Contains(lines, "unconfirmed") ||
|
||||
!strings.Contains(lines, "letta.server") {
|
||||
t.Fatalf("node show does not list the first statement as unconfirmed:\n%s", lines)
|
||||
}
|
||||
|
||||
// The second in a row raises it — the module's own, never the other module's on the machine.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(2*time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||
h0.Add(2*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 1 || keys[0] != key {
|
||||
t.Fatalf("two statements raised %v, not %s", keys, key)
|
||||
}
|
||||
c, _, _ := k.Get(ctx, key)
|
||||
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverSelf || c.Subject.Machine != "anchor" ||
|
||||
!strings.Contains(c.Summary, "letta on anchor") || !strings.Contains(c.Summary, "keeps restarting") ||
|
||||
!strings.Contains(c.Evidence[0].Said, "letta-server") {
|
||||
t.Fatalf("the condition does not say it in words with its evidence: %+v", c)
|
||||
}
|
||||
|
||||
// Standing four hours, it is urgent.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(5*time.Hour), link.StateUnhealthy, link.StateHealthy),
|
||||
time.Now().Add(5*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _, _ := k.Get(ctx, key); c.Severity != conditions.Urgent {
|
||||
t.Fatalf("unhealthy for four hours is still %s", c.Severity)
|
||||
}
|
||||
|
||||
// A new build's start — every start begins in `starting` — clears it: what follows is the new build's.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(6*time.Hour), link.StateStarting, link.StateHealthy),
|
||||
h0.Add(6*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("the first statement that says no resource is unhealthy did not clear it: %v", keys)
|
||||
}
|
||||
// And the streak starts again: one unhealthy statement after it raises nothing.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(7*time.Hour), link.StateUnhealthy), h0.Add(7*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("one statement after a clearing raised %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderHealthStatementIsRefused(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
before := healthRefused.Load()
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0.Add(time.Minute), link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An event said before the report that overtook it arrives late.
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateUnhealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, _, err := inv.HealthOf(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !kept.SaidAt.Equal(h0.Add(time.Minute)) || kept.Resources[0].State != link.StateHealthy {
|
||||
t.Fatalf("the older statement replaced the newer: %+v", kept)
|
||||
}
|
||||
if healthRefused.Load() != before+1 {
|
||||
t.Fatalf("the refusal was not counted")
|
||||
}
|
||||
}
|
||||
|
||||
// **An engine older than the judging states nothing**: its reports raise nothing, keep nothing, and the
|
||||
// gate judges its machine as before — never healthy for having said nothing, never unhealthy.
|
||||
func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
l := nudgingListener{Enrolment: link.Enrolment{Inventory: open.inventory}}
|
||||
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Applied: []string{"letta.server"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || had {
|
||||
t.Fatalf("health kept for an engine that said none: %v %v", had, err)
|
||||
}
|
||||
if lines := healthLines(inventory.NodeHealth{}, false, time.Now()); !strings.Contains(lines[0], "older than the judging") {
|
||||
t.Fatalf("node show: %v", lines)
|
||||
}
|
||||
// And one that does, through the report, is kept.
|
||||
h := aStatement(time.Now().UTC(), link.StateHealthy)
|
||||
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Health: &h}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || !had || len(kept.Resources) != 1 {
|
||||
t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A declared `health` is sent only to an engine whose own statement says it reads it (novox/hq ADR 0240
|
||||
// Phase B): an older engine is strict and would refuse the whole declaration for the field.
|
||||
func TestHealthIsSentOnlyToAnEngineThatSaysItReadsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
reads := func() bool {
|
||||
t.Helper()
|
||||
got, err := engineReadsHealth(ctx, inv, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
if reads() {
|
||||
t.Fatal("an engine that never stated anything is sent health")
|
||||
}
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reads() {
|
||||
t.Fatal("an engine judging liveness alone is sent health")
|
||||
}
|
||||
later := aStatement(h0.Add(time.Minute), link.StateHealthy)
|
||||
later.Contract = link.ReadinessContract
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", later, h0.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reads() {
|
||||
t.Fatal("an engine that reads health is not sent it")
|
||||
}
|
||||
}
|
||||
+247
-112
@@ -2,8 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
@@ -42,7 +40,12 @@ func providedModules() []catalogue.Manifest {
|
||||
// and neither could be swapped for anything, which is the test of whether a thing is a
|
||||
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
|
||||
// to live, and now a module says where it wants it — `facts` in its own manifest.
|
||||
overlay.Manifest(), overlay.DomainManifest(),
|
||||
//
|
||||
// **And the bundle that required it is gone** (novox/hq ADR 0226): `networking` named this
|
||||
// module's requirement and nothing else, so every machine carried two modules for one
|
||||
// network. A machine is assigned the private network itself; what a release stops shipping
|
||||
// is retired at the next start (stores.go, Inventory.RetireUnshipped).
|
||||
overlay.Manifest(),
|
||||
} {
|
||||
var m catalogue.Manifest
|
||||
b, _ := json.Marshal(raw)
|
||||
@@ -56,7 +59,35 @@ var provided = providedModules()
|
||||
|
||||
func moduleCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("module add <file>, module list, or module forget <name>")
|
||||
return errors.New("module add <file>, module check <file>..., module list, or module forget <name>")
|
||||
}
|
||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||
if args[0] == "check" {
|
||||
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
||||
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
||||
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
||||
"judge each module's identity on a machine name this many characters long")
|
||||
given, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *longest < 1 {
|
||||
return errors.New("--longest-machine-name is a length, at least 1")
|
||||
}
|
||||
var paths []string
|
||||
for _, a := range given {
|
||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||
under, err := manifestsUnder(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
paths = append(paths, under...)
|
||||
continue
|
||||
}
|
||||
paths = append(paths, a)
|
||||
}
|
||||
return moduleCheckFor(paths, *longest, os.Stdout)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -71,12 +102,20 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
repo := set.String("source", "", "where this module comes from")
|
||||
ref := set.String("ref", "", "the branch followed there")
|
||||
commit := set.String("commit", "", "the commit this manifest was read at")
|
||||
// **Where inside the repository the module is** (novox/hq ADR 0069). A module is a
|
||||
// repository *and* a directory, and a record that carries only the repository names a
|
||||
// module.json at its root — so every later build of it looks in the wrong place and fails
|
||||
// with "no module.json at its root". Nine modules on this mesh were registered that way
|
||||
// and none of them could be rebuilt (2026-09-28).
|
||||
path := set.String("path", "", "the module's directory inside that repository")
|
||||
self := set.Bool("self", false, "the source is a path on the forge holding the git seat")
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("module add <manifest.json> [--source <repo> --ref <branch> --commit <sha>]")
|
||||
return errors.New("module add <manifest.json> [--source <repo> [--self] [--path P] " +
|
||||
"--ref <branch> --commit <sha>]")
|
||||
}
|
||||
raw, err := os.ReadFile(positionals[0])
|
||||
if err != nil {
|
||||
@@ -86,23 +125,27 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Provenance together or not at all. A source with no commit cannot be compared against
|
||||
// anything, so it would record where the module came from and still never be able to say
|
||||
// the mesh is behind it — which is the one thing recording it is for.
|
||||
if (*repo == "") != (*commit == "") {
|
||||
return errors.New("--source and --commit go together: a source with no commit " +
|
||||
"cannot be compared against anything, and a commit with no source has nothing " +
|
||||
"to be compared with")
|
||||
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{
|
||||
Repository: *repo, Ref: *ref, BuiltFrom: *commit,
|
||||
}); err != nil {
|
||||
if err := namesNoInstallation(m); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, from); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s registered", m.Module)
|
||||
if *commit != "" {
|
||||
fmt.Printf(" from %s", short(*commit))
|
||||
}
|
||||
if *repo != "" && *path == "" {
|
||||
// Said, not refused: a module really at the root is the ordinary case for a repository
|
||||
// of its own. But a repository holding many modules and a record naming none of them is
|
||||
// a module nothing can rebuild, and the person adding it is the one who knows which.
|
||||
fmt.Printf("\n no directory inside %s, so it is built from that repository's root — "+
|
||||
"`--path` if the module lives in a directory there", *repo)
|
||||
}
|
||||
if len(m.Provides) > 0 {
|
||||
fmt.Printf(", providing %s", describeOffers(m.Provides))
|
||||
}
|
||||
@@ -120,6 +163,40 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** (novox/hq ADR 0195): what each module
|
||||
// is, where it runs, whether it is current, and what it says of itself.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Built string `json:"built,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Current bool `json:"current"`
|
||||
Provided bool `json:"provided,omitempty"`
|
||||
// Tools says whether the module answers tools anywhere it runs: a list of its own,
|
||||
// a bundle the runtime serves, or a seat's verbs it claims (novox/hq ADR 0197) —
|
||||
// what the console checks the bus's answers against.
|
||||
Tools bool `json:"tools"`
|
||||
On []string `json:"on"`
|
||||
Provides []string `json:"provides,omitempty"`
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
Claims []string `json:"claims,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
out := make([]listed, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||
for _, c := range m.Claims {
|
||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
fmt.Println("this mesh knows about no modules yet")
|
||||
return nil
|
||||
@@ -261,86 +338,22 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
// made in entirely different ways: on the bus the mesh runs on today an account is a
|
||||
// management call, and on the bus being built it is a row the next composition writes into
|
||||
// the server's user list (novox/hq design 25 §4).
|
||||
busAddress, onNATS, err := broker.OnNATS()
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), busAddress); err != nil {
|
||||
return err
|
||||
}
|
||||
if onNATS {
|
||||
return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress)
|
||||
}
|
||||
|
||||
management, err := broker.ManagementFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The foundation owns the bus; make sure it exists before a module binds onto it.
|
||||
if err := management.EnsureEventExchanges(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
secret := make([]byte, 32)
|
||||
if _, err := rand.Read(secret); err != nil {
|
||||
return err
|
||||
}
|
||||
password := base64.RawURLEncoding.EncodeToString(secret)
|
||||
account, err := management.CreateModuleAccount(ctx, *forNode, module, password, m.Emits, m.Consumes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A consumer's queue, with its dead-letter, is the foundation's to declare — its own account
|
||||
// may not (ADR 0043). Made now, so it exists before the module binds onto it.
|
||||
if len(m.Consumes) > 0 {
|
||||
if err := management.EnsureModuleQueue(ctx, *forNode, module); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
|
||||
}
|
||||
brokerAddr, err := brokerReachableAt(ctx, inv, known, *forNode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The URL and what verifies the broker, together — a mesh's broker presents its own
|
||||
// certificate, in no public trust store, so a URL alone fails at TLS (as `builder issue`).
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
}{
|
||||
URL: fmt.Sprintf("amqps://%s:%s@%s/", account, password, brokerAddr),
|
||||
Fingerprint: known.Fingerprint,
|
||||
// The node and module the account is for, so the runtime names its queue as the mesh
|
||||
// scoped it (<node>.<module>.events) without a manifest having to interpolate a node.
|
||||
Node: *forNode,
|
||||
Module: module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, *forNode, module, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("broker account %s created for %s, scoped to what it emits and consumes\n",
|
||||
account, module)
|
||||
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n",
|
||||
*forNode, *forNode)
|
||||
return nil
|
||||
return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("module has no %q; it has add, list, moved, forget and issue", args[0])
|
||||
return fmt.Errorf("module has no %q; it has add, check, list, moved, forget and issue", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return fmt.Errorf("%s <node> <module>", verb)
|
||||
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
|
||||
// service manager and the package manager — can only go on, or come off, together.
|
||||
if len(args) < 2 {
|
||||
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -354,7 +367,7 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
if verb == "unassign" {
|
||||
act = unassign
|
||||
}
|
||||
said, err := act(ctx, open, args[0], args[1])
|
||||
said, err := act(ctx, open, args[0], args[1:]...)
|
||||
if said != "" {
|
||||
fmt.Println(said)
|
||||
}
|
||||
@@ -391,10 +404,14 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
switch args[0] {
|
||||
case "set":
|
||||
if len(positionals) != 2 {
|
||||
return errors.New("settings set <module> <settings.json> [--node <node>]")
|
||||
return errors.New("settings set <module> <settings.json | {…}> [--node <node>]")
|
||||
}
|
||||
raw, err := os.ReadFile(positionals[1])
|
||||
if err != nil {
|
||||
// A file, or the values themselves when they begin with `{` — which is how the mesh's own
|
||||
// `settings` tool passes them, having no file to hand over (novox/hq issue 198).
|
||||
var raw []byte
|
||||
if strings.HasPrefix(strings.TrimSpace(positionals[1]), "{") {
|
||||
raw = []byte(positionals[1])
|
||||
} else if raw, err = os.ReadFile(positionals[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
var values map[string]any
|
||||
@@ -450,8 +467,8 @@ func describeOffers(offers []catalogue.Offer) string {
|
||||
// database should not change where an existing machine gets its data the day a second one
|
||||
// arrives.
|
||||
func pinCommand(ctx context.Context, args []string, setting bool) error {
|
||||
if setting && len(args) != 3 {
|
||||
return errors.New("pin <node> <provision> <from-node>")
|
||||
if setting && len(args) != 4 {
|
||||
return errors.New("pin <node> <provision> <from-node> <module>")
|
||||
}
|
||||
if !setting && len(args) != 2 {
|
||||
return errors.New("unpin <node> <provision>")
|
||||
@@ -470,17 +487,12 @@ func pinCommand(ctx context.Context, args []string, setting bool) error {
|
||||
fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1])
|
||||
return nil
|
||||
}
|
||||
if args[0] == args[2] {
|
||||
// Allowed by nothing here, and worth saying rather than resolving into a confusing
|
||||
// refusal later: a node providing something to itself is a node-scoped provision, and
|
||||
// this field is for the other kind.
|
||||
return fmt.Errorf("%s cannot get %s from itself; that would be a provision this machine "+
|
||||
"provides, which does not need saying", args[0], args[1])
|
||||
}
|
||||
if err := inv.PinProvision(ctx, args[0], args[1], args[2]); err != nil {
|
||||
// The provider's node may be this same machine: two modules beside the consumer can both
|
||||
// answer a provision, and then the module is the whole question (novox/hq #258).
|
||||
if err := inv.PinProvision(ctx, args[0], args[1], args[2], args[3]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s gets %s from %s\n", args[0], args[1], args[2])
|
||||
fmt.Printf("%s gets %s from %s/%s\n", args[0], args[1], args[2], args[3])
|
||||
fmt.Printf(" run `push %s` to send it\n", args[0])
|
||||
return nil
|
||||
}
|
||||
@@ -498,8 +510,8 @@ const theBrokerSeat = "mesh-broker"
|
||||
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
|
||||
// has adopted it) does the hub stand in, which is where the foundation is by convention.
|
||||
//
|
||||
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the networking
|
||||
// module — not "has an address", which is true of every placed machine and says nothing about
|
||||
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the private network
|
||||
// — not "has an address", which is true of every placed machine and says nothing about
|
||||
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
|
||||
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
|
||||
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
|
||||
@@ -597,7 +609,7 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module,
|
||||
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -617,22 +629,41 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||
}
|
||||
|
||||
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
|
||||
// the runtime (novox/hq ADR 0175), a module otherwise. The username is the same either way — the
|
||||
// runtime is issued through this same path — and the kind is what a reader of the records sees.
|
||||
func busKindOf(module string) string {
|
||||
if module == catalogue.RuntimeModule {
|
||||
return inventory.BusNodeTools
|
||||
}
|
||||
return inventory.BusModule
|
||||
}
|
||||
|
||||
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
|
||||
// secret, and the module's consumer created where the bus can be reached. Split from the minting
|
||||
// so the move can issue every module against a bus whose address it worked out itself
|
||||
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
|
||||
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||
node, busAddress string, known broker.Broker, reachable, user, password string) error {
|
||||
// The seats this module claims, with the verbs each promises (novox/hq ADR 0159): the runtime
|
||||
// serves a claimed seat's verbs with its tools of the same name, and the bus admits only the
|
||||
// holder's subscription — so the runtime tries each claim and the grant decides. Written here
|
||||
// because this file is the one thing the mesh writes that the runtime reads before it speaks.
|
||||
claims, err := claimsFor(ctx, inv, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
Claims []seatClaimed `json:"claims,omitempty"`
|
||||
}{
|
||||
URL: "nats://" + reachable, Fingerprint: known.Fingerprint,
|
||||
Node: node, Module: m.Module, User: user, Password: password,
|
||||
Node: node, Module: m.Module, User: user, Password: password, Claims: claims,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -646,7 +677,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
// durable subscription nobody reads.
|
||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
@@ -670,3 +701,107 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
"machine holding mesh-broker\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// whereItComesFrom is the provenance a module handed over by hand records, and what a record must
|
||||
// say to be worth anything later.
|
||||
//
|
||||
// **A module is a repository and a directory inside it** (novox/hq ADR 0069). A record carrying only
|
||||
// the repository names a module.json at its root, so every later build of it looks in the wrong
|
||||
// place — nine modules on this mesh were registered that way and none of them could be rebuilt
|
||||
// (2026-09-28). The directory cannot be checked from here, because the control plane does not clone;
|
||||
// what can be checked is that the record is whole.
|
||||
func whereItComesFrom(repository, ref, commit, path string, self bool) (inventory.Source, error) {
|
||||
// Provenance together or not at all. A source with no commit cannot be compared against
|
||||
// anything, so it would record where the module came from and still never be able to say the
|
||||
// mesh is behind it — which is the one thing recording it is for.
|
||||
if (repository == "") != (commit == "") {
|
||||
return inventory.Source{}, errors.New("--source and --commit go together: a source with " +
|
||||
"no commit cannot be compared against anything, and a commit with no source has " +
|
||||
"nothing to be compared with")
|
||||
}
|
||||
// A directory or a forge with no repository is half a location, and the half it keeps is the
|
||||
// half nothing can be found with.
|
||||
if repository == "" && (path != "" || self) {
|
||||
return inventory.Source{}, errors.New("--path and --self say where inside a source and " +
|
||||
"which forge holds it, so they need --source: without one there is nothing for them " +
|
||||
"to be part of")
|
||||
}
|
||||
from := inventory.Source{Repository: repository, Ref: ref, BuiltFrom: commit, Path: path}
|
||||
if self {
|
||||
if err := onASeat(repository); err != nil {
|
||||
return inventory.Source{}, err
|
||||
}
|
||||
from.Seat = gitSeat
|
||||
}
|
||||
return from, nil
|
||||
}
|
||||
|
||||
// namesNoInstallation is the mesh refusing a definition that names an installation, at the moment
|
||||
// it would enter the catalogue (novox/hq ADR 0112, ADR 0155). `module check` says the same thing
|
||||
// earlier, where the author is; this is the last moment the mesh can still say no, and a
|
||||
// definition that got past the check — written elsewhere, or checked by nobody — is refused here
|
||||
// in the same words. A name meant on purpose is declared with its reason and passes.
|
||||
func namesNoInstallation(m catalogue.Manifest) error {
|
||||
named := catalogue.InstallationProblems(m)
|
||||
if len(named) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s names an installation, and a definition names none — declare a name meant "+
|
||||
"on purpose under %s with its reason, or take it out:\n - %s",
|
||||
m.Module, catalogue.NamesOnPurpose, strings.Join(named, "\n - "))
|
||||
}
|
||||
|
||||
// seatClaimed is one seat a module claims, as its runtime needs it: the name, the scope (a
|
||||
// node-scoped seat's verb carries the machine, design 33 §4) and the verbs the seat promises.
|
||||
type seatClaimed struct {
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves,omitempty"`
|
||||
}
|
||||
|
||||
// claimsFor joins a module's claims with the seats' protocols from the mesh's records.
|
||||
func claimsFor(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest) ([]seatClaimed, error) {
|
||||
if len(m.Claims) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
seats, err := inv.Seats(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byName := map[string]catalogue.Seat{}
|
||||
for _, s := range seats {
|
||||
byName[s.Name] = s
|
||||
}
|
||||
var out []seatClaimed
|
||||
for _, c := range m.Claims {
|
||||
claimed := seatClaimed{Seat: c.Name, Scope: c.At()}
|
||||
if s, known := byName[c.Name]; known {
|
||||
claimed.Scope = s.Scope
|
||||
// The verbs the runtime serves for the seat: the claim's own when it names them
|
||||
// (ADR 0160), else every verb the seat promises, which its tools then answer.
|
||||
claimed.Serves = c.ServesFor(catalogue.Manifest{Tools: catalogue.VerbNames(s.Serves)})
|
||||
}
|
||||
out = append(out, claimed)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaresTools is whether a module answers tools wherever it runs (novox/hq ADR 0197): it names
|
||||
// tools of its own, its build delivers a bundle the node's runtime serves, or it claims a seat
|
||||
// whose verbs it serves. A module with none is never expected to announce anything.
|
||||
func declaresTools(m catalogue.Manifest) bool {
|
||||
if len(m.Tools) > 0 {
|
||||
return true
|
||||
}
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if len(c.Serves) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// A module that declares none is refused before the account exists, so the bus never carries an
|
||||
// account nothing reads (novox/hq 04-ISSUES/078).
|
||||
func TestAModuleWithNoBrokerSecretCannotBeIssued(t *testing.T) {
|
||||
err := mayIssue(catalogue.Manifest{Module: "step-ca", OwnSecrets: map[string]string{"password": "/run/password"}})
|
||||
err := mayIssue(catalogue.Manifest{Module: "step-ca", OwnSecrets: catalogue.OwnSecrets{"password": {Path: "/run/password"}}})
|
||||
if err == nil {
|
||||
t.Fatal("a module with no broker own secret was issued an account")
|
||||
}
|
||||
@@ -20,7 +20,7 @@ func TestAModuleWithNoBrokerSecretCannotBeIssued(t *testing.T) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
if err := mayIssue(catalogue.Manifest{Module: "redis", OwnSecrets: map[string]string{"broker": "/run/broker"}}); err != nil {
|
||||
if err := mayIssue(catalogue.Manifest{Module: "redis", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}}); err != nil {
|
||||
t.Errorf("a module declaring its broker secret was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,25 +275,9 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// No registry trust is composed here any more: the container runtime's module states it, told
|
||||
// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
|
||||
g, err := overlay.From(nodes, cidr, "")
|
||||
if g != nil {
|
||||
// The artifact store, as this network reaches it. Found rather than configured: the
|
||||
// provider is whichever module offers it, on whichever machine holds that module — and if
|
||||
// nothing does yet (genesis raises the registry before the catalogue knows it), there is
|
||||
// no trust to write and nothing is written (novox/hq ADR 0082).
|
||||
//
|
||||
// Refused rather than composed without it when the question could not be answered: a
|
||||
// declaration missing the trust because a lookup failed is a machine that cannot pull,
|
||||
// delivered by a push that reported success — and nothing recomposes it until the next
|
||||
// push (the shape of novox/hq issues 042/048, reappearing as a race).
|
||||
at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
|
||||
if storeErr != nil {
|
||||
return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
|
||||
}
|
||||
if found {
|
||||
g.TrustRegistry(overlay.InternalName(at) + ":" + port)
|
||||
}
|
||||
}
|
||||
if err != nil && len(refused) > 0 {
|
||||
// The network is missing something, and some machines could not be resolved at all. Those
|
||||
// are almost always the same fact: a node that does not resolve contributes nothing, so
|
||||
@@ -346,9 +330,16 @@ func whoResolves(ctx context.Context, open *stores, requirement string) (
|
||||
refused := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
switch {
|
||||
case unresolvable(err):
|
||||
refused[n.Name] = err.Error()
|
||||
continue
|
||||
case err != nil:
|
||||
// Not a node that does not resolve — a question that went unanswered. Recording it as a
|
||||
// refusal would take the machine off the private network, and the generator that reads
|
||||
// this would then write a roster and a filter without it (novox/hq 04-ISSUES/152).
|
||||
return nil, nil, fmt.Errorf("whether %s answers %q cannot be read: %w",
|
||||
n.Name, requirement, err)
|
||||
}
|
||||
for _, m := range plan.Modules {
|
||||
for _, offered := range m.Offers() {
|
||||
@@ -560,6 +551,9 @@ func onTheNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
|
||||
catalogue.World{Unchecked: true, Holdings: holdings})
|
||||
if err != nil {
|
||||
// Said, not skipped in silence: a machine dropped here loses its address, and every
|
||||
// plan that names it fails in another module's words (novox/hq issue 188).
|
||||
fmt.Fprintf(os.Stderr, "%s is not counted as on the network: it does not resolve: %v\n", p.Name, err)
|
||||
continue
|
||||
}
|
||||
for _, m := range got.Modules {
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -253,11 +254,10 @@ func theResolver(t *testing.T) catalogue.Manifest {
|
||||
return m
|
||||
}
|
||||
|
||||
// The resolver is handed every machine on the private network as a wildcard, the same set and the
|
||||
// same source as the hosts file, and is handed it again when a machine leaves — through the
|
||||
// module's own manifest asking for the fact, with no module of the mesh's own in between (hal
|
||||
// dnsmasq-app conversion, novox/hq 08-connectivity). The runtime on that machine is pointed at the
|
||||
// machine's own address, where the resolver answers for its containers.
|
||||
// The resolver is handed every machine on the private network as a wildcard, and is handed it again
|
||||
// when a machine leaves — through the module's own manifest asking for the fact, with no module of the
|
||||
// mesh's own in between (hal dnsmasq-app conversion, novox/hq 08-connectivity). It is the mesh's one
|
||||
// resolver (ADR 0194), and the container runtime is given no resolver of its own (ADR 0196).
|
||||
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -265,6 +265,12 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
if _, err := assign(ctx, open, "anchor", "dnsmasq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its bus credential, as assigning issues it where the bus is reachable (novox/hq issue 203):
|
||||
// no bus is known to this test, so it is minted here, or composing refuses the placeholder.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "anchor.dnsmasq", Kind: inventory.BusModule, Node: "anchor", Module: "dnsmasq"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zones := func() string {
|
||||
t.Helper()
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
@@ -286,11 +292,12 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
|
||||
}
|
||||
}
|
||||
// The container runtime is given no resolver of its own (novox/hq ADR 0196): it copies its
|
||||
// machine's, which name the mesh's resolver first. A `dns` key would be a second account of where a
|
||||
// container asks, read only when the runtime starts.
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "dnsmasq.runtime-dns" {
|
||||
if !strings.Contains(r["content"].(string), `"10.77.0.1"`) || r["into"] != "json" {
|
||||
t.Errorf("the runtime is not pointed at this machine's own address, written into its file: %v", r)
|
||||
}
|
||||
t.Errorf("the resolver still writes the runtime's own dns: %v", r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,3 +310,28 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
|
||||
}
|
||||
}
|
||||
|
||||
// The roster is the machines and nothing else (novox/hq ADR 0191): each node's internal domain covers
|
||||
// every route on it, and a node's public domains are public DNS's. A routed name in `.Names` was a
|
||||
// private answer for a public name, handed by a resolver serving a LAN to a phone that could not use it.
|
||||
func TestTheRosterNamesOnlyTheMachines(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(with.Names, with.Machines) {
|
||||
t.Fatalf("%s's roster names more than the machines:\n names %v\n machines %v", node, with.Names, with.Machines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+135
-12
@@ -2,15 +2,16 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/token"
|
||||
)
|
||||
|
||||
@@ -45,6 +46,21 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** — the console's discovery reads it
|
||||
// (novox/hq ADR 0195), and a reader that parses a printed column breaks when it is reworded.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Name string `json:"name"`
|
||||
Heard string `json:"heard"`
|
||||
Mode string `json:"mode"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
out := make([]listed, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
out = append(out, listed{Name: n.Name, Heard: heardFrom(n), Mode: modeOf(n), ID: n.ID})
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Said rather than printed as nothing: an empty list and a failed read must never
|
||||
// look the same, and this command answering "none" is only honest because getting
|
||||
@@ -67,6 +83,18 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
// because the damage is already done by the time it prints.
|
||||
return publicDomain(ctx, inv, args[1:])
|
||||
|
||||
case "networks":
|
||||
// Removed by novox/hq ADR 0140, which superseded the record that added it. The filter no
|
||||
// longer names any network: it constrains what arrives from outside the machine and says
|
||||
// nothing about what did not, so there is no list to keep. Answered rather than met with
|
||||
// "unknown command", because this was the documented way to stop a flip cutting a machine's
|
||||
// containers off and somebody will reasonably still type it.
|
||||
return errors.New("`node networks` is gone (novox/hq ADR 0140). The filter constrains what " +
|
||||
"arrives from outside this machine and says nothing about traffic that did not, so no " +
|
||||
"network is named anywhere and nothing needs to be said to keep a machine's own " +
|
||||
"containers reaching outward. The machine reports which of its links face outside; see " +
|
||||
"`node show <name>`")
|
||||
|
||||
case "account":
|
||||
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
|
||||
@@ -258,15 +286,6 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
// chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can
|
||||
// exist before it does. The one-time secret IS the password, so a node's first connection is
|
||||
// already authenticated and enrolment is what happens over it.
|
||||
if management, err := broker.ManagementFromEnvironment(); err == nil {
|
||||
if err := management.CreateNodeAccount(ctx, issued.Node.Name, issued.Secret); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("broker account %s created, scoped to %s and the %s exchange\n\n",
|
||||
issued.Node.Name, link.QueueFor(issued.Node.Name), link.Exchange)
|
||||
} else if !errors.Is(err, broker.ErrNotConfigured) {
|
||||
return err
|
||||
}
|
||||
|
||||
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret,
|
||||
Adopted: issued.Node.Adopted}
|
||||
@@ -361,9 +380,19 @@ func identityCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func brokerCommand(args []string) error {
|
||||
func brokerCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "certificate" {
|
||||
return busCertificate(args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "accounts" {
|
||||
return busAccounts(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||
return consumerReset(ctx, args[1:])
|
||||
}
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("broker show")
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||
"broker consumer-reset <stream> <consumer>")
|
||||
}
|
||||
known, err := broker.FromEnvironment()
|
||||
if errors.Is(err, broker.ErrNotConfigured) {
|
||||
@@ -383,6 +412,45 @@ func brokerCommand(args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||
func consumerReset(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
args, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||
}
|
||||
if err := why.require("broker consumer-reset"); err != nil {
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "broker consumer-reset", args)
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
before, after, err := js.ResetConsumer(args[0], args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("consumer %s on %s re-made to deliver from now\n", args[1], args[0])
|
||||
fmt.Printf(" before: delivers %s, delivered to %d, acknowledged to %d, %d pending, %d unacknowledged\n",
|
||||
before.DeliverPolicy, before.Delivered, before.AckFloor, before.Pending, before.AckPending)
|
||||
fmt.Printf(" after: delivers %s, %d pending; what was pending is dropped. A holder bound to it may need its "+
|
||||
"process restarted to bind again\n", after.DeliverPolicy, after.Pending)
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
||||
//
|
||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
||||
@@ -428,6 +496,12 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
}
|
||||
fmt.Printf("%s\n", node.Name)
|
||||
fmt.Printf(" last heard from %s\n", heardFrom(node))
|
||||
// Which host runs it, as it reported (novox/hq 04-ISSUES/087). Said whenever known, because a
|
||||
// host refuses a declaration carrying a field it does not understand and refuses it WHOLE — so
|
||||
// which host a machine runs is what decides whether the mesh can send it anything new, and
|
||||
// nothing could say it. "not reported" rather than blank: a machine that has not said is a
|
||||
// different thing from one running nothing.
|
||||
fmt.Printf(" host %s\n", orNotReported(node.HostVersion))
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -443,6 +517,37 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
fmt.Printf(" public domain %s\n", domain)
|
||||
}
|
||||
|
||||
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
|
||||
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
|
||||
// is something not working now and they are a description. Unreadable is said, not passed over.
|
||||
open, err := openConditions(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
|
||||
}
|
||||
var here []conditions.Condition
|
||||
for _, c := range open {
|
||||
if concerns(c, name) {
|
||||
here = append(here, c)
|
||||
}
|
||||
}
|
||||
if len(here) > 0 {
|
||||
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
|
||||
for _, line := range conditionLines(here, time.Now()) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
// What it runs for its modules, and how each is (novox/hq ADR 0240): "is it working" answered without
|
||||
// a terminal on the machine.
|
||||
if h, had, err := inv.HealthOf(ctx, name); err != nil {
|
||||
fmt.Printf("\n what it says of what it runs could NOT be read: %v\n", err)
|
||||
} else {
|
||||
fmt.Println()
|
||||
for _, line := range healthLines(h, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -478,3 +583,21 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// orNotReported is a fact a machine states about itself, or the fact that it has not.
|
||||
func orNotReported(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "not reported — this machine has not said since the mesh began keeping it"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||
func printJSON(v any) error {
|
||||
body, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -189,13 +189,17 @@ func readPrivateKey(path string) (string, error) {
|
||||
func personIssue(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("operator issue", flag.ContinueOnError)
|
||||
invokes := set.String("invokes", "", "the tools this person may call, comma-separated, or * for every one")
|
||||
if err := set.Parse(args); err != nil {
|
||||
// Flags on either side of the name, because the usage this command prints puts them after it —
|
||||
// and the standard parser stops at the first thing that is not a flag, so the order the command
|
||||
// documents was the one order it refused (2026-09-28).
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if set.NArg() != 1 {
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("operator issue <name> --invokes <tool,tool|*>")
|
||||
}
|
||||
name := set.Arg(0)
|
||||
name := positionals[0]
|
||||
if *invokes == "" {
|
||||
return errors.New(
|
||||
"say what this person may call: --invokes mesh-catalog.catalog_tools,gitea.repo_create, " +
|
||||
|
||||
@@ -1,26 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func entry(module string, on ...string) inventory.Entry {
|
||||
b := &catalogue.Build{}
|
||||
for _, o := range on {
|
||||
b.On = append(b.On, catalogue.BuildsOn{Arg: "X", Module: o, Artifact: "runtime"})
|
||||
// entry is a module as the catalogue holds it: built, so its manifest carries no `build` any more.
|
||||
func entry(module string, _ ...string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: module}}
|
||||
}
|
||||
|
||||
// stoodOn is what each module's newest build recorded it was handed.
|
||||
func stoodOn(edges map[string][]string) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for module, bases := range edges {
|
||||
for _, b := range bases {
|
||||
out[module] = append(out[module], catalogue.ArtifactStoreScheme+b+"/runtime@sha256:"+strings.Repeat("0", 64))
|
||||
}
|
||||
}
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: module, Build: b}}
|
||||
return out
|
||||
}
|
||||
|
||||
// A module built before the module it stands on is built against the old one and reports success
|
||||
// (novox/hq 04-ISSUES/131). So bases come first, however the set arrived.
|
||||
func TestBasesAreBuiltBeforeWhatStandsOnThem(t *testing.T) {
|
||||
in := []inventory.Entry{entry("app", "runtime"), entry("runtime", "base"), entry("other"), entry("base")}
|
||||
got := orderByBases(in)
|
||||
in := []inventory.Entry{entry("app"), entry("runtime"), entry("other"), entry("base")}
|
||||
edges := stoodOn(map[string][]string{"app": {"runtime"}, "runtime": {"base"}})
|
||||
got := orderByBases(in, edges)
|
||||
pos := map[string]int{}
|
||||
for i, e := range got {
|
||||
pos[e.Manifest.Module] = i
|
||||
@@ -32,12 +45,31 @@ func TestBasesAreBuiltBeforeWhatStandsOnThem(t *testing.T) {
|
||||
t.Fatalf("an entry was lost or doubled: %d", len(got))
|
||||
}
|
||||
// A base outside the set is not waited for: it is not being rebuilt.
|
||||
got = orderByBases([]inventory.Entry{entry("app", "elsewhere")})
|
||||
got = orderByBases([]inventory.Entry{entry("app")}, stoodOn(map[string][]string{"app": {"elsewhere"}}))
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a dependency outside the set changed the set: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module registered from its manifest and never built still names its bases there; once built,
|
||||
// the recorded edge is what says so. Both are read, and a module never stands on itself.
|
||||
func TestWhatStandsOnAModuleIsReadFromItsBuildOrItsManifest(t *testing.T) {
|
||||
built := entry("gitea")
|
||||
edges := stoodOn(map[string][]string{"gitea": {"mesh-tools"}})
|
||||
if !standsOnModule(built, "mesh-tools", edges) {
|
||||
t.Fatal("a recorded edge was not read")
|
||||
}
|
||||
if standsOnModule(built, "gitea", edges) || standsOnModule(built, "postgres", edges) {
|
||||
t.Fatal("an edge was invented")
|
||||
}
|
||||
fresh := inventory.Entry{Manifest: catalogue.Manifest{Module: "plex", Build: &catalogue.Build{
|
||||
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
|
||||
}}}
|
||||
if !standsOnModule(fresh, "mesh-tools", nil) {
|
||||
t.Fatal("a manifest's own base was not read")
|
||||
}
|
||||
}
|
||||
|
||||
// A merge names a repository the way the forge does; a source is recorded the way a build was
|
||||
// asked for. The two meet on owner/repo and branch, whichever form the record took.
|
||||
func TestAMergeMatchesTheSourcesBuiltFromIt(t *testing.T) {
|
||||
@@ -61,3 +93,220 @@ func TestAMergeMatchesTheSourcesBuiltFromIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A merge made before the source was last seen is history: it does not move the source, and a
|
||||
// merge that says nothing about when it was made is taken as news.
|
||||
func TestAMergeOlderThanTheLastLookIsHistory(t *testing.T) {
|
||||
seen := time.Date(2026, 9, 28, 3, 0, 0, 0, time.UTC)
|
||||
if !isHistory("2026-09-28T02:00:00Z", seen) {
|
||||
t.Fatal("an older merge was taken as news")
|
||||
}
|
||||
if isHistory("2026-09-28T04:00:00Z", seen) {
|
||||
t.Fatal("a newer merge was taken as history")
|
||||
}
|
||||
if isHistory("", seen) || isHistory("2026-09-28T02:00:00Z", time.Time{}) {
|
||||
t.Fatal("a merge or a source with no time on it was refused")
|
||||
}
|
||||
}
|
||||
|
||||
// A module as the catalogue holds it: built from a repository, at a directory inside it.
|
||||
func fromRepo(module, repository, path string) inventory.Entry {
|
||||
return inventory.Entry{
|
||||
Manifest: catalogue.Manifest{Module: module},
|
||||
Source: inventory.Source{Repository: repository, Path: path, Ref: "main"},
|
||||
}
|
||||
}
|
||||
|
||||
// A merge rebuilds the modules whose own directories it changed, and everything when what it changed
|
||||
// is shared. One repository holding many modules is the ordinary case here, and rebuilding all of
|
||||
// them for a change to one is what exhausted a registry's pull limit the first night this ran.
|
||||
func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
gitea := fromRepo("gitea", repo, "modules/gitea")
|
||||
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
|
||||
known := []inventory.Entry{gitea, keycloak, fromRepo("plex", repo, "modules/plex")}
|
||||
candidates := []inventory.Entry{gitea, keycloak}
|
||||
merge := func(paths []string, truncated bool) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main",
|
||||
Paths: paths, PathsTruncated: truncated}
|
||||
}
|
||||
named := func(entries []inventory.Entry) string {
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
return strings.Join(names, ",")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
m link.SourceMoved
|
||||
want string
|
||||
}{
|
||||
{"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"},
|
||||
{"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"},
|
||||
{"a file at the root no build reads (issue 280)", merge([]string{"tsconfig.json"}, false), ""},
|
||||
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
|
||||
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
|
||||
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
|
||||
// novox/hq issue 252: a module the mesh has never registered is still a module, when the merge
|
||||
// shows it is one — and a directory that may be shared code is still shared.
|
||||
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
|
||||
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
|
||||
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
|
||||
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), ""},
|
||||
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""},
|
||||
{"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose recipe packages source from another repository is affected when that repository
|
||||
// moves — the manifest the mesh keeps says nothing about it, so the record of what the build read is
|
||||
// the only thing that can say so.
|
||||
func TestAModuleIsAffectedByTheRepositoryItPackages(t *testing.T) {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main",
|
||||
CloneURL: "http://forge.internal:20000/novox/mesh-controller.git"}
|
||||
for _, read := range [][]inventory.ReadRepository{
|
||||
{{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"}},
|
||||
{{Repository: "novox/mesh-controller"}},
|
||||
{{Repository: "https://elsewhere.example/novox/other"}, {Repository: "novox/mesh-controller.git", Ref: "main"}},
|
||||
} {
|
||||
if !readsFrom(read, m) {
|
||||
t.Errorf("%+v was not matched by the merge", read)
|
||||
}
|
||||
}
|
||||
for _, read := range [][]inventory.ReadRepository{
|
||||
nil,
|
||||
{{Repository: "novox/mesh-host", Ref: "main"}},
|
||||
{{Repository: "novox/mesh-controller", Ref: "release"}},
|
||||
} {
|
||||
if readsFrom(read, m) {
|
||||
t.Errorf("%+v was matched by a merge that is not its", read)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What a module handed over by hand records about where it came from, and what is refused.
|
||||
func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
|
||||
// The whole location: a repository on the mesh's own forge, the directory inside it, the branch
|
||||
// and the commit the manifest was read at.
|
||||
from, err := whereItComesFrom("novox/mesh-catalog", "main", "c0ffee", "modules/gitea", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if from.Path != "modules/gitea" || from.Seat != "git" || from.Repository != "novox/mesh-catalog" {
|
||||
t.Fatalf("the source records as %+v", from)
|
||||
}
|
||||
// A manifest with no provenance at all is legitimate: fixing something in a hurry.
|
||||
if from, err := whereItComesFrom("", "", "", "", false); err != nil || !reflect.DeepEqual(from, inventory.Source{}) {
|
||||
t.Fatalf("a manifest handed over with no provenance was refused: %+v, %v", from, err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
repository, ref, commit, path string
|
||||
self bool
|
||||
}{
|
||||
{what: "a source with no commit", repository: "novox/mesh-catalog", commit: ""},
|
||||
{what: "a commit with no source", commit: "c0ffee"},
|
||||
{what: "a directory inside nothing", path: "modules/gitea"},
|
||||
{what: "a forge holding nothing", self: true},
|
||||
{what: "an address given as a path on the forge", repository: "http://forge.internal:20000/novox/x.git", commit: "c0ffee", self: true},
|
||||
} {
|
||||
if _, err := whereItComesFrom(c.repository, c.ref, c.commit, c.path, c.self); err == nil {
|
||||
t.Errorf("%s was recorded as a source", c.what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a module once built at a commit still follows its branch — a merge into it
|
||||
// matches the module, and a plan re-asks the branch, not the old commit.
|
||||
func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main"}
|
||||
pinned := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a"}
|
||||
if !sourceIs(pinned, m) {
|
||||
t.Error("a module whose record names a commit is left out of a merge into its branch")
|
||||
}
|
||||
full := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a1d2c3b4a5f60718293a4b5c6d7e8f9012"}
|
||||
if !sourceIs(full, m) {
|
||||
t.Error("a full commit hash is read as a branch")
|
||||
}
|
||||
if got := followedBranch("9c97a8a"); got != "" {
|
||||
t.Errorf("a plan would re-ask the old commit %q", got)
|
||||
}
|
||||
if got := followedBranch("release"); got != "release" {
|
||||
t.Errorf("a branch is not followed as named: %q", got)
|
||||
}
|
||||
// A module that follows another branch is still not this merge's.
|
||||
if sourceIs(inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "release"}, m) {
|
||||
t.Error("a module following another branch was matched")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issues 278 and 280: a merge touching the code of a module the mesh does not hold — the
|
||||
// catalogue's reference module, whose manifest it left alone — read as shared and rebuilt every module
|
||||
// built from the repository (103 of them on 2026-10-06, 88 byte-identical); so did a merge-check.sh added at
|
||||
// the root. No build reads a file outside its module's directory, so neither rebuilds anything, said by the
|
||||
// announcer or not.
|
||||
func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
gitea := fromRepo("gitea", repo, "modules/gitea")
|
||||
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
|
||||
known := []inventory.Entry{gitea, keycloak}
|
||||
candidates := []inventory.Entry{gitea, keycloak}
|
||||
merge := func(paths, modules []string, said bool) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Paths: paths,
|
||||
ModuleDirs: modules, ModuleDirsSaid: said}
|
||||
}
|
||||
named := func(entries []inventory.Entry) string {
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
return strings.Join(names, ",")
|
||||
}
|
||||
showcase := []string{"modules/showcase/index.ts"}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
m link.SourceMoved
|
||||
want string
|
||||
}{
|
||||
{"a module held by none, said", merge(showcase, []string{"modules/showcase"}, true), ""},
|
||||
{"beside a held one's change", merge(append([]string{"modules/gitea/index.ts"}, showcase...),
|
||||
[]string{"modules/gitea", "modules/showcase"}, true), "gitea"},
|
||||
{"deeper inside it", merge([]string{"modules/showcase/daemon/index.ts"}, []string{"modules/showcase"}, true), ""},
|
||||
{"a directory holding no manifest, read by no build", merge([]string{"modules/lib/x.go"}, nil, true), ""},
|
||||
{"one of two files in no module", merge([]string{"modules/showcase/index.ts", "modules/lib/x.go"},
|
||||
[]string{"modules/showcase"}, true), ""},
|
||||
{"the root is never a module directory", merge([]string{"tsconfig.json"}, []string{"", "/", "."}, true), ""},
|
||||
{"not said: still no build reads it", merge(showcase, []string{"modules/showcase"}, false), ""},
|
||||
{"an old announcer saying nothing", merge(showcase, nil, false), ""},
|
||||
{"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the announcer says reaches the controller as it is sent: the two fields, by their names on the
|
||||
// wire, and an older announcement without them reads as not said.
|
||||
func TestTheAnnouncerSaysWhichDirectoriesAreModules(t *testing.T) {
|
||||
var m link.SourceMoved
|
||||
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc",`+
|
||||
`"paths":["modules/showcase/index.ts"],"module_dirs":["modules/showcase"],"module_dirs_said":true}`), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.ModuleDirsSaid || !reflect.DeepEqual(m.ModuleDirs, []string{"modules/showcase"}) {
|
||||
t.Fatalf("the announcer's word was lost: %+v", m)
|
||||
}
|
||||
var old link.SourceMoved
|
||||
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc","paths":["x"]}`), &old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if old.ModuleDirsSaid || old.ModuleDirs != nil {
|
||||
t.Fatalf("an older announcement says nothing about module directories: %+v", old)
|
||||
}
|
||||
}
|
||||
|
||||
+541
-144
@@ -7,16 +7,18 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// working out what one machine should be.
|
||||
@@ -25,7 +27,36 @@ import (
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// notResolvable marks the one failure in planFor that is a statement about the node: its assigned
|
||||
// modules do not compose. Every other failure means the mesh could not be *asked* — the store was
|
||||
// unreachable, a key could not be read — and says nothing about the node at all.
|
||||
//
|
||||
// The distinction exists because three callers gather something across every machine and must carry
|
||||
// on when one machine's set is broken. Each of them read a plain error as "their set does not
|
||||
// resolve", and so read a store that was briefly unreachable as a machine that runs nothing. On the
|
||||
// roster of routed names that is not a degraded answer but a false one: it states, to every machine
|
||||
// at once, that another machine's names do not exist. A control node spent hours replacing every
|
||||
// container it ran, on a six-minute cycle, because each pass restarted the store this is read from,
|
||||
// the read failed, one name left the roster, and the roster is part of every container's identity
|
||||
// (novox/hq 04-ISSUES/152, and 04-ISSUES/151 for why a changed roster is a changed container).
|
||||
//
|
||||
// So: skip a node that cannot resolve, and never a node that could not be read.
|
||||
type notResolvable struct{ err error }
|
||||
|
||||
func (n notResolvable) Error() string { return n.err.Error() }
|
||||
func (n notResolvable) Unwrap() error { return n.err }
|
||||
|
||||
// unresolvable reports whether err is a node's own set failing to compose, rather than the mesh
|
||||
// being unable to answer.
|
||||
func unresolvable(err error) bool {
|
||||
var n notResolvable
|
||||
return errors.As(err, &n)
|
||||
}
|
||||
|
||||
// planFor works out everything a node should run, from what was assigned to it.
|
||||
//
|
||||
// A failure to compose the node's own modules is wrapped as notResolvable; every other failure is
|
||||
// returned as it is. Callers gathering across the mesh must tell them apart — see notResolvable.
|
||||
func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
@@ -60,6 +91,12 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
if err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
// Where each of its consumers of a provision that keeps data was last sent (novox/hq ADR 0232):
|
||||
// a resolution that would answer one from anywhere else keeps it there, and says so.
|
||||
world.Bound, err = inv.BindingsFor(ctx, nodeName)
|
||||
if err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
@@ -95,8 +132,11 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
At: onNetwork[nodeName], PublicDomain: publicDomain,
|
||||
Account: who.Account, AccountHome: who.AccountHome}, world)
|
||||
if err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
// The node's own set does not compose. Marked, because this is the only failure here that
|
||||
// a mesh-wide gatherer may pass over — see notResolvable.
|
||||
return catalogue.Resolution{}, nil, notResolvable{err}
|
||||
}
|
||||
logUnheld(nodeName, resolved.Unheld)
|
||||
|
||||
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
|
||||
// password a provider is told to create is the one its consumer was given — and sealed to
|
||||
@@ -132,7 +172,14 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
}
|
||||
continue
|
||||
}
|
||||
secret, err := inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From, n.Local)
|
||||
var secret inventory.Secret
|
||||
var err error
|
||||
if n.SharedOwn != "" {
|
||||
// The provider's one credential, sealed to this consumer too (novox/hq ADR 0158).
|
||||
secret, err = inv.SharedSecretFor(ctx, n.Name, nodeName, n.For, n.From, providerModuleOf(resolved, open, ctx, n), n.Local, n.SharedOwn)
|
||||
} else {
|
||||
secret, err = inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From, n.Local)
|
||||
}
|
||||
if err != nil {
|
||||
// Said rather than skipped. A machine that resolves cleanly and receives no
|
||||
// credential is one that will fail to authenticate at some later, less obvious
|
||||
@@ -147,8 +194,12 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
// Settings for everything that resolved, including modules nobody assigned directly: a
|
||||
// requirement pulled in by something else is still configurable, and finding out that it is
|
||||
// not only when you try would be an arbitrary line nobody could predict.
|
||||
//
|
||||
// A setting that reaches nothing, or cannot compose with the definition it was stored for,
|
||||
// no longer refuses the machine here: it is judged where it is stored, and a definition that
|
||||
// moved under it costs that module its place in the declaration, said by name (novox/hq ADR
|
||||
// 0163, rule 6 — see Compose).
|
||||
settings := catalogue.SettingsBy{}
|
||||
var stray []string
|
||||
for _, m := range resolved.Modules {
|
||||
layers, err := inv.SettingsFor(ctx, nodeName, m.Module)
|
||||
if err != nil {
|
||||
@@ -158,13 +209,6 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
continue
|
||||
}
|
||||
settings[m.Module] = layers
|
||||
stray = append(stray, catalogue.UnusedSettings(m, layers)...)
|
||||
}
|
||||
if len(stray) > 0 {
|
||||
// Somebody set something that reaches no file. Said here rather than discovered by the
|
||||
// machine not behaving differently, which is the slowest way there is.
|
||||
return catalogue.Resolution{}, nil, fmt.Errorf(
|
||||
"these settings reach nothing:\n - %s", strings.Join(stray, "\n - "))
|
||||
}
|
||||
return resolved, settings, nil
|
||||
}
|
||||
@@ -238,8 +282,9 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
for _, o := range others {
|
||||
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true, Holdings: holdings})
|
||||
if err != nil {
|
||||
// Their set does not resolve for some other reason. Not this node's problem to
|
||||
// report, and nothing of theirs is running, so it offers nothing.
|
||||
// Said, not skipped: a machine dropped here offers nothing and holds nothing as far
|
||||
// as every other machine's plan can tell (novox/hq issue 188).
|
||||
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
|
||||
continue
|
||||
}
|
||||
firstHeld = append(firstHeld, got.Claims...)
|
||||
@@ -270,8 +315,22 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
world := catalogue.World{Offered: offered, Held: firstHeld, Holdings: holdings}
|
||||
var held []catalogue.Held
|
||||
for _, o := range others {
|
||||
got, err := catalogue.Resolve(shelf, o.assigned, o.node, world)
|
||||
// Each machine is resolved with its own pins, as its plan is: a machine that needs one to
|
||||
// settle two providers would otherwise be refused here and vanish from the mesh — every
|
||||
// seat it holds unheld, every build that needs one refused (2026-10-01, the control node;
|
||||
// novox/hq issue 188).
|
||||
theirs := world
|
||||
if pins, err := inv.PinsFor(ctx, o.node.Name); err == nil {
|
||||
theirs.Pinned = pins
|
||||
}
|
||||
got, err := catalogue.Resolve(shelf, o.assigned, o.node, theirs)
|
||||
if err != nil {
|
||||
// Said only for the whole-mesh view. With one machine excluded, the others are
|
||||
// resolved without its offers, and one that consumes them cannot resolve here by
|
||||
// design — that is not the machine being dropped, it is the view being partial.
|
||||
if exclude == "" {
|
||||
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
held = append(held, got.Claims...)
|
||||
@@ -320,13 +379,19 @@ func declarationFor(ctx context.Context, open *stores, node string,
|
||||
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
|
||||
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
|
||||
// machine "waiting" means — the read needs no number to be right about that.
|
||||
type Choosing bool
|
||||
type Choosing int
|
||||
|
||||
const (
|
||||
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
||||
Allocating Choosing = true
|
||||
// Reading is every question: what is assigned is used, and nothing is created.
|
||||
Reading Choosing = false
|
||||
Reading Choosing = iota
|
||||
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
||||
Allocating
|
||||
// Foreseeing is Reading, asked ahead of a send (the self-check's D1, novox/hq issue 275): nothing
|
||||
// is created, and an own secret the next send WOULD make is composed with a stand-in and named
|
||||
// (sendable.foreseen) rather than failing the composition — while one the send would be refused
|
||||
// (a bus credential nobody issued, a given secret under an old key) is refused here as it would
|
||||
// be there. Never sent: the stand-in is not a sealed value.
|
||||
Foreseeing
|
||||
)
|
||||
|
||||
func declarationWith(ctx context.Context, open *stores, node string,
|
||||
@@ -345,7 +410,132 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
return sendable{Resources: composed.Resources, Adoption: adoption}, nil
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
if !known {
|
||||
before = nil
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// boundToData is every binding of this machine's consumers to a provision that keeps their data
|
||||
// (novox/hq ADR 0232), as a send records it. Not a consumer left out of the declaration: the machine
|
||||
// is not told anything new about it, so nothing about where it is bound has been sent.
|
||||
func boundToData(plan catalogue.Resolution, leftOut map[string]string) []inventory.Binding {
|
||||
var out []inventory.Binding
|
||||
seen := map[[2]string]bool{}
|
||||
for _, n := range plan.Needs {
|
||||
if !n.KeepsData || n.ByRecord || n.Module == "" {
|
||||
continue
|
||||
}
|
||||
if _, left := leftOut[n.For]; left {
|
||||
continue
|
||||
}
|
||||
key := [2]string{n.For, n.Name}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, inventory.Binding{Machine: plan.Node, Consumer: n.For, Provision: n.Name,
|
||||
Provider: catalogue.Chosen{Node: n.From, Module: n.Module}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
|
||||
// issue 259): the module's current build for each module in it, and for a module left out of it
|
||||
// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
|
||||
// that is not known. Never nil, so a send through here always records what it knows.
|
||||
func carriedBuilds(modules []string, leftOut map[string]string, current map[string]inventory.CurrentBuild,
|
||||
before map[string]string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, m := range modules {
|
||||
if _, left := leftOut[m]; left {
|
||||
if was, kept := before[m]; kept {
|
||||
out[m] = was
|
||||
}
|
||||
continue
|
||||
}
|
||||
out[m] = current[m].Commit
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
|
||||
// for a reordering nobody made.
|
||||
func sortedKeysOf(m map[string]string) []string {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR
|
||||
// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out.
|
||||
func reportLeftOut(node string, declared sendable) {
|
||||
for _, m := range declared.LeftOut {
|
||||
fmt.Printf("%s: %s left out — a setting stored for it cannot compose with its definition; "+
|
||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||
node, m, declared.leftOutWhy[m])
|
||||
}
|
||||
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||
// 0225): the machine is sent everything else, and the consumer is named.
|
||||
for _, o := range declared.withheld {
|
||||
fmt.Printf("%s: %s\n", node, o)
|
||||
}
|
||||
// And whom it no longer serves because they are bound elsewhere (novox/hq issue 274).
|
||||
for _, u := range declared.unbound {
|
||||
fmt.Printf("%s: %s\n", node, u)
|
||||
}
|
||||
}
|
||||
|
||||
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
|
||||
//
|
||||
// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is
|
||||
// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted
|
||||
// by `module issue` and sealed by it; a push that made a random one would deliver a file the process
|
||||
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
|
||||
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
|
||||
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
|
||||
if name != "broker" {
|
||||
return nil
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return err
|
||||
} else if !minted {
|
||||
return fmt.Errorf(
|
||||
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
||||
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
||||
"`module issue %s --node %s`, then push again",
|
||||
module, node, user, module, node)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
@@ -353,7 +543,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||
inv := open.inventory
|
||||
grants, err := grantsFor(ctx, open, node)
|
||||
grants, withheld, unbound, err := grantsFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
@@ -365,7 +555,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// consumer is told are all derived from it.
|
||||
// What this machine was already given, for a composition that may not allocate.
|
||||
already := map[string]map[int]int{}
|
||||
if choosing == Reading {
|
||||
if choosing != Allocating {
|
||||
held, err := inv.PortsFor(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
@@ -385,7 +575,10 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
for _, m := range plan.Modules {
|
||||
g, err := catalogue.GivenPorts(m, settings[m.Module])
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
// A given port its definition no longer publishes: the module is left out of the
|
||||
// declaration, by name, when it is composed (novox/hq ADR 0163, rule 6) — never the
|
||||
// machine refused here for it.
|
||||
continue
|
||||
}
|
||||
if g != nil {
|
||||
given[m.Module] = g
|
||||
@@ -448,6 +641,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// And each module's own secrets — a superuser password, an administrator, an account. Made
|
||||
// per node, so a module running on three machines has three.
|
||||
needed := map[string]map[string]string{}
|
||||
foreseen := map[string]map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
for name := range m.OwnSecrets {
|
||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||
@@ -455,10 +649,29 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
var sealed string
|
||||
var err error
|
||||
if choosing == Allocating {
|
||||
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
|
||||
} else {
|
||||
var held bool
|
||||
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
|
||||
if err == nil && !held && choosing == Foreseeing {
|
||||
// Asked ahead of the send: what the send would do about it, by the send's own
|
||||
// rules. Made by it — a stand-in, named; refused by it — refused here, the same
|
||||
// words (novox/hq issue 275).
|
||||
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if foreseen[m.Module] == nil {
|
||||
foreseen[m.Module] = map[string]bool{}
|
||||
}
|
||||
foreseen[m.Module][name] = true
|
||||
continue
|
||||
}
|
||||
if err == nil && !held {
|
||||
// Never issued, so this machine cannot be running it. Left out rather than
|
||||
// invented: an empty string here would compose a declaration that differs
|
||||
@@ -550,43 +763,38 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
}
|
||||
|
||||
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
||||
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
|
||||
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
|
||||
// to serve and knows nothing about what they mean.
|
||||
// Kept apart from the machines, because a fact about the machines must not be handed the names
|
||||
// the mesh merely serves (novox/hq 04-ISSUES/111).
|
||||
// **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal
|
||||
// domain, `<node>.internal`, and every route on it is a name under that domain (ADR 0151), which
|
||||
// the resolver answers with one wildcard per machine — so no route needs a line of its own. A
|
||||
// node's public domains are the operator's and public DNS answers them; the mesh gives no private
|
||||
// answer for any of them. The roster once carried every routed name, public ones included, and a
|
||||
// resolver that also serves a LAN handed a phone a tunnel address for the mail server.
|
||||
// `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111).
|
||||
machines := make(map[string]string, len(names))
|
||||
for name, at := range names {
|
||||
machines[name] = at
|
||||
}
|
||||
routes, err := routeNamesInTheMesh(ctx, open)
|
||||
|
||||
// And every zone a module in the mesh answers itself (novox/hq ADR 0199), for the mesh's resolver
|
||||
// to forward.
|
||||
zones, err := zonesInTheMesh(ctx, open)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
for name, at := range routes {
|
||||
names[name] = at
|
||||
|
||||
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
|
||||
// lists every holder of the mesh's resolver.
|
||||
replicas, err := replicatedHolders(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// The ports the mesh itself needs open, which no module declares. Read from the broker this
|
||||
// control plane was told about rather than written down twice: the address a node is handed in
|
||||
// its token and the port its machine must accept on are the same fact.
|
||||
//
|
||||
// **Only on the node that listens on it** (novox/hq issue: the broker opening leaked onto
|
||||
// every node). The opening exists to WIDEN the broker's port to from-anywhere — a machine
|
||||
// enrolling is not on the mesh yet, so the broker's own `from: mesh` listen would refuse its
|
||||
// first dial. That widening belongs on the broker's host and nowhere else: a node that only
|
||||
// dials out needs no incoming rule, and an opening for a port nothing here listens on is a
|
||||
// from-anywhere hole for a dead port. So the foundation port is kept only when a module
|
||||
// resolved onto THIS node actually listens on it.
|
||||
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
||||
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
||||
// before it had an address on the private network. A machine joins through the tunnel now, and
|
||||
// every link to the bus crosses it, so its reach is what the `nats` module declares: the mesh.
|
||||
// Nothing the mesh itself needs is opened beyond what a module declares.
|
||||
var foundation []int
|
||||
if b, err := broker.FromEnvironment(); err == nil {
|
||||
if _, port, err := net.SplitHostPort(b.Address); err == nil {
|
||||
if n, err := strconv.Atoi(port); err == nil {
|
||||
foundation = foundationPortsFor(n, plan.Modules)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
|
||||
// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The
|
||||
@@ -640,36 +848,62 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// Which of this machine's links face outside, which is what the derived filter is written
|
||||
// around (novox/hq ADR 0140). Reported by the machine, never set.
|
||||
outwardLinks, err := inv.OutwardLinksOf(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
|
||||
// 0222): the artifact store as this network reaches it, which the container runtime is told to
|
||||
// trust (ADR 0082). The same address composed into every reference the mesh built.
|
||||
var reach map[string]string
|
||||
if artifactStore != "" {
|
||||
reach = map[string]string{"mesh-artifact-store": artifactStore}
|
||||
}
|
||||
readsHealth, err := engineReadsHealth(ctx, inv, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
ReadsHealth: readsHealth,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
BusUsers: busUsers,
|
||||
Machines: machines, Zones: zones, Holders: replicas,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||
BusUsers: busUsers, Withheld: withheld, Unbound: unbound,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
|
||||
// ADR 0066).
|
||||
// engineReadsHealth says whether a machine's node-engine reads a declared `health` (novox/hq ADR 0240
|
||||
// Phase B), by its own newest statement: one older, or one that never stated anything, is not sent the
|
||||
// field, because it parses strictly and would refuse the whole declaration for it.
|
||||
func engineReadsHealth(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.ReadinessContract, nil
|
||||
}
|
||||
|
||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||
// answering listen is published on there.
|
||||
//
|
||||
// **Mesh-wide, so any container resolves any routed name to its proxy** — including an internal
|
||||
// ACME validator, which cannot complete a challenge for a name it cannot reach. A routed name is
|
||||
// composed on the consumer's node (from its label and that node's public domain) and served by the
|
||||
// node answering the consumer's route requirement; this gathers both.
|
||||
//
|
||||
// It reads route names off resolutions rather than a table because there is no table: a route is a
|
||||
// contribution, computed from what each node runs. Name-agnostic — a contribution counts as a
|
||||
// routed name only because it carried a label the mesh composed, never because the mesh knows what
|
||||
// "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
|
||||
// the rest their names.
|
||||
func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
|
||||
// Read across every machine's resolution, as the roster once read routed names: a node whose set does
|
||||
// not compose declares nothing and is passed over, so one broken machine does not cost the rest their
|
||||
// zones; a store that cannot be read is raised, naming the machine, because returning the zones
|
||||
// without it would withdraw them from the resolver as if the operator had (novox/hq 04-ISSUES/152).
|
||||
// What the mesh refuses about the zones together — one declared twice, one shadowing the mesh's
|
||||
// suffix or a node's public domain — is refused here, by name.
|
||||
func zonesInTheMesh(ctx context.Context, open *stores) ([]catalogue.ZoneAt, error) {
|
||||
inv := open.inventory
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
|
||||
}
|
||||
address := map[string]string{}
|
||||
for _, p := range places {
|
||||
@@ -677,53 +911,47 @@ func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string,
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
|
||||
}
|
||||
|
||||
out := map[string]string{}
|
||||
var zones []catalogue.ZoneAt
|
||||
var public []string
|
||||
for _, n := range nodes {
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
switch {
|
||||
case unresolvable(err):
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("the zones %s answers cannot be read: %w", n.Name, err)
|
||||
}
|
||||
if plan.PublicDomain != "" {
|
||||
public = append(public, plan.PublicDomain)
|
||||
}
|
||||
for _, m := range plan.Modules {
|
||||
for to := range m.Contributes {
|
||||
values, asks, err := plan.ContributionsFrom(to, m.Module, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !asks {
|
||||
continue
|
||||
}
|
||||
// A routed name, and only that: a contribution the mesh composed a name for from a
|
||||
// label it was given. A grant that happens to carry a `name` of its own — a database
|
||||
// name — carries no label and is left alone.
|
||||
if _, labelled := values["label"]; !labelled {
|
||||
continue
|
||||
}
|
||||
name, _ := values["name"].(string)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
// The node that serves it: whoever answers this consumer's route requirement, or
|
||||
// this same node when the proxy is beside the consumer.
|
||||
serving := n.Name
|
||||
for _, need := range plan.Needs {
|
||||
if need.Name == to && need.For == m.Module {
|
||||
serving = need.From
|
||||
break
|
||||
}
|
||||
}
|
||||
if at := address[serving]; at != "" {
|
||||
out[strings.ToLower(name)] = at
|
||||
}
|
||||
if m.Zone == nil {
|
||||
continue
|
||||
}
|
||||
at := address[n.Name]
|
||||
if at == "" {
|
||||
// Not on the private network yet: nothing could reach its answerer.
|
||||
continue
|
||||
}
|
||||
published, layers, err := portsGivenOn(ctx, inv, n.Name, m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the zone %s declares on %s cannot be read: %w", m.Module, n.Name, err)
|
||||
}
|
||||
z, err := catalogue.ZoneOn(m, layers, published, n.Name, at)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zones = append(zones, *z)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
if problems := catalogue.ZonesProblems(zones, overlay.Suffix(), public); len(problems) > 0 {
|
||||
return nil, fmt.Errorf("the mesh's zones cannot be forwarded:\n - %s", strings.Join(problems, "\n - "))
|
||||
}
|
||||
return zones, nil
|
||||
}
|
||||
|
||||
// certificateFor is what the mesh certifies about one machine's internal name.
|
||||
@@ -793,39 +1021,62 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
|
||||
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
||||
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
||||
// the mesh hands over something it cannot itself use.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
||||
//
|
||||
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
|
||||
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
|
||||
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
|
||||
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
|
||||
//
|
||||
// **A provider is granted exactly the consumers whose own resolution binds them to it** (novox/hq
|
||||
// issue 274). The pair credentials on record say only whom this node was ever asked by: after a
|
||||
// consumer of a provision that keeps its data was moved and pinned back (issue 273, ADR 0232), the
|
||||
// credential from the provider it left was still on record, so that provider went on being asked for
|
||||
// five databases nobody used and never retired them. A credential whose consumer is bound elsewhere
|
||||
// is withdrawn here like one nobody asks for — the provider retires it and keeps its data (ADR 0230)
|
||||
// — and returned beside the grants, for plan and push to say. It stays on record: it is the key to
|
||||
// the login the provider keeps until `cleanup delete`, and to that data should a person pin it back.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) (
|
||||
[]catalogue.Grant, []catalogue.Overflow, []catalogue.Unbound, error) {
|
||||
inv := open.inventory
|
||||
issued, err := inv.SecretsFrom(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
||||
// the mesh names machines.
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
||||
// from a record beside it. A provider told to create a password and not what to create it for
|
||||
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
||||
out := make([]catalogue.Grant, 0, len(issued))
|
||||
var withheld []catalogue.Overflow
|
||||
var unbound []catalogue.Unbound
|
||||
for _, s := range issued {
|
||||
plan, settings, err := planFor(ctx, open, s.Consumer)
|
||||
if err != nil {
|
||||
switch {
|
||||
case unresolvable(err):
|
||||
// Their set does not resolve. Skipped rather than fatal: this node is not the place
|
||||
// to report another machine's problem, and a grant for something that is not going to
|
||||
// run would have the provider create a user nothing uses.
|
||||
continue
|
||||
case err != nil:
|
||||
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
||||
// nothing — and withholding a grant on that reading takes a consumer's access away
|
||||
// (novox/hq 04-ISSUES/152).
|
||||
return nil, nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
}
|
||||
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// A port in there is the consumer's software port until this. The consumer is on another
|
||||
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
||||
@@ -833,7 +1084,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
||||
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
||||
from := s.ConsumerModule
|
||||
@@ -843,10 +1094,20 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// working for ever after its consumer went away.
|
||||
from = ""
|
||||
}
|
||||
if bound := plan.BindsFrom(s.Name, s.ConsumerModule, s.Local); from != "" && !slices.Contains(bound, node) {
|
||||
// It still asks, and not of this node: its resolution — the same one read above, so an
|
||||
// unreadable one is the error above and never an empty answer here (issue 152) — binds
|
||||
// this credential to another provider, or under this local name to none. Withdrawn
|
||||
// exactly as a credential nobody asks for, and said.
|
||||
from = ""
|
||||
unbound = append(unbound, catalogue.Unbound{Provision: s.Name, Provider: node,
|
||||
Consumer: s.Consumer, Module: s.ConsumerModule, Local: s.Local, BoundTo: bound})
|
||||
}
|
||||
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
||||
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
||||
// remedy a short slug rather than a login a provider silently shortened.
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
|
||||
// this provision, as the consumer's resolution states it from the provider's offer (ADR
|
||||
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
|
||||
// provider silently shortened — and rather than this whole machine refused for it.
|
||||
slug := ""
|
||||
for _, mm := range plan.Modules {
|
||||
if mm.Module == s.ConsumerModule {
|
||||
@@ -855,15 +1116,32 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
}
|
||||
}
|
||||
if from != "" {
|
||||
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
||||
return nil, err
|
||||
bound := boundOfGrant(plan, s, node)
|
||||
source := catalogue.IdentitySource(slug, s.ConsumerModule)
|
||||
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
|
||||
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
|
||||
Consumer: s.Consumer, Module: s.ConsumerModule,
|
||||
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, catalogue.Grant{
|
||||
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
||||
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
||||
}
|
||||
return out, nil
|
||||
return out, withheld, unbound, nil
|
||||
}
|
||||
|
||||
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
|
||||
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
|
||||
// than to none: what the provider keeps of it is not known here.
|
||||
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
|
||||
for _, n := range consumer.Needs {
|
||||
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
|
||||
return n.Identity
|
||||
}
|
||||
}
|
||||
return catalogue.DefaultIdentityBound
|
||||
}
|
||||
|
||||
// listensLines is what a person is told about what this module would open, and why — the same
|
||||
@@ -932,6 +1210,25 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
||||
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
||||
// without --json allocates nothing.
|
||||
if record, err := open.inventory.NodeByName(ctx, args[0]); err == nil {
|
||||
left := plan.LeftOut(settings, record.Adopted)
|
||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||
}
|
||||
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||
for _, o := range plan.Overflowing() {
|
||||
fmt.Printf("%s: %s\n", args[0], o)
|
||||
}
|
||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||
// stored before its definition moved from under it.
|
||||
for _, m := range plan.Modules {
|
||||
for _, stray := range catalogue.UnusedSettings(m, settings[m.Module]) {
|
||||
fmt.Printf("%s: a setting reaches nothing — %s\n", args[0], stray)
|
||||
}
|
||||
}
|
||||
fmt.Printf("%s would run:\n", args[0])
|
||||
for _, m := range plan.Modules {
|
||||
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
|
||||
@@ -1238,6 +1535,20 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||
//
|
||||
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
|
||||
// resource of that module, so the question is whether it is here.
|
||||
list, missing, err := busUserList(ctx, inv, onThisNode)
|
||||
if len(missing) > 0 {
|
||||
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
return list, err
|
||||
}
|
||||
|
||||
// busUserList is composeBusUsers without saying anything: the list, and the users left out of it
|
||||
// for want of a credential. Asked on every send to decide whether the machine holding the bus must
|
||||
// go first (novox/hq issue 249), where saying the same missing users each time would bury them.
|
||||
func busUserList(ctx context.Context, inv *inventory.Inventory,
|
||||
onThisNode []catalogue.Manifest) (string, []string, error) {
|
||||
holdsTheBus := false
|
||||
for _, m := range onThisNode {
|
||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||
@@ -1245,52 +1556,138 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||
}
|
||||
}
|
||||
if !holdsTheBus {
|
||||
return "", nil
|
||||
return "", nil, nil
|
||||
}
|
||||
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
hashes := make(map[string]string, len(kept))
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
filled, missing := broker.WithPasswords(users, hashes)
|
||||
if len(missing) > 0 {
|
||||
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
if len(filled) == 0 {
|
||||
return "", fmt.Errorf(
|
||||
return "", missing, fmt.Errorf(
|
||||
"this machine runs the bus and not one user has a credential, so the composed list " +
|
||||
"would refuse every connection in the mesh")
|
||||
}
|
||||
return broker.ComposeAccounts(filled)
|
||||
list, err := broker.ComposeAccounts(filled)
|
||||
return list, missing, err
|
||||
}
|
||||
|
||||
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
|
||||
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
|
||||
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
|
||||
// not on the mesh yet and its first dial would be refused. That widening belongs on the broker's
|
||||
// host alone: a node that only dials out needs no incoming rule, and an opening for a port
|
||||
// nothing here listens on is a from-anywhere hole for a dead port.
|
||||
func foundationPortsFor(brokerPort int, modules []catalogue.Manifest) []int {
|
||||
for _, m := range modules {
|
||||
for _, l := range m.Listens {
|
||||
if l.Port == brokerPort {
|
||||
return []int{brokerPort}
|
||||
}
|
||||
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
||||
// own set when the provider is here, else the one the catalogue says offers it.
|
||||
func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string {
|
||||
for _, m := range resolved.Modules {
|
||||
if _, shared := m.SharedCredentialOf(n.Name); shared {
|
||||
return m.Module
|
||||
}
|
||||
}
|
||||
return nil
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for name, m := range shelf {
|
||||
if _, shared := m.SharedCredentialOf(n.Name); shared {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
|
||||
// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
||||
//
|
||||
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
||||
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
||||
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
||||
// burying the line about the node it acted on. A command says what concerns its own act instead
|
||||
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
||||
var (
|
||||
unheldLogged = map[string]string{}
|
||||
unheldLoggedMu sync.Mutex
|
||||
logUnheldChanges bool
|
||||
)
|
||||
|
||||
// logUnheld logs a node's unmet seat dependencies when they differ from what was last logged for
|
||||
// it, including when they become none — in the serving controller, and nowhere else.
|
||||
func logUnheld(node string, unheld []catalogue.Unheld) {
|
||||
if !logUnheldChanges {
|
||||
return
|
||||
}
|
||||
lines := make([]string, 0, len(unheld))
|
||||
for _, u := range unheld {
|
||||
lines = append(lines, u.String())
|
||||
}
|
||||
now := strings.Join(lines, "\n")
|
||||
unheldLoggedMu.Lock()
|
||||
before, seen := unheldLogged[node]
|
||||
unheldLogged[node] = now
|
||||
unheldLoggedMu.Unlock()
|
||||
if (seen && before == now) || (!seen && now == "") {
|
||||
return
|
||||
}
|
||||
if now == "" {
|
||||
fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
||||
node, len(lines), strings.Join(lines, "\n "))
|
||||
}
|
||||
|
||||
// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over
|
||||
// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not —
|
||||
// which includes every one of a module just assigned — and each now met that was not. Nothing about
|
||||
// any other node, and nothing that was already true before the act.
|
||||
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
|
||||
judge := func(names []string) map[string]catalogue.Unheld {
|
||||
var set []catalogue.Manifest
|
||||
for _, n := range names {
|
||||
if m, known := shelf[n]; known {
|
||||
set = append(set, m)
|
||||
}
|
||||
}
|
||||
out := map[string]catalogue.Unheld{}
|
||||
for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
|
||||
out[u.Module+" "+u.Seat] = u
|
||||
}
|
||||
return out
|
||||
}
|
||||
was, now := judge(before), judge(after)
|
||||
var lines []string
|
||||
for _, k := range sortedNames(now) {
|
||||
if _, already := was[k]; !already {
|
||||
lines = append(lines, "but "+now[k].String())
|
||||
}
|
||||
}
|
||||
for _, k := range sortedNames(was) {
|
||||
if _, still := now[k]; still {
|
||||
continue
|
||||
}
|
||||
u := was[k]
|
||||
if !slices.Contains(after, u.Module) {
|
||||
continue // went with its module, which says nothing about the seat
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("and %s on %s now has %s held", u.Module, node, u.Seat))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func sortedNames[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A plan follows what is done to the build queue (novox/hq ADR 0219).
|
||||
//
|
||||
// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
|
||||
//
|
||||
// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
|
||||
// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
|
||||
// sends a reader looking for a fault that is a decision. It says what it waits on instead.
|
||||
// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
|
||||
// failed plan's failed modules and the plan goes on from that tier as if they had built the
|
||||
// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
|
||||
// beside it — beside it, the plan would either ask it again or stay failed on an outcome the
|
||||
// rebuild has already replaced.
|
||||
|
||||
// pauseView is whether the build seat takes work: the holders that said they are paused, and
|
||||
// whether that is every holder.
|
||||
type pauseView struct {
|
||||
Nodes []string
|
||||
All bool
|
||||
}
|
||||
|
||||
// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
|
||||
// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
|
||||
func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
|
||||
if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
|
||||
return "", false
|
||||
}
|
||||
var asked *time.Time
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
|
||||
if asked == nil || s.AskedAt.Before(*asked) {
|
||||
asked = s.AskedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
if asked == nil {
|
||||
return "", false
|
||||
}
|
||||
waited := now.Sub(*asked)
|
||||
said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
|
||||
strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
|
||||
// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
|
||||
// longer than a day is named.
|
||||
if waited > pausedTooLong {
|
||||
said += " — PAUSED OVER A DAY: `resume` takes builds again"
|
||||
}
|
||||
return said, true
|
||||
}
|
||||
|
||||
// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
|
||||
const pausedTooLong = 24 * time.Hour
|
||||
|
||||
// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
|
||||
// where the seat being paused changes what a plan says.
|
||||
func awaitsABuild(plans []inventory.Plan) bool {
|
||||
for _, p := range plans {
|
||||
if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
|
||||
// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
|
||||
// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
|
||||
// reads as late, which is what it said before this existed.
|
||||
func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
|
||||
if !awaitsABuild(plans) {
|
||||
return pauseView{}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
seat := buildSeatAmong(entries)
|
||||
holders := holdersAmong(entries, seat)
|
||||
if len(holders) == 0 {
|
||||
return pauseView{}
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
}
|
||||
defer js.Close()
|
||||
said, err := link.PausedSaid(js, seat, holders)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
}
|
||||
return pauseOf(holders, said)
|
||||
}
|
||||
|
||||
// pauseOf is the view from what each holder said.
|
||||
func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
|
||||
var v pauseView
|
||||
for _, n := range holders {
|
||||
if said[n].Paused {
|
||||
v.Nodes = append(v.Nodes, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(v.Nodes)
|
||||
v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
|
||||
return v
|
||||
}
|
||||
|
||||
// failedIn is the modules of a plan's current tier that failed to build, sorted.
|
||||
func failedIn(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
|
||||
// that holds what this one held now (issue 254, ADR 0218).
|
||||
func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
return q, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
switch {
|
||||
case p.State == inventory.PlanDone:
|
||||
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||
case p.State == inventory.PlanSuperseded:
|
||||
return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
|
||||
case p.Open():
|
||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||
}
|
||||
if len(failedIn(p)) > 0 {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
stopped := stoppedRollouts(p)
|
||||
if len(stopped) == 0 {
|
||||
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
|
||||
p.Tier, p.ID, p.Note)
|
||||
}
|
||||
// **A build that failed its gate is not sent again** (novox/hq ADR 0236): it was put back on its first
|
||||
// machine, and retrying would judge the build the mesh put back, or send the failed one by hand.
|
||||
for _, m := range stopped {
|
||||
if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed && !noVerdictOnItsBuild(g) {
|
||||
return fmt.Errorf("%s failed its gate on %s (%s) and was put back: a build that failed its gate is not "+
|
||||
"sent again — a newer merge, or `rebuild %s`, makes a new build, judged at the gate again",
|
||||
m, strings.Join(g.Machines, ", "), g.Why, m)
|
||||
}
|
||||
}
|
||||
// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
|
||||
// sent — a newer build, and sending this one again would put the older build back on its machines.
|
||||
for _, m := range stopped {
|
||||
if q, found := newerPlanFor(m, p, plans); found {
|
||||
return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
|
||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
|
||||
// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
|
||||
func stoppedRollouts(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
|
||||
len(s.First) > 0 && s.Why != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
|
||||
func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
for _, tier := range q.Tiers {
|
||||
for _, m := range tier {
|
||||
if m == module {
|
||||
return q, true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
|
||||
// variable so a test of a retried rollout needs no machine.
|
||||
var sendRollout = sendToEach
|
||||
|
||||
// resumed sets a failed plan building again once nothing in its tier is failed.
|
||||
func resumed(p *inventory.Plan, why string) {
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = why
|
||||
}
|
||||
}
|
||||
|
||||
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||
// that tier: what follows is the plan going on as if they had built the first time.
|
||||
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans = append(plans, recent...)
|
||||
if err := retryRefusal(p, plans); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
|
||||
return retryTierWhole(ctx, open, &p, again)
|
||||
}
|
||||
if len(failedIn(p)) == 0 {
|
||||
return retryRollouts(ctx, open, &p)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
failed := failedIn(p)
|
||||
var asked []string
|
||||
for _, m := range failed {
|
||||
askModule(ctx, &p, m, byName)
|
||||
if s := p.Modules[m]; s.State == "asked" {
|
||||
asked = append(asked, m+" as "+s.Build)
|
||||
}
|
||||
}
|
||||
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.State != inventory.PlanBuilding {
|
||||
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
|
||||
p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
|
||||
}
|
||||
|
||||
// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
|
||||
// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
|
||||
func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
defer release()
|
||||
openPlans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 20)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
p, found := planHolding(module, openPlans, recent)
|
||||
if !found {
|
||||
return false, "", nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
was := p.State
|
||||
askModule(ctx, &p, module, byName)
|
||||
s := p.Modules[module]
|
||||
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if s.State != "asked" {
|
||||
return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
|
||||
}
|
||||
said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
|
||||
s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
|
||||
switch {
|
||||
case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
|
||||
said += "; the plan had failed and builds again from this tier"
|
||||
case was == inventory.PlanFailed:
|
||||
said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
|
||||
}
|
||||
return true, said, nil
|
||||
}
|
||||
|
||||
// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
|
||||
// yet built, else the newest failed plan whose current tier does, and that no open plan of its
|
||||
// repository supersedes.
|
||||
func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
|
||||
holds := func(p inventory.Plan) bool {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if m == module {
|
||||
s := p.Modules[m]
|
||||
return s == nil || s.State != "built"
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, p := range openPlans {
|
||||
if holds(p) {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
sorted := append([]inventory.Plan(nil), recent...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
|
||||
for _, p := range sorted {
|
||||
if p.State != inventory.PlanFailed || !holds(p) {
|
||||
continue
|
||||
}
|
||||
if _, superseded := newerOpenPlan(p, openPlans); superseded {
|
||||
continue
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
|
||||
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
|
||||
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
|
||||
func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
|
||||
// Every machine once, for all the modules stopped there (novox/hq issue 281).
|
||||
stopped := stoppedRollouts(*p)
|
||||
var machines []string
|
||||
for _, m := range stopped {
|
||||
for _, n := range p.Modules[m].First {
|
||||
if !slices.Contains(machines, n) {
|
||||
machines = append(machines, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(machines)
|
||||
sent, err := sendRollout(ctx, open, machines)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
|
||||
strings.Join(stopped, ", "), strings.Join(machines, ", "), p.ID, err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
var said []string
|
||||
for _, m := range stopped {
|
||||
s := p.Modules[m]
|
||||
var again []string
|
||||
for _, n := range sent {
|
||||
if slices.Contains(s.First, n) {
|
||||
again = append(again, n)
|
||||
}
|
||||
}
|
||||
s.First, s.FirstAt, s.Why = again, &now, ""
|
||||
said = append(said, m+" to "+strings.Join(again, ", "))
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
|
||||
"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
|
||||
}
|
||||
|
||||
// noVerdictOnItsBuild says a failed gate said nothing about the module's build (novox/hq issue 281): its
|
||||
// send changed nothing of the module there — the machine already ran that build, carried there by an
|
||||
// earlier send of the same tier, or one identical to it. Such a module was blamed for its machine.
|
||||
func noVerdictOnItsBuild(g *inventory.PlanGate) bool {
|
||||
return g.Rollback == gateUnchanged || (g.From != "" && sameCommit(g.From, g.To))
|
||||
}
|
||||
|
||||
// unjudgedAtGate is the modules of a plan's current tier stopped at a gate that was no verdict on their
|
||||
// build, sorted.
|
||||
func unjudgedAtGate(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.Gate != nil && s.Gate.Verdict == inventory.GateFailed && noVerdictOnItsBuild(s.Gate) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// retryTierWhole retries a plan stopped at a gate that judged no build of the module it stopped on
|
||||
// (issue 281): that module is asked again under a new id — its old build may be marked failed, and a new
|
||||
// verdict is what takes the gate's condition away — and every module of the tier sent first and never
|
||||
// passed is sent again, the tier whole, one send per machine, judged again. What passed stays passed.
|
||||
func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again []string) (string, error) {
|
||||
inv := open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
var resent []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
s := p.Modules[m]
|
||||
if s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(again, m) ||
|
||||
(s.Gate != nil && s.Gate.Verdict == inventory.GatePassed) {
|
||||
continue
|
||||
}
|
||||
sendAgain(s)
|
||||
resent = append(resent, m)
|
||||
}
|
||||
var asked []string
|
||||
for _, m := range again {
|
||||
sendAgain(p.Modules[m])
|
||||
askModule(ctx, p, m, byName)
|
||||
if s := p.Modules[m]; s.State == "asked" {
|
||||
asked = append(asked, m+" as "+s.Build)
|
||||
}
|
||||
}
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
|
||||
strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.State != inventory.PlanBuilding {
|
||||
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; %s sent again with the tier, one send per machine, "+
|
||||
"judged again", p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", "), orNone(strings.Join(resent, ", "))), nil
|
||||
}
|
||||
|
||||
// sendAgain forgets a module's first send, so its plan sends it again.
|
||||
func sendAgain(s *inventory.PlanModule) {
|
||||
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 213: for the moment a machine hands its controller over, the container and the
|
||||
// process both run the plan timer on one store. Only the one holding the plans moves them; the other
|
||||
// leaves them alone, and moves them once they are let go.
|
||||
func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now().UTC()
|
||||
// Every tier done: the next step is the plan's last, and needs nothing but the store.
|
||||
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
|
||||
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The other controller: its own connections to the same store, holding the plans.
|
||||
other, err := inventory.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(other.Close)
|
||||
release, err := other.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(release) // before the close above: a pool waits for a connection still held
|
||||
|
||||
advancePlans(ctx, open)
|
||||
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() {
|
||||
t.Fatalf("a controller moved a plan another held: %+v %v", p, err)
|
||||
}
|
||||
|
||||
release()
|
||||
advancePlans(ctx, open)
|
||||
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone {
|
||||
t.Fatalf("the plan did not move once it was let go: %+v %v", p, err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,192 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, to-be 48 §6, Phase C).
|
||||
//
|
||||
// With twelve consumers of the database provision and thirty-six of a route, one provider down would be
|
||||
// twelve conditions for one fault and twelve gates failed for something none of them did. So a consumer's
|
||||
// check names, in `needs`, the provision it exercises; while **that provision's provider for this
|
||||
// consumer** — the one the controller composed the consumer against: its recorded binding (ADR 0232), or
|
||||
// the provider its credential for the provision is from — is unhealthy on the record, what the check finds
|
||||
// is held under the provider's condition: listed there as waiting on it, raised as nothing of its own, and
|
||||
// the consumer's gate waits rather than fails. The provider's condition is urgent while consumers wait.
|
||||
//
|
||||
// **Only what the check finds is held.** A consumer that is down or restarting is its own, whatever its
|
||||
// provider does; so is anything a check that names no provision finds, and anything found while the
|
||||
// provider is healthy. A machine-level fault is never pinned on a module (issue 281), and this does not
|
||||
// change that.
|
||||
|
||||
// holding is what one reading of the record needs to say who waits on whom: every machine's newest
|
||||
// statement, the open conditions, and the catalogue — read once, asked many times.
|
||||
type holding struct {
|
||||
ctx context.Context
|
||||
inv *inventory.Inventory
|
||||
healths map[string]inventory.NodeHealth
|
||||
open []conditions.Condition
|
||||
shelf map[string]catalogue.Manifest
|
||||
// providers memoises providerFor by machine, consumer and provision.
|
||||
providers map[string]providerLookup
|
||||
}
|
||||
|
||||
type providerLookup struct {
|
||||
chosen catalogue.Chosen
|
||||
ok bool
|
||||
}
|
||||
|
||||
// readHolding reads what the hold is judged from.
|
||||
func readHolding(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||
healths, err := inv.Healths(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
||||
}
|
||||
|
||||
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
||||
// may be held. Down and restarting are liveness, the resource's own.
|
||||
func heldFinding(r inventory.ResourceHealth) bool {
|
||||
return r.State == link.StateUnhealthy && r.Check != "" && r.Needs != "" &&
|
||||
r.Reason != "down" && r.Reason != "restarting"
|
||||
}
|
||||
|
||||
// providerFor is the provider composed for a consumer's provision: its recorded binding, else the
|
||||
// machine its credential for the provision comes from and the module there that provides it.
|
||||
func (h *holding) providerFor(machine, consumer, provision string) (catalogue.Chosen, bool) {
|
||||
key := machine + "\x00" + consumer + "\x00" + provision
|
||||
if p, known := h.providers[key]; known {
|
||||
return p.chosen, p.ok
|
||||
}
|
||||
chosen, ok := h.lookUpProvider(machine, consumer, provision)
|
||||
h.providers[key] = providerLookup{chosen, ok}
|
||||
return chosen, ok
|
||||
}
|
||||
|
||||
func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue.Chosen, bool) {
|
||||
if bound, err := h.inv.BindingsFor(h.ctx, machine); err == nil {
|
||||
if c, ok := bound[consumer][provision]; ok && c.Node != "" && c.Module != "" {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
secrets, err := h.inv.SecretsOf(h.ctx, machine, consumer)
|
||||
if err != nil {
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
for _, s := range secrets {
|
||||
if s.Name != provision || s.Provider == "" {
|
||||
continue
|
||||
}
|
||||
if h.shelf == nil {
|
||||
if h.shelf, err = h.inv.Catalogue(h.ctx); err != nil {
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
}
|
||||
assigned, err := h.inv.Assigned(h.ctx, s.Provider)
|
||||
if err != nil {
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
for _, module := range assigned {
|
||||
for _, offer := range h.shelf[module].Offers() {
|
||||
if offer == provision {
|
||||
return catalogue.Chosen{Node: s.Provider, Module: module}, true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
|
||||
// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
|
||||
// statement says a resource of it is unhealthy.
|
||||
func (h *holding) unhealthy(p catalogue.Chosen) bool {
|
||||
key := moduleUnhealthyKey(p.Module, p.Node)
|
||||
for _, c := range h.open {
|
||||
if c.Key == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, r := range h.healths[p.Node].Resources {
|
||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
||||
// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
|
||||
// is the consumer's own.
|
||||
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||
var on catalogue.Chosen
|
||||
for _, r := range rs {
|
||||
if r.State != link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
if !heldFinding(r) {
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
p, ok := h.providerFor(machine, module, r.Needs)
|
||||
if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
on = p
|
||||
}
|
||||
return on, on.Module != ""
|
||||
}
|
||||
|
||||
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
||||
func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
||||
var out []string
|
||||
for machine, nh := range h.healths {
|
||||
byModule := map[string][]inventory.ResourceHealth{}
|
||||
for _, r := range nh.Resources {
|
||||
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||
byModule[r.Module] = append(byModule[r.Module], r)
|
||||
}
|
||||
}
|
||||
for module, rs := range byModule {
|
||||
if on, held := h.heldUnder(machine, module, rs); held && on == p {
|
||||
out = append(out, module+" on "+machine)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
||||
func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
||||
out := map[string]catalogue.Chosen{}
|
||||
byModule := map[string][]inventory.ResourceHealth{}
|
||||
for _, r := range h.healths[machine].Resources {
|
||||
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||
byModule[r.Module] = append(byModule[r.Module], r)
|
||||
}
|
||||
}
|
||||
for module, rs := range byModule {
|
||||
if on, held := h.heldUnder(machine, module, rs); held {
|
||||
out[module] = on
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moduleUnhealthyKey is a module's health condition's key on a machine.
|
||||
func moduleUnhealthyKey(module, machine string) string {
|
||||
return conditions.ScopeModule + "." + module + "." + machine + ".unhealthy"
|
||||
}
|
||||
|
||||
// waitingWords is the provider's evidence that consumers wait on it.
|
||||
func waitingWords(waiters []string) string {
|
||||
return fmt.Sprintf("waiting on it — %s", strings.Join(waiters, ", "))
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one
|
||||
// unhealthy database provider and three consumers failing their checks that need it — one condition, at the
|
||||
// provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is
|
||||
// healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised
|
||||
// on its own from the start.
|
||||
func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
k := conditionsFrom
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||
consumers := []string{"shop", "wiki", "crm"}
|
||||
for _, c := range consumers {
|
||||
register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}})
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "db"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"}
|
||||
for _, c := range consumers {
|
||||
if _, err := inv.Assign(ctx, machineOf[c], c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c,
|
||||
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
at := h0
|
||||
say := func(machine string, rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy,
|
||||
Reason: "its command: the database refuses connections", Check: "exec"}
|
||||
dbUp := dbDown
|
||||
dbUp.State, dbUp.Reason = link.StateHealthy, ""
|
||||
failing := func(module string) link.ResourceHealth {
|
||||
return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
}
|
||||
fine := func(module string) link.ResourceHealth {
|
||||
r := failing(module)
|
||||
r.State, r.Reason = link.StateHealthy, ""
|
||||
return r
|
||||
}
|
||||
openKeys := func() []conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// Two looks of the provider down and its consumers failing, in either order on each machine.
|
||||
for look := 0; look < 2; look++ {
|
||||
say("laptop", failing("shop"), failing("wiki"))
|
||||
say("anchor", dbDown, failing("crm"))
|
||||
}
|
||||
raised := openKeys()
|
||||
if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" {
|
||||
var keys []string
|
||||
for _, c := range raised {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys)
|
||||
}
|
||||
c := raised[0]
|
||||
if c.Severity != conditions.Urgent {
|
||||
t.Errorf("consumers wait on the provider and its condition is %s", c.Severity)
|
||||
}
|
||||
said := c.Evidence[0].Said
|
||||
for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} {
|
||||
if !strings.Contains(said, w) {
|
||||
t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said)
|
||||
}
|
||||
}
|
||||
|
||||
// Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too.
|
||||
f, err := gatherGateFacts(ctx, open, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.open, f.openErr, f.judged = raised, nil, false
|
||||
for _, m := range consumers {
|
||||
h, why := moduleHealthWord(m, machineOf[m], h0, f)
|
||||
if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") {
|
||||
t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why)
|
||||
}
|
||||
}
|
||||
// And a whole judging past the bound puts nothing back: it waits.
|
||||
long := h0.Add(-time.Hour)
|
||||
for _, m := range []string{"anchor", "laptop"} {
|
||||
f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true}
|
||||
}
|
||||
gatherWas := gatherGateFacts
|
||||
defer func() { gatherGateFacts = gatherWas }()
|
||||
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil }
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long}
|
||||
verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now())
|
||||
if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") {
|
||||
t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last)
|
||||
}
|
||||
|
||||
// The provider healthy again: a consumer still failing is now its own, at once (its streak stood).
|
||||
say("anchor", dbUp, fine("crm"))
|
||||
say("laptop", failing("shop"), fine("wiki"))
|
||||
var keys []string
|
||||
for _, c := range openKeys() {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) {
|
||||
t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer failing while its provider is healthy is raised on its own.
|
||||
func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
|
||||
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
|
||||
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"}
|
||||
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy,
|
||||
Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
for i := 1; i <= 2; i++ {
|
||||
at := h0.Add(time.Duration(i) * time.Minute)
|
||||
_ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at)
|
||||
if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
list, _ := conditionsFrom.Open(ctx)
|
||||
if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" {
|
||||
t.Fatalf("a consumer failing beside a healthy provider raised %v", list)
|
||||
}
|
||||
}
|
||||
+849
-173
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user