Compare commits
289
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18c58b4903 | ||
|
|
8c381172e7 | ||
|
|
d984816844 | ||
|
|
d70bb1f388 | ||
|
|
151faa10a1 | ||
|
|
a07dd78c3e | ||
|
|
3fdbea4204 | ||
|
|
af60d2719c | ||
|
|
de5292323f | ||
|
|
a47976be4e | ||
|
|
4446027108 | ||
|
|
e42d72a80a | ||
|
|
37fd73f378 | ||
|
|
7543f64911 | ||
|
|
5cf8938f55 | ||
|
|
a18c17d3cc | ||
|
|
5f24a47df6 | ||
|
|
435468c151 | ||
|
|
b5a441a978 | ||
|
|
0ab61d1be8 | ||
|
|
2347a4c73b | ||
|
|
ceffbf500a | ||
|
|
d55e148a1d | ||
|
|
ab74bbb56b | ||
|
|
320dde41d9 | ||
|
|
e603b2cae1 | ||
|
|
87780f482b | ||
|
|
867a3f81a9 | ||
|
|
9aeb5f4493 | ||
|
|
15a374659e | ||
|
|
868e3ae377 | ||
|
|
ce77a2caaa | ||
|
|
0d1bb035cb | ||
|
|
45b6c347b2 | ||
|
|
d03e24f423 | ||
|
|
f75cd0239d | ||
|
|
f7928a2bf7 | ||
|
|
20e58907b0 | ||
|
|
f877c33c7c | ||
|
|
0b7e740b29 | ||
|
|
755457cc37 | ||
|
|
4cc6b6c566 | ||
|
|
9e11790bd7 | ||
|
|
3ed08c3464 | ||
|
|
9f072c110c | ||
|
|
60cc9ec363 | ||
|
|
5d22d7df86 | ||
|
|
86057f4551 | ||
|
|
e4acab32d6 | ||
|
|
743e6478a7 | ||
|
|
ede2772b3c | ||
|
|
b34055f580 | ||
|
|
ee13e75bf3 | ||
|
|
480657d95b | ||
|
|
38c87a7e47 | ||
|
|
6acce60f8b | ||
|
|
4390589c61 | ||
|
|
e6a28a2d28 | ||
|
|
32bd8b5d5f | ||
|
|
c2ec0a601a | ||
|
|
34050dc6a3 | ||
|
|
a5500f732f | ||
|
|
3ec4473b29 | ||
|
|
df8c1ccf1f | ||
|
|
bbee20adf9 | ||
|
|
afed887217 | ||
|
|
622fb1e9cb | ||
|
|
5d5bdf9e32 | ||
|
|
4af9f1147d | ||
|
|
704b21b3c5 | ||
|
|
5b455b8a24 | ||
|
|
0e97413875 | ||
|
|
da8a5e9bc0 | ||
|
|
847ffd6d12 | ||
|
|
0214b45469 | ||
|
|
2bee290eea | ||
|
|
76304d9db8 | ||
|
|
b7084a19e9 | ||
|
|
c975dbf7b0 | ||
|
|
98e8568749 | ||
|
|
58889235d1 | ||
|
|
48b2906b5c | ||
|
|
ea7d91bff1 | ||
|
|
4a75cc51dd | ||
|
|
535c7641ac | ||
|
|
c707bf18d8 | ||
|
|
060f2a9ca5 | ||
|
|
1666747a22 | ||
|
|
04730b54a8 | ||
|
|
45870ab31c | ||
|
|
fd9308d328 | ||
|
|
3479a35462 | ||
|
|
cebc593627 | ||
|
|
2959c81435 | ||
|
|
8933864a49 | ||
|
|
df9e5728a9 | ||
|
|
2d18f438c5 | ||
|
|
3ce5a0239b | ||
|
|
70885ab3c4 | ||
|
|
65508d9df9 | ||
|
|
8123ca5654 | ||
|
|
d9b86376d3 | ||
|
|
ca3f43c400 | ||
|
|
b482122d66 | ||
|
|
2f5cc146fe | ||
|
|
c2ea9b4c51 | ||
|
|
b03d257f81 | ||
|
|
1601cb1743 | ||
|
|
873b19566a | ||
|
|
b0753e2aac | ||
|
|
a69c0e50dc | ||
|
|
72214ae989 | ||
|
|
5c606fd1d4 | ||
|
|
372faebf28 | ||
|
|
675466a8c3 | ||
|
|
cd49b50c40 | ||
|
|
41eefc2f8c | ||
|
|
c3946e5d72 | ||
|
|
524295926f | ||
|
|
bff5b7f0e4 | ||
|
|
cef22a2526 | ||
|
|
deef9421a2 | ||
|
|
3c19e9ee93 | ||
|
|
3d94f87bb0 | ||
|
|
4644f25edd | ||
|
|
c58e0a571e | ||
|
|
332fbaa9b1 | ||
|
|
7a0c219f96 | ||
|
|
985423ae61 | ||
|
|
cfacda2625 | ||
|
|
bf6beec465 | ||
|
|
9f3aa91e81 | ||
|
|
4bdcef3f73 | ||
|
|
8ddf8b7005 | ||
|
|
8bbb1b19ae | ||
|
|
e32ca00aed | ||
|
|
1219ec8229 | ||
|
|
e324f5d821 | ||
|
|
6f1a05366c | ||
|
|
0a1dc5f93f | ||
|
|
ae0b384497 | ||
|
|
f3875a6b60 | ||
|
|
f09c5ebdec | ||
|
|
6d2e4c299a | ||
|
|
b654b320a1 | ||
|
|
4cc8588afb | ||
|
|
584aa2d7d7 | ||
|
|
c9dd07f919 | ||
|
|
404b4409e7 | ||
|
|
6e6cd5e26d | ||
|
|
e658a14296 | ||
|
|
1826b68a22 | ||
|
|
05576db018 | ||
|
|
a688dcff39 | ||
|
|
8f1aec0832 | ||
|
|
95556b8def | ||
|
|
94c928b1f5 | ||
|
|
cd6757f1e3 | ||
|
|
ef3e5ae9b0 | ||
|
|
1d244d5aa2 | ||
|
|
bcdb1ad4f8 | ||
|
|
e0d69ab7a0 | ||
|
|
6eea92c323 | ||
|
|
b12c4b82e8 | ||
|
|
5740874ce3 | ||
|
|
bbf70fbda4 | ||
|
|
1c9bc9cf00 | ||
|
|
3275a16beb | ||
|
|
584ee99196 | ||
|
|
e44c726a38 | ||
|
|
a641ed7d15 | ||
|
|
2a13917d6e | ||
|
|
49102e1645 | ||
|
|
eb7ee921b5 | ||
|
|
b1baa8d8ba | ||
|
|
4473bc7d73 | ||
|
|
3bdab29a5a | ||
|
|
79a3396340 | ||
|
|
6ec4139871 | ||
|
|
454a3db69a | ||
|
|
7b2a9eae80 | ||
|
|
5795e03fad | ||
|
|
cbac703e81 | ||
|
|
18221c4079 | ||
|
|
b1546ed28a | ||
|
|
db0cb1e15e | ||
|
|
ee1d7e5cae | ||
|
|
43cab56452 | ||
|
|
56d6313262 | ||
|
|
7f3ff32361 | ||
|
|
21ac8fd858 | ||
|
|
2819915e8e | ||
|
|
7892019d9e | ||
|
|
e63af0df17 | ||
|
|
961f2e82ea | ||
|
|
e727589e48 | ||
|
|
2f157148c6 | ||
|
|
13be75eb65 | ||
|
|
13711fb5b9 | ||
|
|
fc573f951a | ||
|
|
c0c379af2b | ||
|
|
1ded7dff34 | ||
|
|
50c55a773e | ||
|
|
daf52b3aeb | ||
|
|
c6e234455f | ||
|
|
4dbd4066ca | ||
|
|
45a1783ba9 | ||
|
|
84deb530ee | ||
|
|
b3787e4d8e | ||
|
|
a681227af3 | ||
|
|
d8f36a8fb7 | ||
|
|
f0fd7fe886 | ||
|
|
41a9ff15dd | ||
|
|
ba685b17f5 | ||
|
|
d170491256 | ||
|
|
6bb1c9b768 | ||
|
|
f8daec01d3 | ||
|
|
f3c15e00af | ||
|
|
9038d4b0d3 | ||
|
|
f10aedac58 | ||
|
|
b538397055 | ||
|
|
dd10889209 | ||
|
|
875ebc093e | ||
|
|
1e28342ede | ||
|
|
286257d802 | ||
|
|
bc623bc1f9 | ||
|
|
67d54d9d09 | ||
|
|
ce9dc8e157 | ||
|
|
c672b3df6d | ||
|
|
25deb499bf | ||
|
|
83a9cf6e3a | ||
|
|
7b050abdcb | ||
|
|
7953d2889d | ||
|
|
c3b4f187e1 | ||
|
|
edd33f21d2 | ||
|
|
24347ea5a0 | ||
|
|
5ad36127ee | ||
|
|
a1a6f6493d | ||
|
|
e9f060e0be | ||
|
|
3cfc121670 | ||
|
|
a3bf5da9b1 | ||
|
|
d20b52cb19 | ||
|
|
7ca5a45ecc | ||
|
|
db7e4869ae | ||
|
|
539a1c16cd | ||
|
|
9eddf36503 | ||
|
|
05eccbf22b | ||
|
|
01fc9e1d73 | ||
|
|
d4de4543cc | ||
|
|
a371de37fe | ||
|
|
77acb40092 | ||
|
|
5facd8b08e | ||
|
|
72edb458cc | ||
|
|
10f76cafc4 | ||
|
|
2843834813 | ||
|
|
3e10ad1535 | ||
|
|
27b13e4586 | ||
|
|
142a5ad71d | ||
|
|
63a99d4323 | ||
|
|
0cd8bc0bfb | ||
|
|
7385d9c092 | ||
|
|
db9e9c5b87 | ||
|
|
4ead5a3df7 | ||
|
|
a23b190747 | ||
|
|
3be6e86f77 | ||
|
|
b0c4b849a7 | ||
|
|
b849af53dd | ||
|
|
74bd90df7e | ||
|
|
30599c1858 | ||
|
|
10d4712f82 | ||
|
|
e737e13bbb | ||
|
|
4b1dee6b79 | ||
|
|
4ab441ce16 | ||
|
|
f996cf7b6b | ||
|
|
bcfcfaaaf3 | ||
|
|
a3181e356b | ||
|
|
dd2fd28c92 | ||
|
|
9d165c7500 | ||
|
|
7630072e63 | ||
|
|
1e9dcf0667 | ||
|
|
cd6610cdf7 | ||
|
|
facc551ff2 | ||
|
|
71b3aeaf2c | ||
|
|
fafa3fd9ef | ||
|
|
6e5a1df575 | ||
|
|
61d00a234a | ||
|
|
4d203b5a5d | ||
|
|
6196a4815e | ||
|
|
0a0f5b4a5a |
@@ -6,3 +6,5 @@
|
||||
/mesh-controller
|
||||
/postgres-provisioner
|
||||
/redis-provisioner
|
||||
# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
|
||||
/cmd/mesh-controller/mesh-controller
|
||||
|
||||
+3
-1
@@ -27,9 +27,11 @@ WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
COPY vendor/ vendor/
|
||||
|
||||
# `COPY . .` brings `.git` along, so Go is told not to stamp the commit: the version is said by -X, and a
|
||||
# rebuild of an unchanged program is the same bytes (novox/hq ADR 0267 rule 6, issue 363).
|
||||
COPY . .
|
||||
ARG VERSION=development
|
||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath -buildvcs=false \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o /mesh-controller ./cmd/mesh-controller
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Release 57: config and provisioning
|
||||
|
||||
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
|
||||
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
|
||||
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
|
||||
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
|
||||
@@ -11,7 +11,7 @@ WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mesh-builder
|
||||
RUN CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags '-s -w' -o /mesh-builder ./cmd/mesh-builder
|
||||
|
||||
FROM ${ALPINE_BASE}
|
||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
||||
// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
|
||||
// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
|
||||
// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
|
||||
// it runs, instead of the build queue handing it out again from the start (issue 594).
|
||||
//
|
||||
// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
|
||||
// without them, and a restarted one removes them before it takes work again.
|
||||
//
|
||||
// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
|
||||
// when the build ends.
|
||||
// - drained: written by the ask, before it reads whether a build is in hand.
|
||||
//
|
||||
// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
|
||||
// or the agent sees the drain and hands the request back; never neither.
|
||||
|
||||
func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
|
||||
func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
|
||||
|
||||
// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
|
||||
// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
|
||||
// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
|
||||
// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
|
||||
// came, which is a fault to see.
|
||||
const (
|
||||
drainLapse = 2 * time.Minute
|
||||
drainTooLong = 15*time.Minute + 5*time.Minute
|
||||
)
|
||||
|
||||
// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
|
||||
func drainedAt() (began, asked time.Time, ok bool) {
|
||||
info, err := os.Stat(drainMark())
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false
|
||||
}
|
||||
asked = info.ModTime()
|
||||
began = asked
|
||||
if raw, err := os.ReadFile(drainMark()); err == nil {
|
||||
if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
|
||||
began = t
|
||||
}
|
||||
}
|
||||
return began, asked, true
|
||||
}
|
||||
|
||||
// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
|
||||
func drained() bool { return drainedSince(time.Now()) }
|
||||
|
||||
func drainedSince(now time.Time) bool {
|
||||
_, asked, ok := drainedAt()
|
||||
return ok && now.Sub(asked) < drainLapse
|
||||
}
|
||||
|
||||
// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
|
||||
// longer than its node-engine's bound and the slack.
|
||||
func drainedTooLong(now time.Time) error {
|
||||
began, asked, ok := drainedAt()
|
||||
if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
|
||||
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
|
||||
}
|
||||
|
||||
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
||||
// back to the queue.
|
||||
func claim() bool {
|
||||
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
||||
if closeErr := f.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
if drained() {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// holding says what is in hand, for the ask to print: written once the request is read.
|
||||
func holding(what string) {
|
||||
_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// released ends the build in hand.
|
||||
func released() {
|
||||
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// undrained clears both marks at the start of a holder: a container started again is asked afresh.
|
||||
func undrained() error {
|
||||
for _, mark := range []string{drainMark(), inHandMark()} {
|
||||
if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove %s: %w", mark, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
|
||||
func busy() (string, error) {
|
||||
// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
|
||||
// ask: its time is the last ask.
|
||||
began := time.Now().UTC()
|
||||
if was, _, ok := drainedAt(); ok && drained() {
|
||||
began = was.UTC()
|
||||
}
|
||||
if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
|
||||
}
|
||||
held, err := os.ReadFile(inHandMark())
|
||||
if err != nil {
|
||||
return "", errors.New("idle: no build in hand, and this agent takes no new one")
|
||||
}
|
||||
since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
|
||||
if what == "" {
|
||||
what = "a build"
|
||||
}
|
||||
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
|
||||
// agent takes no new build request (novox/hq ADR 0335 §5).
|
||||
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("an agent never asked is drained")
|
||||
}
|
||||
// A build is claimed and runs.
|
||||
if !claim() {
|
||||
t.Fatal("an agent never asked refused a build")
|
||||
}
|
||||
holding("novox/mesh-controller (check-280)")
|
||||
said, err := busy()
|
||||
if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
|
||||
t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
|
||||
}
|
||||
// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
|
||||
if !drained() {
|
||||
t.Fatal("the ask did not drain the agent")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a new build")
|
||||
}
|
||||
// The build in hand is still in hand after the refused claim, and busy goes on saying so.
|
||||
if _, err := busy(); err != nil {
|
||||
t.Fatalf("a refused claim ended the build in hand: %v", err)
|
||||
}
|
||||
// It ends: idle, and still drained.
|
||||
released()
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an agent with nothing in hand answered busy")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a build after its last one ended")
|
||||
}
|
||||
// Started again, it is asked afresh.
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() || !claim() {
|
||||
t.Fatal("a holder started again is still drained")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
|
||||
// neither, which would replace the container under a build just taken.
|
||||
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
for i := 0; i < 200; i++ {
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var claimed bool
|
||||
var askErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); claimed = claim() }()
|
||||
go func() { defer wg.Done(); _, askErr = busy() }()
|
||||
wg.Wait()
|
||||
if claimed && askErr != nil {
|
||||
t.Fatalf("round %d: the agent took a build and the ask said idle", i)
|
||||
}
|
||||
released()
|
||||
}
|
||||
}
|
||||
|
||||
// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
|
||||
// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
|
||||
func TestADrainNoLongerAskedLapses(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
if !drained() || claim() {
|
||||
t.Fatal("an agent just asked takes work")
|
||||
}
|
||||
// The last ask was three minutes ago.
|
||||
old := time.Now().Add(-3 * time.Minute)
|
||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("a drain no longer asked for three minutes still holds")
|
||||
}
|
||||
if !claim() {
|
||||
t.Fatal("an agent whose drain lapsed takes no work")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
|
||||
// replacement it was drained for never came. One asked again after a lapse starts a new drain.
|
||||
func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
now := time.Now()
|
||||
if err := drainedTooLong(now); err != nil {
|
||||
t.Fatalf("an agent drained a moment ago is not ready: %v", err)
|
||||
}
|
||||
// The drain began 25 minutes ago, and the node-engine asked a moment ago.
|
||||
began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
|
||||
if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := drainedTooLong(time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "has not come") {
|
||||
t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
|
||||
}
|
||||
// Each ask keeps when the drain began.
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
if err := drainedTooLong(time.Now()); err == nil {
|
||||
t.Fatal("an ask started the drain again")
|
||||
}
|
||||
// The asks stopped: the drain lapsed, and so did the fault.
|
||||
old := time.Now().Add(-3 * time.Minute)
|
||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := drainedTooLong(time.Now()); err != nil {
|
||||
t.Fatalf("a lapsed drain still reads not ready: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
|
||||
// node-engine's whole bound.
|
||||
func TestAnUnknownCommandFails(t *testing.T) {
|
||||
was := os.Args
|
||||
t.Cleanup(func() { os.Args = was })
|
||||
os.Args = []string{"mesh-builder", "some-later-word"}
|
||||
if err := run(); err == nil {
|
||||
t.Fatal("an unknown command exited 0")
|
||||
}
|
||||
os.Args = []string{"mesh-builder", "help"}
|
||||
if err := run(); err != nil {
|
||||
t.Fatalf("help failed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,9 @@ type holder struct {
|
||||
say func(link.HolderState) error
|
||||
// remove runs what a kill needs outside the build: the containers left behind.
|
||||
remove builder.Runner
|
||||
// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
|
||||
// this machine publishes to no artifact store.
|
||||
scratch builder.Scratch
|
||||
|
||||
mu sync.Mutex
|
||||
paused bool
|
||||
@@ -131,6 +134,34 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
|
||||
// after. Every holder sweeps the one store; what one let go of, another finds gone.
|
||||
func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
|
||||
sweep := func() {
|
||||
if h.scratch == nil {
|
||||
return
|
||||
}
|
||||
n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
|
||||
case n > 0:
|
||||
fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
|
||||
}
|
||||
}
|
||||
sweep()
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
sweep()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// begin records the build this machine took, with the cancel that ends it.
|
||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||
|
||||
+350
-63
@@ -18,12 +18,15 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -54,6 +57,8 @@ is dialled except the broker.
|
||||
MESH_BINDING a file the mesh wrote saying where the artifact store is
|
||||
MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is
|
||||
MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it
|
||||
MESH_GENESIS_PACKAGE_REGISTRY the document the installer sealed at genesis, read while no
|
||||
binding is present (novox/hq ADR 0336)
|
||||
MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap)
|
||||
MESH_NPM_TOKEN the token for it, likewise
|
||||
MESH_NPM_SCOPE the scope it answers for (default: @novox)
|
||||
@@ -66,6 +71,17 @@ broker to take work from or a registry to publish into:
|
||||
|
||||
Without --registry the artifacts stay in this machine's container runtime, named by the
|
||||
digest of their own configuration. The result is printed as JSON.
|
||||
|
||||
mesh-builder ready
|
||||
|
||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||
1 before: its container's health check.
|
||||
|
||||
mesh-builder busy
|
||||
|
||||
exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
|
||||
from the first ask the agent takes no new build: its container's busy command, asked before the
|
||||
container is replaced (novox/hq ADR 0335).
|
||||
`
|
||||
|
||||
func run() error {
|
||||
@@ -76,12 +92,42 @@ func run() error {
|
||||
return nil
|
||||
case "build":
|
||||
return buildOnce(context.Background(), os.Args[2:])
|
||||
default:
|
||||
case "ready":
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
|
||||
// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
|
||||
if err := ready(); err != nil {
|
||||
return err
|
||||
}
|
||||
return drainedTooLong(time.Now())
|
||||
case "busy":
|
||||
// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
|
||||
// nothing is in hand.
|
||||
said, err := busy()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
case "help", "-h", "--help":
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
default:
|
||||
// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
|
||||
// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
|
||||
fmt.Fprint(os.Stderr, usage)
|
||||
return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
|
||||
}
|
||||
}
|
||||
|
||||
if err := notReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
|
||||
// differently never takes a build another one made.
|
||||
builder.Version = builder.SelfVersion(version)
|
||||
if err := undrained(); err != nil {
|
||||
return err
|
||||
}
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -134,7 +180,13 @@ func run() error {
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
||||
// in hand and then says idle, and its container is replaced between builds.
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
||||
Paused: func() bool { return h.Paused() || drained() },
|
||||
Taking: func() { taking(seat) },
|
||||
Claim: claim,
|
||||
})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
@@ -158,8 +210,13 @@ func run() error {
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
// Check builds are let go of after a week (novox/hq ADR 0350 §5).
|
||||
h.scratch = publisher
|
||||
go h.expireChecks(ctx, time.Hour, time.Now)
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
defer released()
|
||||
holding(work.Request().Repository + " (" + work.Request().ID + ")")
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
@@ -216,7 +273,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
||||
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
||||
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
||||
// One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq
|
||||
// issue 716).
|
||||
var saying sync.Mutex
|
||||
say := func(step, message string) {
|
||||
saying.Lock()
|
||||
defer saying.Unlock()
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
@@ -242,7 +304,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
say("build", what)
|
||||
|
||||
npmrc, err := packagesFrom()
|
||||
npmrc, err := packagesFrom(on)
|
||||
if err == nil {
|
||||
say("packages", saysRegistry(npmrc))
|
||||
}
|
||||
registryHere, isRegistry := publisher.(builder.Registry)
|
||||
var built builder.Result
|
||||
if request.Check != nil {
|
||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
||||
@@ -252,11 +318,38 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if r, ok := publisher.(builder.Registry); ok {
|
||||
registry = r.Address
|
||||
}
|
||||
spec := checkSpecOf(request)
|
||||
// The judge cache the build agent declares as its own data, never in the workspace: only judge builds and
|
||||
// mesh-lab's replays mount it, never a check's containers, so no pull request's script can write what a judge
|
||||
// is built with or kept as, or what the replays run (novox/hq ADR 0358 §6, issue 709).
|
||||
spec.JudgeCache = judgeCache()
|
||||
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
|
||||
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
|
||||
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
|
||||
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
|
||||
if isRegistry && len(spec.Builds) > 0 {
|
||||
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
|
||||
Running: request.Check.Running, Seats: request.Seats, Named: namedOf(request.Check.Named)}
|
||||
}
|
||||
var v builder.CheckVerdict
|
||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
||||
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
|
||||
if err == nil {
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built),
|
||||
Release: v.Release, JudgedBy: v.JudgedBy, BesideAt: v.BesideAt, Validator: v.Validator,
|
||||
Siblings: siblingsOut(v.Siblings), SiblingsUnknown: v.SiblingsUnknown, JudgeFrom: v.JudgeFrom}
|
||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
||||
for _, m := range v.Members {
|
||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||
Commit: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
|
||||
if request.Check.Try {
|
||||
if line, err := json.Marshal(triedOf(v)); err == nil {
|
||||
say(link.StepTried, string(line))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
@@ -264,9 +357,22 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// after a clone that then fails at npm ci.
|
||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||
// docker client (ADR 0219).
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
if request.Reuse != nil && isRegistry && !request.DryRun {
|
||||
// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
|
||||
reuse := builder.Reuse{Scratch: registryHere}
|
||||
for _, p := range request.Reuse.Pulls {
|
||||
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
|
||||
}
|
||||
// Only what a verdict names is taken (ADR 0350 §2).
|
||||
reuse.Named = namedOf(request.Reuse.Builds)
|
||||
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
} else {
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
}
|
||||
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
||||
result.Mirrored = built.Mirrored
|
||||
@@ -302,6 +408,19 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
result.Against = built.Against
|
||||
result.SourceFingerprint = built.Source
|
||||
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
||||
result.ReuseKey = built.ReuseKey
|
||||
if built.Reused != nil {
|
||||
f := built.Reused.From
|
||||
result.Reused = built.Reused.Words()
|
||||
result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
|
||||
Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
|
||||
} else if request.Reuse != nil {
|
||||
result.BuiltWhy = built.Why
|
||||
if result.BuiltWhy == "" {
|
||||
result.BuiltWhy = "it has no artifact to build"
|
||||
}
|
||||
}
|
||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
||||
for _, r := range built.Read {
|
||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
@@ -309,6 +428,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
for _, s := range built.Sources {
|
||||
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
||||
}
|
||||
// The list of predefined keys the manifest was judged against, for the controller to publish
|
||||
// (novox/hq ADR 0348 §7).
|
||||
if p := built.Predefined; p != nil {
|
||||
result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation}
|
||||
}
|
||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||
}
|
||||
}
|
||||
@@ -348,17 +472,88 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly, Release: c.Release, SiblingsUnknown: c.SiblingsUnknown}
|
||||
for _, s := range c.Siblings {
|
||||
spec.Siblings = append(spec.Siblings, builder.Sibling{Repository: s.Repository, Number: s.Number, Head: s.Head,
|
||||
Base: s.Base, Dir: s.Dir})
|
||||
}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref, Else: b.Else}
|
||||
}
|
||||
for _, b := range c.Builds {
|
||||
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
|
||||
Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
|
||||
}
|
||||
for _, m := range c.Members {
|
||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||
Ref: m.Ref, Paths: m.Paths})
|
||||
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
// siblingsOut is the siblings a check read as the outcome carries them (novox/hq ADR 0382 §4).
|
||||
func siblingsOut(read []builder.JudgedSibling) []link.CheckSibling {
|
||||
var out []link.CheckSibling
|
||||
for _, s := range read {
|
||||
out = append(out, link.CheckSibling{Repository: s.Repository, Number: s.Number, Head: s.Head, Base: s.Base,
|
||||
BaseCommit: s.BaseCommit})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeCache is the judge cache the build agent's manifest declares (MESH_JUDGE_CACHE, novox/hq ADR 0358 §6): a
|
||||
// directory mounted at the same path on the host, so a judge build's mounts name what the agent sees. Empty when the
|
||||
// agent declares none, and every judge is then built cold under its check's root.
|
||||
func judgeCache() string { return os.Getenv("MESH_JUDGE_CACHE") }
|
||||
|
||||
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
|
||||
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
|
||||
func triedOf(v builder.CheckVerdict) link.Tried {
|
||||
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
|
||||
Head: v.Head, Trunk: v.Trunk}
|
||||
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
|
||||
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
|
||||
}
|
||||
if v.Repo != nil && v.Repo.Failed != "" {
|
||||
t.Failed = strings.Split(v.Repo.Failed, "\n")
|
||||
}
|
||||
if t.Verdict == "" {
|
||||
t.Verdict = "error"
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// namedOf are the builds the seat's verdicts name, as the builder believes a scratch record by them (novox/hq ADR
|
||||
// 0350 §2, issue 703): each module's key with every artifact's digest.
|
||||
func namedOf(builds []link.CheckedBuild) []builder.NamedBuild {
|
||||
var out []builder.NamedBuild
|
||||
for _, b := range builds {
|
||||
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
|
||||
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
|
||||
Digests: map[string]string{}}
|
||||
for _, m := range b.Built.Made {
|
||||
named.Digests[m.Name] = builder.DigestOf(m.Reference)
|
||||
}
|
||||
out = append(out, named)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
||||
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
||||
var out []link.CheckBuilt
|
||||
for _, b := range built {
|
||||
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(),
|
||||
Kept: b.Kept, Same: b.Same, SameAs: b.SameAs}
|
||||
for _, m := range b.Made {
|
||||
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// layerOf is one layer of a check as the outcome carries it.
|
||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||
if l == nil {
|
||||
@@ -368,79 +563,171 @@ func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||
}
|
||||
|
||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||
// (novox/hq ADR 0076, issue 053).
|
||||
// (novox/hq ADR 0076, issue 053) — on this machine.
|
||||
//
|
||||
// Preferably from the mesh: a package-registry binding names the endpoint the way the artifact
|
||||
// store's binding does, and a sealed token file the credential the way the broker's does. The
|
||||
// environment variables remain for a builder run by a person, and for the bootstrap, where there is
|
||||
// no registry yet — there the result is disabled and a build that needs no mesh-published dependency
|
||||
// builds anyway.
|
||||
func packagesFrom() (builder.Npmrc, error) {
|
||||
// **The binding wins, then the genesis document** (novox/hq ADR 0336 §3, §4). The build agent's
|
||||
// npm-package-registry requirement is optional: on genesis's node, before the gitea module is assigned,
|
||||
// nothing provides it, and the installer sealed the forge it raised to this agent as one document
|
||||
// instead. Once the requirement binds, the binding names the endpoint the way the artifact store's
|
||||
// does and a sealed token file the credential, and the genesis document is left unread. A binding
|
||||
// file named and absent is a requirement not bound here, never a fault: the environment names the
|
||||
// path whether or not the mesh wrote it.
|
||||
//
|
||||
// The environment variables remain for a builder run by a person. With none of the three the result
|
||||
// is disabled and says this machine, so a build that needs the registry is refused naming it (§6),
|
||||
// and a build that needs none builds anyway.
|
||||
func packagesFrom(machine string) (builder.Npmrc, error) {
|
||||
scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE"))
|
||||
if scope == "" {
|
||||
scope = "@novox"
|
||||
}
|
||||
none := builder.Npmrc{Machine: machine}
|
||||
|
||||
registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY"))
|
||||
var username string
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return builder.Npmrc{}, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err)
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
// Not bound on this machine: the genesis document, or nothing.
|
||||
case err != nil:
|
||||
return none, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err)
|
||||
default:
|
||||
return fromBinding(path, raw, scope, machine)
|
||||
}
|
||||
var told struct {
|
||||
From string `json:"from"`
|
||||
At string `json:"at"`
|
||||
As string `json:"as"`
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_GENESIS_PACKAGE_REGISTRY")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
// Not this mesh's genesis node, or the installer has not sealed it yet.
|
||||
case err != nil:
|
||||
return none, fmt.Errorf("cannot read the genesis document for the package registry: %w", err)
|
||||
default:
|
||||
return fromGenesis(path, raw, scope, machine)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &told); err != nil {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s is not a binding: %w", path, err)
|
||||
}
|
||||
if told.At == "" {
|
||||
return builder.Npmrc{}, fmt.Errorf(
|
||||
"%s says the package registry is on %q and gives no address for it", path, told.From)
|
||||
}
|
||||
// Composed from what the provider serves, so nothing here knows gitea's URL shape from
|
||||
// another registry's: it states its port, the path its registry answers on, and the scheme.
|
||||
scheme := "https"
|
||||
if s, ok := told.Serves["scheme"]; ok {
|
||||
scheme = fmt.Sprintf("%v", s)
|
||||
}
|
||||
port, ok := told.Serves["port"]
|
||||
if !ok {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s says nothing about which port the package registry answers on", path)
|
||||
}
|
||||
npmPath, ok := told.Serves["npm-path"]
|
||||
if !ok {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s says nothing about the path the package registry answers on", path)
|
||||
}
|
||||
registry = fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath)
|
||||
username = told.As
|
||||
}
|
||||
|
||||
// The credential the mesh sealed to this machine. The mesh authenticates the ordinary way — a
|
||||
// generated password the provider only applies (novox/hq ADR 0048) — so with a username this is
|
||||
// a password (basic auth); without one it is a bearer token a provider minted.
|
||||
// A builder run by a person.
|
||||
registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY"))
|
||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
username := strings.TrimSpace(os.Getenv("MESH_NPM_USER"))
|
||||
if registry == "" && secret == "" {
|
||||
return none, nil
|
||||
}
|
||||
n := builder.Npmrc{Scope: scope, Registry: registry, From: builder.FromEnvironment, Machine: machine}
|
||||
if username != "" {
|
||||
n.Username, n.Password = username, secret
|
||||
} else {
|
||||
n.Token = secret
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// fromBinding composes the registry from the npm-package-registry binding the mesh wrote and the
|
||||
// credential it sealed beside it. The mesh authenticates the ordinary way — a generated password the
|
||||
// provider only applies (novox/hq ADR 0048) — so with a username the secret is a password (basic
|
||||
// auth); without one it is a bearer token a provider minted.
|
||||
func fromBinding(path string, raw []byte, scope, machine string) (builder.Npmrc, error) {
|
||||
none := builder.Npmrc{Machine: machine}
|
||||
var told struct {
|
||||
From string `json:"from"`
|
||||
At string `json:"at"`
|
||||
As string `json:"as"`
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &told); err != nil {
|
||||
return none, fmt.Errorf("%s is not a binding: %w", path, err)
|
||||
}
|
||||
if told.At == "" {
|
||||
return none, fmt.Errorf(
|
||||
"%s says the package registry is on %q and gives no address for it", path, told.From)
|
||||
}
|
||||
// Composed from what the provider serves, so nothing here knows gitea's URL shape from
|
||||
// another registry's: it states its port, the path its registry answers on, and the scheme.
|
||||
scheme := "https"
|
||||
if s, ok := told.Serves["scheme"]; ok {
|
||||
scheme = fmt.Sprintf("%v", s)
|
||||
}
|
||||
port, ok := told.Serves["port"]
|
||||
if !ok {
|
||||
return none, fmt.Errorf("%s says nothing about which port the package registry answers on", path)
|
||||
}
|
||||
npmPath, ok := told.Serves["npm-path"]
|
||||
if !ok {
|
||||
return none, fmt.Errorf("%s says nothing about the path the package registry answers on", path)
|
||||
}
|
||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||
if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return builder.Npmrc{}, fmt.Errorf("cannot read this builder's package-registry credential: %w", err)
|
||||
return none, fmt.Errorf("cannot read this builder's package-registry credential: %w", err)
|
||||
}
|
||||
secret = strings.TrimSpace(string(raw))
|
||||
}
|
||||
if u := strings.TrimSpace(os.Getenv("MESH_NPM_USER")); u != "" {
|
||||
username = u
|
||||
n := builder.Npmrc{Scope: scope, Registry: fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath),
|
||||
From: builder.FromBinding, Machine: machine}
|
||||
if told.As != "" {
|
||||
n.Username, n.Password = told.As, secret
|
||||
} else {
|
||||
n.Token = secret
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
if registry == "" && secret == "" {
|
||||
return builder.Npmrc{}, nil
|
||||
// GenesisRegistry is the genesis document (novox/hq ADR 0336 §3): what the installer sealed to the
|
||||
// build agent on its node at genesis's packages step, as the `genesis-package-registry` own-secret —
|
||||
// the forge it raised, the port this node gave the package registry, its npm path, and the account it
|
||||
// made with that account's password. mesh-host writes it; the field names are the two repositories'
|
||||
// agreement.
|
||||
type GenesisRegistry struct {
|
||||
Scheme string `json:"scheme"`
|
||||
At string `json:"at"`
|
||||
Port int `json:"port"`
|
||||
NpmPath string `json:"npm-path"`
|
||||
As string `json:"as"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// fromGenesis composes the registry from the genesis document.
|
||||
func fromGenesis(path string, raw []byte, scope, machine string) (builder.Npmrc, error) {
|
||||
none := builder.Npmrc{Machine: machine}
|
||||
var g GenesisRegistry
|
||||
if err := json.Unmarshal(raw, &g); err != nil {
|
||||
return none, fmt.Errorf("%s is not the genesis document for the package registry: %w", path, err)
|
||||
}
|
||||
if username != "" {
|
||||
return builder.Npmrc{Scope: scope, Registry: registry, Username: username, Password: secret}, nil
|
||||
var missing []string
|
||||
for name, value := range map[string]string{"at": g.At, "npm-path": g.NpmPath, "as": g.As, "password": g.Password} {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
missing = append(missing, name)
|
||||
}
|
||||
}
|
||||
return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil
|
||||
if g.Port <= 0 {
|
||||
missing = append(missing, "port")
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
sort.Strings(missing)
|
||||
return none, fmt.Errorf("%s, the genesis document for the package registry, says nothing of %s",
|
||||
path, strings.Join(missing, ", "))
|
||||
}
|
||||
scheme := g.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
return builder.Npmrc{Scope: scope, Registry: fmt.Sprintf("%s://%s:%d%s", scheme, g.At, g.Port, g.NpmPath),
|
||||
Username: g.As, Password: g.Password, From: builder.FromGenesis, Machine: machine}, nil
|
||||
}
|
||||
|
||||
// saysRegistry is the line each build's log carries about the package registry (novox/hq ADR 0336
|
||||
// §4): which of the binding and the genesis document it resolved from, or that it has neither here.
|
||||
func saysRegistry(n builder.Npmrc) string {
|
||||
if n.Enabled() {
|
||||
return "the package registry is resolved from " + n.From
|
||||
}
|
||||
machine := n.Machine
|
||||
if machine == "" {
|
||||
machine = "this machine"
|
||||
}
|
||||
return fmt.Sprintf("no package registry: %s is unbound on %s and no genesis document is here; "+
|
||||
"a build needing it is refused", builder.RegistryRequirement, machine)
|
||||
}
|
||||
|
||||
// forgeFrom is the git credential this builder may offer a clone, composed from the same binding
|
||||
|
||||
@@ -84,10 +84,15 @@ func buildOnce(ctx context.Context, args []string) error {
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
|
||||
npmrc, err := packagesFrom()
|
||||
machine := os.Getenv("MESH_NODE")
|
||||
if machine == "" {
|
||||
machine, _ = os.Hostname()
|
||||
}
|
||||
npmrc, err := packagesFrom(machine)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " [packages] %s\n", saysRegistry(npmrc))
|
||||
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc,
|
||||
forgeFrom(),
|
||||
func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) })
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// The build agent's package registry (novox/hq ADR 0336, issue 121): its npm-package-registry
|
||||
// requirement is optional, so the binding may be absent, and on genesis's node the installer seals
|
||||
// the forge it raised as one genesis document instead.
|
||||
|
||||
// registryFiles names the three paths the build agent's environment always names, writing only those
|
||||
// given.
|
||||
func registryFiles(t *testing.T, binding, token, genesis string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for env, content := range map[string]string{
|
||||
"MESH_PACKAGE_BINDING": binding,
|
||||
"MESH_NPM_TOKEN_FILE": token,
|
||||
"MESH_GENESIS_PACKAGE_REGISTRY": genesis,
|
||||
} {
|
||||
path := filepath.Join(dir, env)
|
||||
t.Setenv(env, path)
|
||||
if content != "" {
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, env := range []string{"MESH_NPM_REGISTRY", "MESH_NPM_TOKEN", "MESH_NPM_USER", "MESH_NPM_SCOPE"} {
|
||||
t.Setenv(env, "")
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
aBinding = `{"from":"gitea","at":"anchor.mesh","as":"anchor-agent",
|
||||
"serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`
|
||||
aGenesisDocument = `{"scheme":"http","at":"127.0.0.1","port":3100,"npm-path":"/api/packages/novox/npm/",
|
||||
"as":"mesh-builder","password":"genesis-secret"}`
|
||||
)
|
||||
|
||||
func TestUntilTheRequirementBindsTheGenesisDocumentIsRead(t *testing.T) {
|
||||
registryFiles(t, "", "", aGenesisDocument)
|
||||
n, err := packagesFrom("anchor")
|
||||
if err != nil {
|
||||
t.Fatalf("an absent binding beside a genesis document was refused: %v", err)
|
||||
}
|
||||
if n.From != builder.FromGenesis || n.Registry != "http://127.0.0.1:3100/api/packages/novox/npm/" ||
|
||||
n.Username != "mesh-builder" || n.Password != "genesis-secret" || n.Scope != "@novox" {
|
||||
t.Fatalf("the genesis document was not what the registry came from: %+v", n)
|
||||
}
|
||||
file, err := n.File()
|
||||
if err != nil || !strings.Contains(file, "//127.0.0.1:3100/api/packages/novox/npm/:username=mesh-builder") {
|
||||
t.Fatalf("the .npmrc is not written from the genesis document: %v\n%s", err, file)
|
||||
}
|
||||
if got := saysRegistry(n); !strings.Contains(got, "genesis document") {
|
||||
t.Errorf("the log does not say the genesis document: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARealBindingWinsOverTheGenesisDocument(t *testing.T) {
|
||||
registryFiles(t, aBinding, "bound-secret\n", aGenesisDocument)
|
||||
n, err := packagesFrom("anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n.From != builder.FromBinding || n.Registry != "http://anchor.mesh:3000/api/packages/novox/npm/" ||
|
||||
n.Username != "anchor-agent" || n.Password != "bound-secret" {
|
||||
t.Fatalf("the binding did not win over the genesis document: %+v", n)
|
||||
}
|
||||
if got := saysRegistry(n); !strings.Contains(got, "binding") {
|
||||
t.Errorf("the log does not say the binding: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNeitherTheAgentHasNoRegistryAndSaysWhere(t *testing.T) {
|
||||
registryFiles(t, "", "", "")
|
||||
n, err := packagesFrom("workstation")
|
||||
if err != nil {
|
||||
t.Fatalf("an unbound optional requirement was a fault: %v", err)
|
||||
}
|
||||
if n.Enabled() || n.Machine != "workstation" {
|
||||
t.Fatalf("a registry came from nowhere: %+v", n)
|
||||
}
|
||||
got := saysRegistry(n)
|
||||
for _, want := range []string{"npm-package-registry", "unbound", "workstation"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("the log does not say %q: %s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGenesisDocumentMissingAFieldIsRefusedByName(t *testing.T) {
|
||||
registryFiles(t, "", "", `{"at":"127.0.0.1","port":3000,"npm-path":"/api/packages/novox/npm/","as":"mesh-builder"}`)
|
||||
if _, err := packagesFrom("anchor"); err == nil || !strings.Contains(err.Error(), "password") {
|
||||
t.Fatalf("a genesis document with no password was taken: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp
|
||||
// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its
|
||||
// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without
|
||||
// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole
|
||||
// grace on each machine.
|
||||
//
|
||||
// The mark lives in the container's own temporary directory, never in the workspace the machine keeps:
|
||||
// a new container starts without it, and a restarted one removes it before it binds again.
|
||||
func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") }
|
||||
|
||||
// notReady removes the mark: a holder starting is not taking work until it says so again.
|
||||
func notReady() error {
|
||||
if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its
|
||||
// check then says it is not ready, which is the truth the gate should hear.
|
||||
func taking(seat string) {
|
||||
if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat)
|
||||
}
|
||||
|
||||
// ready is the health check: nil when this container's agent is bound and pulling build work.
|
||||
func ready() error {
|
||||
if _, err := os.Stat(readyMark()); err != nil {
|
||||
return fmt.Errorf("not taking build work yet: %s is not there", readyMark())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The health check passes only between the agent saying it takes work and its next start (novox/hq
|
||||
// issue 544).
|
||||
func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent that never took work is ready")
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if err := ready(); err != nil {
|
||||
t.Fatalf("an agent taking work is not ready: %v", err)
|
||||
}
|
||||
// Started again, it is not ready until it binds again.
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent starting again is ready before it takes work")
|
||||
}
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatalf("removing a mark that is not there failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(readyMark()); err == nil {
|
||||
t.Fatal("the mark is still there")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A pull request into a release branch reaches the check as one** (novox/hq ADR 0358): the build seat hands the
|
||||
// check the release branch and, beside it, the trunk each repository is read at when it has no branch of that name;
|
||||
// and it hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own.
|
||||
func TestTheBuildSeatHandsTheCheckItsReleaseBranch(t *testing.T) {
|
||||
spec := checkSpecOf(link.BuildRequest{ID: "b", Repository: "r", Ref: "abc", Check: &link.CheckRequest{
|
||||
Owner: "novox", Repo: "mesh-catalog", Base: "feat/57", Release: "feat/57",
|
||||
Beside: map[string]link.CheckedOut{"mesh-host": {Repository: "h", Ref: "feat/57", Else: "main"}}}})
|
||||
if spec.Release != "feat/57" || spec.Beside["mesh-host"].Ref != "feat/57" || spec.Beside["mesh-host"].Else != "main" {
|
||||
t.Fatalf("the check was handed release %q and mesh-host %+v", spec.Release, spec.Beside["mesh-host"])
|
||||
}
|
||||
t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache")
|
||||
if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" {
|
||||
t.Fatalf("the judge cache the manifest declares is read as %q", got)
|
||||
}
|
||||
t.Setenv("MESH_JUDGE_CACHE", "")
|
||||
if got := judgeCache(); got != "" {
|
||||
t.Fatalf("with no judge cache declared, %q is used", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The siblings a check is asked to read reach the build seat's spec and, as read, the outcome (novox/hq ADR 0382).
|
||||
func TestTheSiblingsAnAskNamesReachTheSpecAndTheOutcome(t *testing.T) {
|
||||
spec := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main",
|
||||
Siblings: []link.CheckSibling{{Repository: "novox/mesh-controller", Number: 346, Head: "a", Base: "main", Dir: "mesh-controller"}},
|
||||
SiblingsUnknown: "why"}})
|
||||
if len(spec.Siblings) != 1 || spec.Siblings[0] != (builder.Sibling{Repository: "novox/mesh-controller", Number: 346,
|
||||
Head: "a", Base: "main", Dir: "mesh-controller"}) || spec.SiblingsUnknown != "why" {
|
||||
t.Fatalf("the spec reads %+v", spec)
|
||||
}
|
||||
out := siblingsOut([]builder.JudgedSibling{{Repository: "novox/mesh-controller", Number: 346, Head: "a", Base: "main", BaseCommit: "b"}})
|
||||
if len(out) != 1 || out[0].BaseCommit != "b" || out[0].Number != 346 {
|
||||
t.Fatalf("the outcome reads %+v", out)
|
||||
}
|
||||
if plain := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "o", Repo: "r"}}); plain.Siblings != nil {
|
||||
t.Fatalf("a request that names none has siblings %+v", plain.Siblings)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
||||
"version)": true, // --version
|
||||
`"stopping")`: true, // the loop.s shutdown line
|
||||
"usage)": true, // --help text, for a human
|
||||
"said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
|
||||
}
|
||||
for _, file := range []string{"once.go", "main.go"} {
|
||||
src, err := os.ReadFile(file)
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
|
||||
// check's verdict, how long it took, and each line naming what failed.
|
||||
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
|
||||
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
|
||||
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
|
||||
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
|
||||
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
|
||||
tried.Summary != "its merge-check.sh failed: TestX" {
|
||||
t.Fatalf("a failed try: %+v", tried)
|
||||
}
|
||||
// A head that does not merge fails both layers: the check's own word.
|
||||
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
|
||||
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
|
||||
t.Fatalf("a try that does not merge: %+v", conflict)
|
||||
}
|
||||
// No verdict is an error, never a pass.
|
||||
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
|
||||
t.Fatalf("a try with no verdict: %+v", none)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A provision whose credential only a person outside the mesh can give (novox/hq ADR 0337, issue 164, issue 165):
|
||||
// given once for its provider, at the controller's terminal, and sealed in one act to every consumer that binds the
|
||||
// provision from that provider now. The value is never read back, printed or kept.
|
||||
|
||||
// outsideOffer is the provider module's own name when `secret accept <node> <module> <name>` names a provision that
|
||||
// module offers with a credential issued outside the mesh, and "" when it names anything else (an own secret, or a
|
||||
// pair with --provider).
|
||||
func outsideOffer(ctx context.Context, module, name, provider string) (string, error) {
|
||||
if provider != "" {
|
||||
return "", nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if m, known := shelf[module]; known && m.IssuedOutsideOf(name) {
|
||||
return module, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// offeredOutsideBy is a module that offers the provision with a credential issued outside the mesh, or "".
|
||||
func offeredOutsideBy(ctx context.Context, provision string) (string, error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
module, _ := catalogue.OffersIssuedOutside(shelf, provision)
|
||||
return module, nil
|
||||
}
|
||||
|
||||
// consumersFrom is every consumer that binds a provision from one provider now, found by resolving every machine as
|
||||
// its next send would, and the machines that could not be resolved, with why.
|
||||
func consumersFrom(ctx context.Context, open *stores, provider, providerModule, provision string) (
|
||||
[]inventory.PairEnd, []string, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var out []inventory.PairEnd
|
||||
var unresolved []string
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
unresolved = append(unresolved, fmt.Sprintf("%s: %s", n.Name, oneLine(err.Error())))
|
||||
continue
|
||||
}
|
||||
out = append(out, bindsFrom(plan, n.Name, provider, providerModule, provision)...)
|
||||
}
|
||||
return out, unresolved, nil
|
||||
}
|
||||
|
||||
// bindsFrom is every consumer on one machine whose resolution answers the provision from that provider. Pure.
|
||||
func bindsFrom(plan catalogue.Resolution, machine, provider, providerModule, provision string) []inventory.PairEnd {
|
||||
var out []inventory.PairEnd
|
||||
for _, need := range plan.Needs {
|
||||
if need.ByRecord || need.Name != provision || need.From != provider {
|
||||
continue
|
||||
}
|
||||
if need.Module != "" && need.Module != providerModule {
|
||||
continue
|
||||
}
|
||||
out = append(out, inventory.PairEnd{Consumer: machine, ConsumerModule: need.For, Local: need.Local})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return out[i].ConsumerModule+"\x00"+out[i].Local < out[j].ConsumerModule+"\x00"+out[j].Local
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// acceptForProvision gives a provision's credential issued outside the mesh, once for its provider (ADR 0337 §3).
|
||||
func acceptForProvision(ctx context.Context, provider, providerModule, provision, value string) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
consumers, unresolved, err := consumersFrom(ctx, open, provider, providerModule, provision)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accepted, err := open.inventory.AcceptSecretForProvision(ctx, provision, provider, providerModule, value, consumers)
|
||||
value = ""
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Print(acceptedSaid(provider, providerModule, provision, accepted, unresolved))
|
||||
if len(accepted.Sealed) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := sendTo(ctx, open, accepted.Machines); err != nil {
|
||||
return fmt.Errorf("%s is sealed and could not be sent: %w; `push %s` sends it", provision, err,
|
||||
strings.Join(accepted.Machines, "`, `push "))
|
||||
}
|
||||
fmt.Printf(" sent %s; each consumer starts again on it\n", strings.Join(accepted.Machines, ", "))
|
||||
return nil
|
||||
}
|
||||
|
||||
// acceptedSaid is what one accept for a provider did, in words that never carry the value. Pure.
|
||||
func acceptedSaid(provider, providerModule, provision string, a inventory.ProvisionAccepted, unresolved []string) string {
|
||||
var b strings.Builder
|
||||
if len(a.Sealed) == 0 {
|
||||
fmt.Fprintf(&b, "%s from %s on %s: no consumer binds it now, so nothing was sealed; a consumer that binds "+
|
||||
"later waits for it, and this line given again seals it to every consumer then\n", provision,
|
||||
providerModule, provider)
|
||||
} else {
|
||||
var names []string
|
||||
for _, c := range a.Sealed {
|
||||
names = append(names, fmt.Sprintf("%s on %s%s", c.ConsumerModule, c.Consumer, asLocal(c.Local)))
|
||||
}
|
||||
fmt.Fprintf(&b, "%s from %s on %s is given, sealed to %s and to %s\n", provision, providerModule, provider,
|
||||
strings.Join(names, ", "), provider)
|
||||
fmt.Fprintf(&b, " the mesh cannot read it back, never makes one of its own for it, and will not rotate it\n")
|
||||
}
|
||||
for _, c := range a.Unsealable {
|
||||
fmt.Fprintf(&b, " not sealed to %s on %s: no sealing key on that machine; give it again once it has one\n",
|
||||
c.ConsumerModule, c.Consumer)
|
||||
}
|
||||
for _, u := range unresolved {
|
||||
fmt.Fprintf(&b, " not known whether a consumer on %s binds it, which did not resolve; give it again once it does\n", u)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -26,8 +26,11 @@ import (
|
||||
//
|
||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
||||
// its service manager restarts it as a candidate (serve, in push.go).
|
||||
// Another holder seen, or the key's age passed with no renewal made, is the lease lost: the gate
|
||||
// closes at once and the process exits, so its service manager restarts it as a candidate (serve,
|
||||
// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the
|
||||
// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal
|
||||
// made opens the gate again.
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
@@ -64,6 +67,11 @@ type actor struct {
|
||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
// renewEvery and poll, when set, are how often the serving controller renews the lease it holds and
|
||||
// looks again at one another holds, in place of the lease's own: a test's, so two controllers hand the
|
||||
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
|
||||
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
|
||||
renewEvery, poll time.Duration
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
@@ -223,7 +231,7 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
|
||||
+70
-17
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"io"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -54,6 +55,9 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
||||
if err := refusedMovingTheController(modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
||||
// what the reads before it would say about a machine left out is not the answer's.
|
||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
@@ -125,22 +129,28 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
||||
for _, line := range settled {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||
// node's list, and every other node's, is `status`'s.
|
||||
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||
var issued []string
|
||||
for _, module := range added {
|
||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||
answer += "\n " + line
|
||||
issued = append(issued, line)
|
||||
}
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
// **The one resolution the rest of the answer is said from** (novox/hq issue 237), taken after
|
||||
// the write: what this act changed about this node's unmet seat dependencies, and nothing else
|
||||
// (novox/hq ADR 0207) — a dependency of a module just assigned, or one this assignment met; the
|
||||
// rest of the node's list, and every other node's, is `status`'s — and whether the node resolves.
|
||||
after := resolvedAfter(ctx, open, shelf, node)
|
||||
for _, line := range unheldChangeTo(shelf, node, before, after.modules, after.unheld) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
for _, line := range issued {
|
||||
answer += "\n " + line
|
||||
}
|
||||
plan, err := after.plan, after.err
|
||||
if err != nil {
|
||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||
@@ -193,8 +203,9 @@ func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, mo
|
||||
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Two modules declaring one package, path or unit is refused before anything is recorded
|
||||
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
||||
// Two modules declaring one path or unit, or one package present and absent, is refused before anything
|
||||
// is recorded (novox/hq ADR 0210, ADR 0303, 04-ISSUES/235): kept, the node would not resolve until one came
|
||||
// off again. Two modules declaring one package present is not a collision (ADR 0303, issue 628).
|
||||
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -218,6 +229,8 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -262,15 +275,22 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
}
|
||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, strings.Join(modules, ", "), node)
|
||||
var left []string
|
||||
for _, a := range assigned {
|
||||
if !slices.Contains(modules, a) {
|
||||
left = append(left, a)
|
||||
}
|
||||
}
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
after := resolvedAfter(ctx, open, shelf, node)
|
||||
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// Whether the node itself still resolves, from the same resolution: before, only the rest of the
|
||||
// mesh's reads said it, on standard error and once per other machine.
|
||||
if after.err != nil {
|
||||
if unresolvable(after.err) {
|
||||
answer += fmt.Sprintf("\n but %s does not resolve now, so nothing will be sent to it:", node)
|
||||
} else {
|
||||
answer += fmt.Sprintf("\n but whether %s still resolves could not be read:", node)
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimRight(after.err.Error(), "\n"), "\n") {
|
||||
answer += "\n " + strings.TrimSpace(line)
|
||||
}
|
||||
}
|
||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||
answer += "\n " + line
|
||||
@@ -278,6 +298,39 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
||||
// (novox/hq issue 237): the plan or why there is none, the modules it has on the node, and its unmet
|
||||
// seat dependencies.
|
||||
type afterAct struct {
|
||||
plan catalogue.Resolution
|
||||
err error
|
||||
modules []string
|
||||
unheld []catalogue.Unheld
|
||||
}
|
||||
|
||||
// resolvedAfter resolves the node as `plan` does, once, after the act's write. When the node resolves,
|
||||
// its modules and unmet seat dependencies are the resolution's — the closure, so a holder pulled in by
|
||||
// a requirement counts as `plan` counts it. When it does not, they are judged over the assignments read
|
||||
// after the write, which can only find more unmet than the closure would: a seat said held there is
|
||||
// held in the closure too.
|
||||
func resolvedAfter(ctx context.Context, open *stores, shelf map[string]catalogue.Manifest, node string) afterAct {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err == nil {
|
||||
out := afterAct{plan: plan, unheld: plan.Unheld}
|
||||
for _, m := range plan.Modules {
|
||||
out.modules = append(out.modules, m.Module)
|
||||
}
|
||||
return out
|
||||
}
|
||||
out := afterAct{err: err}
|
||||
assigned, readErr := open.inventory.Assigned(ctx, node)
|
||||
if readErr != nil {
|
||||
return out
|
||||
}
|
||||
out.modules, out.unheld = assigned, unheldOver(shelf, node, assigned)
|
||||
return out
|
||||
}
|
||||
|
||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||
// command API and the controller seat's verbs as they do from the command line.
|
||||
|
||||
@@ -140,7 +140,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open}.Announceable(ctx)
|
||||
announced, err := following{open: open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -424,8 +424,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
||||
}
|
||||
// And its settings on this machine, composed against its definition (rule 1, rule 6).
|
||||
for _, layer := range c.layers {
|
||||
keys := make([]string, 0, len(layer.Values))
|
||||
for k := range layer.Values {
|
||||
said := layer.Whole()
|
||||
keys := make([]string, 0, len(said))
|
||||
for k := range said {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
@@ -722,7 +723,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
return "", err
|
||||
}
|
||||
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks}
|
||||
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks, local: with.LocalNetworks}
|
||||
filtering, err := inv.FilteringOf(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -930,6 +931,9 @@ type derivedFilter struct {
|
||||
// The filter constrains what arrives on them; everything arriving elsewhere is this machine's
|
||||
// own guest and is not filtered.
|
||||
outwardLinks []string
|
||||
// local is the machine's local networks, which a port from `local-network` stays open to (novox/hq
|
||||
// ADR 0340).
|
||||
local []string
|
||||
}
|
||||
|
||||
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
||||
@@ -977,6 +981,17 @@ func (d derivedFilter) fate(r inventory.Reach) string {
|
||||
if !onMesh {
|
||||
return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by)
|
||||
}
|
||||
case catalogue.FromLocalNetwork:
|
||||
// The private network and this machine's local networks (novox/hq ADR 0340): it closes to
|
||||
// everything else outside, and says which local networks stay.
|
||||
if !onMesh {
|
||||
if len(d.local) == 0 {
|
||||
return fmt.Sprintf("%s — declared by %s from the local network, and this machine "+
|
||||
"reported none", closesOutside, by)
|
||||
}
|
||||
return fmt.Sprintf("WILL CLOSE to everything outside the private network and %s — declared "+
|
||||
"by %s from the local network", strings.Join(d.local, ", "), by)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("declared by %s (from %s)", by, rule.From)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
|
||||
// for longer than one apply interval (novox/hq issue 233).
|
||||
const kindApplyFailing = "apply-failing"
|
||||
|
||||
// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
|
||||
// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
|
||||
const applyEvery = 5 * time.Minute
|
||||
|
||||
// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
|
||||
// (mesh-host's probe.go): "the package manager (pacman)".
|
||||
const packageManagerKey = "the package manager ("
|
||||
|
||||
// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
|
||||
// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
|
||||
// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
|
||||
// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
|
||||
// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
|
||||
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
|
||||
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
|
||||
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
|
||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
heard := map[string]bool{}
|
||||
for _, m := range f.machines {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
||||
heard[m.name] = true
|
||||
}
|
||||
}
|
||||
var out []conditions.Observation
|
||||
said := map[string]bool{}
|
||||
for _, d := range f.doing {
|
||||
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
|
||||
continue
|
||||
}
|
||||
out = append(out, applyFailing(d, f.now))
|
||||
said[d.Node] = true
|
||||
}
|
||||
// **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's
|
||||
// newest word, judged by the same bound — the client says a failure once, and fills again on every change —
|
||||
// on a machine whose node-engine applied what it was sent.
|
||||
failed := map[string][]inventory.ProvReport{}
|
||||
for _, r := range f.provFailed {
|
||||
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
|
||||
continue
|
||||
}
|
||||
failed[r.Node] = append(failed[r.Node], r)
|
||||
}
|
||||
for _, node := range sortedNames(failed) {
|
||||
out = append(out, provisioningFailing(node, failed[node], f.now))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
|
||||
// place.
|
||||
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
|
||||
sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
|
||||
var names, words []string
|
||||
since := failed[0].At
|
||||
for _, r := range failed {
|
||||
names = append(names, r.Module)
|
||||
words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
|
||||
if r.At.Before(since) {
|
||||
since = r.At
|
||||
}
|
||||
}
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
||||
Severity: conditions.Warning, Said: strings.Join(words, "; "),
|
||||
Resolved: m + " applies what it is sent again"}
|
||||
o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m,
|
||||
strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery))
|
||||
o.Headline = m + " is not applying its update"
|
||||
o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+
|
||||
"details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since)))
|
||||
return o
|
||||
}
|
||||
|
||||
func orNoReason(s string) string {
|
||||
if s == "" {
|
||||
return "no reason given"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// applyFailing is one machine's S21 condition.
|
||||
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
|
||||
m := d.Node
|
||||
since := d.Since.UTC().Format("2006-01-02 15:04 MST")
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
||||
Severity: conditions.Warning, Said: doingWords(d),
|
||||
Resolved: m + " applies what it is sent again"}
|
||||
how := "refused its declaration whole"
|
||||
if d.Outcome != inventory.OutcomeRefused {
|
||||
how = "applied its declaration only in part"
|
||||
}
|
||||
o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
|
||||
o.Headline = m + " is not applying its update"
|
||||
o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
|
||||
m, plainHow(d), humanDuration(now.Sub(*d.Since)))
|
||||
if tool := unansweringTool(d); tool != "" {
|
||||
o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
|
||||
"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
|
||||
o.Headline = fmt.Sprintf("%s cannot install packages", m)
|
||||
o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
|
||||
"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
|
||||
"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// plainHow is how a machine failed, as the operator reads it.
|
||||
func plainHow(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused every update"
|
||||
}
|
||||
return "applied its updates only in part"
|
||||
}
|
||||
|
||||
// unansweringTool is the package manager a machine said does not answer, or "".
|
||||
func unansweringTool(d inventory.Doing) string {
|
||||
for _, r := range d.Failed {
|
||||
if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
|
||||
return strings.TrimSuffix(rest, ")")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// failedNames is what failed, by name: the refusal, or the failed resources' ids.
|
||||
func failedNames(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused"
|
||||
}
|
||||
ids := make([]string, 0, len(d.Failed))
|
||||
for _, r := range d.Failed {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return "failed " + strings.Join(ids, ", ")
|
||||
}
|
||||
|
||||
// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
|
||||
func doingWords(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused: " + oneLine(d.Refused)
|
||||
}
|
||||
var parts []string
|
||||
for _, r := range d.Failed {
|
||||
parts = append(parts, r.ID+": "+oneLine(r.Error))
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
|
||||
// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
|
||||
// the same failure on times reports since for before now.
|
||||
func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
|
||||
since := now.Add(-forHowLong)
|
||||
return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
|
||||
Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
|
||||
{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
|
||||
"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
|
||||
"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
|
||||
{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
|
||||
"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
|
||||
"file /etc/pacman.conf could not be read: No such file or directory"}}}
|
||||
}
|
||||
|
||||
// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
|
||||
// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
|
||||
func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
|
||||
got := watchApplyFailing(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
|
||||
t.Errorf("the condition is not the machine's: %s", o.Key())
|
||||
}
|
||||
if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
|
||||
t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
|
||||
}
|
||||
if strings.Contains(o.Summary, "/etc/") {
|
||||
t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
|
||||
}
|
||||
if !strings.Contains(o.Said, "could not be read") {
|
||||
t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
|
||||
}
|
||||
if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
|
||||
t.Errorf("the plain words do not say it: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
|
||||
// node-engine applied the rest.
|
||||
func TestARefusalThatGoesOnIsRaised(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
since := now.Add(-time.Hour)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
|
||||
Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
|
||||
got := watchApplyFailing(f)
|
||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
|
||||
!strings.Contains(got[0].Said, "pacman does not answer") {
|
||||
t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
|
||||
// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
|
||||
// younger is not yet; and a silent or sleeping machine is S1's.
|
||||
func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
|
||||
if got := watchApplyFailing(f); len(got) != 1 {
|
||||
t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
|
||||
}
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
|
||||
}
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
||||
f.machines[0].lastHeard = now.Add(-time.Hour)
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a silent machine was raised for its last report: %+v", got)
|
||||
}
|
||||
f = calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
||||
f.doing[0].Node = "a-machine-the-mesh-does-not-know"
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a machine not heard of was raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
|
||||
// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
|
||||
// is cleared through the keeper on the next tick.
|
||||
func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
|
||||
failing := calm(now)
|
||||
failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
|
||||
w.see(t.Context(), failing)
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
|
||||
t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
|
||||
}
|
||||
|
||||
clean := calm(now.Add(5 * time.Minute))
|
||||
clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
|
||||
w.see(t.Context(), clean)
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -213,6 +213,8 @@ type asker struct {
|
||||
|
||||
saidNoRouter bool
|
||||
saidNoGrant bool
|
||||
// saidGrantCaveat is whether a grant held with a caveat was said, so it is said once.
|
||||
saidGrantCaveat bool
|
||||
|
||||
mu sync.Mutex
|
||||
nudged chan struct{}
|
||||
@@ -317,6 +319,13 @@ func (a *asker) reconcile(ctx context.Context) error {
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why)
|
||||
}
|
||||
if why != "" && !a.saidGrantCaveat {
|
||||
// Held, with a caveat worth one line (an older bus module cannot confirm the reload).
|
||||
a.logf("the controller asks: %s", why)
|
||||
a.saidGrantCaveat = true
|
||||
} else if why == "" {
|
||||
a.saidGrantCaveat = false
|
||||
}
|
||||
a.saidNoGrant = false
|
||||
}
|
||||
channels := ""
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -808,3 +809,207 @@ func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testin
|
||||
t.Errorf("the explanation says its verdict twice: %q", e)
|
||||
}
|
||||
}
|
||||
|
||||
// hq issue 353, the review: the bus holds the controller's grant to ask only when both facts hold — the mesh
|
||||
// composes the controller's own grant, and the bus's module says it is in force on the running server — and
|
||||
// each fact not known is not held, said with why.
|
||||
func TestTheGrantIsHeldOnlyWhenComposedAndInForceOnTheBus(t *testing.T) {
|
||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
||||
no := func(why string) grantJudge {
|
||||
return func(context.Context) (bool, string, error) { return false, why, nil }
|
||||
}
|
||||
broken := func(context.Context) (bool, string, error) { return false, "", errors.New("the store is away") }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
composed, inForce grantJudge
|
||||
held bool
|
||||
says string
|
||||
}{
|
||||
{"composed and in force", yes, yes, true, ""},
|
||||
{"not composed", no("no router is assigned"), yes, false, "no router is assigned"},
|
||||
{"composed, written, not reloaded", yes, no("the server has not reloaded"), false, "has not reloaded"},
|
||||
{"the composition unknown", broken, yes, false, "could not be worked out: the store is away"},
|
||||
{"the bus unknown", yes, broken, false, "could not be worked out: the store is away"},
|
||||
{"nothing judges the bus", yes, nil, false, "not judged"},
|
||||
} {
|
||||
held, why, err := grantHeldBy(c.composed, c.inForce, time.Now)(context.Background())
|
||||
if err != nil || held != c.held || !strings.Contains(why, c.says) {
|
||||
t.Errorf("%s: held %v (%q, %v)", c.name, held, why, err)
|
||||
}
|
||||
}
|
||||
// Remembered for grantLookEvery, then judged again.
|
||||
now := time.Date(2026, 10, 9, 18, 0, 0, 0, time.UTC)
|
||||
inForce := false
|
||||
judge := grantHeldBy(yes, func(context.Context) (bool, string, error) { return inForce, "not yet", nil },
|
||||
func() time.Time { return now })
|
||||
if held, _, _ := judge(context.Background()); held {
|
||||
t.Fatal("held before the bus enforces it")
|
||||
}
|
||||
inForce = true
|
||||
if held, _, _ := judge(context.Background()); held {
|
||||
t.Error("judged again inside grantLookEvery")
|
||||
}
|
||||
now = now.Add(grantLookEvery)
|
||||
if held, _, _ := judge(context.Background()); !held {
|
||||
t.Error("not judged again after grantLookEvery")
|
||||
}
|
||||
}
|
||||
|
||||
// The controller's own grant, as composed: present while a router that takes asks is assigned, absent
|
||||
// otherwise — whatever else in the user list changes.
|
||||
func TestTheControllersOwnGrantToAskIsReadFromTheComposition(t *testing.T) {
|
||||
router := broker.Declared{Module: "messenger", Holds: []broker.Seat{{Name: broker.AsksSeat, Scope: "mesh",
|
||||
Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "decided"}}}}
|
||||
other := broker.Declared{Module: "dunst"}
|
||||
with := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {router, other}}}
|
||||
without := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {other}}}
|
||||
if may, err := controllerMayAsk(with); err != nil || !may {
|
||||
t.Errorf("with a router assigned: %v %v", may, err)
|
||||
}
|
||||
if may, err := controllerMayAsk(without); err != nil || may {
|
||||
t.Errorf("with no router assigned: %v %v", may, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus module's answer, with the record's word on the list sent as the check only where the answer does
|
||||
// not settle it (reviews 22 and 31 of PR 188); each way it is not held names its own remedy.
|
||||
func TestTheBusModulesAnswerNamesTheRemedyForEachCase(t *testing.T) {
|
||||
const (
|
||||
reloaded = `{"allowed":true,"in_force":true,"file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:56:05Z"}`
|
||||
notLoaded = `{"allowed":true,"in_force":false,"why":"the user list on the machine allows it, and the server has not reloaded since it was written","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`
|
||||
noVarz = `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`
|
||||
noStat = `{"allowed":true,"in_force":false,"why":"when the bus's user list was written could not be read inside mesh-broker-nats: exit 1"}`
|
||||
notAllowed = `{"allowed":false,"in_force":false,"why":"the user list on the machine does not allow it"}`
|
||||
)
|
||||
current := busListState{Machine: "anchor"}
|
||||
behind := busListState{Machine: "anchor", Behind: true}
|
||||
stepped := busListState{Machine: "anchor", Behind: true, BusStep: "sending anchor would replace the bus; `bus upgrade --why …`"}
|
||||
uncomposed := busListState{Machine: "anchor", Behind: true, Uncomposed: errors.New("module foo requires a provision nobody provides")}
|
||||
for _, c := range []struct {
|
||||
name, answer string
|
||||
askErr error
|
||||
list busListState
|
||||
listErr error
|
||||
held bool
|
||||
says, not string
|
||||
}{
|
||||
{"in force", reloaded, nil, behind, nil, true, "", "push"},
|
||||
{"written, not reloaded", notLoaded, nil, current, nil, false, "within about 5 s", "push"},
|
||||
{"in force not known: varz", noVarz, nil, current, nil, false, "is not known: the server's last configuration load", "wait"},
|
||||
{"in force not known: stat", noStat, nil, current, nil, false, "is not known: when the bus's user list was written", "wait"},
|
||||
{"not allowed, list behind", notAllowed, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
||||
{"not allowed, bus step pending", notAllowed, nil, stepped, nil, false, "`bus upgrade", "`push anchor`"},
|
||||
{"not allowed, list uncomposed", notAllowed, nil, uncomposed, nil, false, "`plan anchor`", "`push anchor`"},
|
||||
{"older module, allowed, list current", `{"allowed":true}`, nil, current, nil, true, "not confirmed", "push"},
|
||||
{"older module, allowed, list behind", `{"allowed":true}`, nil, behind, nil, true, "not confirmed", "push"},
|
||||
{"older module, allowed, list unknown", `{"allowed":true}`, nil, current, errors.New("the store is away"), true, "`bus upgrade`", "push"},
|
||||
{"older module, not allowed, list behind", `{"allowed":false}`, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
||||
{"module cannot answer, list current", "", errors.New("no responders"), current, nil, true, "could not answer", "push"},
|
||||
{"module cannot answer, list behind", "", errors.New("no responders"), behind, nil, false, "`push anchor` carries it", ""},
|
||||
{"module cannot answer, list unknown", "", errors.New("no responders"), current, errors.New("the store is away"), false, "could not be worked out", ""},
|
||||
{"answer unreadable, list behind", `not json`, nil, behind, nil, false, "could not be read", ""},
|
||||
} {
|
||||
judge := busGrantBy(
|
||||
func(context.Context) (string, json.RawMessage, error) {
|
||||
return "anchor", json.RawMessage(c.answer), c.askErr
|
||||
},
|
||||
func(context.Context) (busListState, error) { return c.list, c.listErr })
|
||||
held, why, _ := judgeGrantOnce(judge)
|
||||
if held != c.held || !strings.Contains(why, c.says) || (c.not != "" && strings.Contains(why, c.not)) {
|
||||
t.Errorf("%s: held %v (%q)", c.name, held, why)
|
||||
}
|
||||
}
|
||||
none := busGrantBy(func(context.Context) (string, json.RawMessage, error) { return "", nil, nil }, nil)
|
||||
if held, why, _ := judgeGrantOnce(none); held || !strings.Contains(why, "no module holds the bus") {
|
||||
t.Errorf("no bus: held %v (%q)", held, why)
|
||||
}
|
||||
}
|
||||
|
||||
// judgeGrantOnce is the whole judgement with the composed grant held, as the asker makes it.
|
||||
func judgeGrantOnce(bus grantJudge) (bool, string, error) {
|
||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
||||
return grantHeldBy(yes, bus, time.Now)(context.Background())
|
||||
}
|
||||
|
||||
// The orders reviews 22 and 31 of PR 188 found: this controller live before the bus module that answers
|
||||
// in_force, with the bus's machine behind on its list — and, after mesh-catalog #166 merged, unpushable until a
|
||||
// person's bus upgrade. An older module that allows the ask asks (once, with the caveat said); a new module that
|
||||
// has not reloaded names the wait, not a push; one that cannot say whether it reloaded names what is not known.
|
||||
func TestAnOlderBusModuleNeitherSilencesTheAsksNorNamesTheWrongRemedy(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, answer string
|
||||
list busListState
|
||||
asks bool
|
||||
says, not string
|
||||
}{
|
||||
{"older module, list current", `{"allowed":true}`, busListState{Machine: "anchor"}, true, "not confirmed", ""},
|
||||
{"older module, list behind, bus step pending", `{"allowed":true}`,
|
||||
busListState{Machine: "anchor", Behind: true, BusStep: "a planned step"}, true, "not confirmed", ""},
|
||||
{"new module, not reloaded",
|
||||
`{"allowed":true,"in_force":false,"why":"not reloaded","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`,
|
||||
busListState{Machine: "anchor"}, false, "within about 5 s", "push"},
|
||||
{"new module, reload not known", `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`,
|
||||
busListState{Machine: "anchor"}, false, "is not known", "wait"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
var said []string
|
||||
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
var raised [][]conditions.Observation
|
||||
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
||||
raised = append(raised, obs)
|
||||
return nil
|
||||
}
|
||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
||||
r.a.grantHeld = grantHeldBy(yes, busGrantBy(
|
||||
func(context.Context) (string, json.RawMessage, error) {
|
||||
return "anchor", json.RawMessage(c.answer), nil
|
||||
},
|
||||
func(context.Context) (busListState, error) { return c.list, nil }),
|
||||
func() time.Time { return r.now })
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
_ = r.a.reconcile(context.Background())
|
||||
sent := r.asksSent(t)
|
||||
if c.asks {
|
||||
if len(sent) != 1 || sent[0].About != heldCondition().Key {
|
||||
t.Fatalf("not asked: %+v", sent)
|
||||
}
|
||||
n := 0
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, c.says) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("the caveat said %d times: %q", n, said)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(sent) != 0 {
|
||||
t.Fatalf("asked while the bus does not hold the grant: %+v", sent)
|
||||
}
|
||||
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
||||
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
||||
t.Fatalf("not said as undelivered: %+v", raised)
|
||||
}
|
||||
summary := raised[len(raised)-1][0].Summary
|
||||
if !strings.Contains(summary, c.says) || (c.not != "" && strings.Contains(summary, c.not)) {
|
||||
t.Errorf("the condition names the wrong remedy: %q", summary)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The serving asker is wired to the real judgement: with records and a bus it cannot read, the grant is not
|
||||
// held, and so nothing is asked (a stub saying yes, or no judgement at all, would ask).
|
||||
func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
||||
a := servingAsker(&stores{}, nil, nil, nil)
|
||||
if a.grantHeld == nil {
|
||||
t.Fatal("the serving asker does not judge the bus's grant")
|
||||
}
|
||||
held, why, err := a.grantHeld(context.Background())
|
||||
if err != nil || held || !strings.Contains(why, "could not be worked out") {
|
||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,12 +235,27 @@ func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, err
|
||||
}
|
||||
}
|
||||
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353): the user list the
|
||||
// machine holding the bus was last sent is the one the mesh composes now (brokerBehind, the same judgement a
|
||||
// push makes to send that machine first). While it is behind, the controller's ask is refused by the bus,
|
||||
// whatever the record says of the router, so nothing is asked and the operator is told to push that machine.
|
||||
// Judged at most every grantLookEvery: composing the list resolves the bus's machine whole.
|
||||
func grantHeldIn(open *stores) func(ctx context.Context) (bool, string, error) {
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353), judged on the two
|
||||
// facts that make it so, and failing closed on each:
|
||||
//
|
||||
// 1. **composed**: the user list the mesh composes now grants the controller itself to publish its ask
|
||||
// (controllerMayAsk) — the controller's own grant, not whether any user of the list changed;
|
||||
// 2. **in force**: the bus's own module, on the machine holding the bus, says the user list there allows it
|
||||
// and the server reloaded after that list was written (the nats module's `nats_user_can`, `in_force`) — a
|
||||
// list sent is not a list the bus enforces until it reloads.
|
||||
//
|
||||
// Anything that cannot be worked out is not held, with why, so the conditions that need the operator are said
|
||||
// as undelivered rather than asked into a refusal.
|
||||
func grantHeldIn(open *stores, conn *nats.Conn) func(ctx context.Context) (bool, string, error) {
|
||||
return grantHeldBy(composedGrantIn(open), busGrantIn(open, conn), time.Now)
|
||||
}
|
||||
|
||||
// grantJudge is one of the two facts: whether it holds, and why not.
|
||||
type grantJudge func(ctx context.Context) (bool, string, error)
|
||||
|
||||
// grantHeldBy is the judgement over the two facts, remembered for grantLookEvery: composing the list resolves
|
||||
// the bus's machine whole, and asking the bus's module is a call.
|
||||
func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx context.Context) (bool, string, error) {
|
||||
var mu sync.Mutex
|
||||
var at time.Time
|
||||
var held bool
|
||||
@@ -248,22 +263,251 @@ func grantHeldIn(open *stores) func(ctx context.Context) (bool, string, error) {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !at.IsZero() && time.Since(at) < grantLookEvery {
|
||||
if !at.IsZero() && now().Sub(at) < grantLookEvery {
|
||||
return held, why, nil
|
||||
}
|
||||
machine, behind, err := brokerBehind(ctx, open, nil)
|
||||
held, why = judgeGrant(ctx, composed, inForce)
|
||||
at = now()
|
||||
return held, why, nil
|
||||
}
|
||||
}
|
||||
|
||||
// judgeGrant is held only when both facts hold; an error is a fact not known, never a yes. A fact that holds
|
||||
// with a caveat (a reload not confirmed) gives that caveat as the why of a yes.
|
||||
func judgeGrant(ctx context.Context, composed, inForce grantJudge) (bool, string) {
|
||||
var caveat []string
|
||||
for _, fact := range []struct {
|
||||
judge grantJudge
|
||||
what string
|
||||
}{{composed, "whether the mesh composes the controller a grant to ask"}, {inForce, "whether the bus enforces it"}} {
|
||||
if fact.judge == nil {
|
||||
return false, fact.what + " is not judged here"
|
||||
}
|
||||
ok, why, err := fact.judge(ctx)
|
||||
if err != nil {
|
||||
return false, fact.what + " could not be worked out: " + err.Error()
|
||||
}
|
||||
if !ok {
|
||||
return false, why
|
||||
}
|
||||
if why != "" {
|
||||
caveat = append(caveat, why)
|
||||
}
|
||||
}
|
||||
return true, strings.Join(caveat, "; ")
|
||||
}
|
||||
|
||||
// controllerMayAsk says whether the controller's own grant, as composed from these records, lets it publish its
|
||||
// ask (ADR 0259 §3: composed while a router that takes asks under its asker's name is assigned).
|
||||
func controllerMayAsk(records broker.Records) (bool, error) {
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Kind != broker.KindController {
|
||||
continue
|
||||
}
|
||||
perms, err := broker.PermissionsFor(u)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return broker.MayPublish(perms, asks.AskSubject(askerName)), nil
|
||||
}
|
||||
return false, errors.New("the composition holds no controller")
|
||||
}
|
||||
|
||||
// composedGrantIn is the first fact, from the mesh's records.
|
||||
func composedGrantIn(open *stores) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return false, "", errors.New("the mesh's records are not open")
|
||||
}
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
at, held, why = time.Now(), !behind, ""
|
||||
if behind {
|
||||
why = fmt.Sprintf("the bus's user list on %s is behind what the mesh composes, so the bus has not been "+
|
||||
"given the controller's grant to ask; `push %s` carries it", machine, machine)
|
||||
may, err := controllerMayAsk(records)
|
||||
if err != nil || may {
|
||||
return may, "", err
|
||||
}
|
||||
return held, why, nil
|
||||
return false, "the mesh composes the controller no grant to ask: no router that takes asks under its " +
|
||||
"asker's name is assigned", nil
|
||||
}
|
||||
}
|
||||
|
||||
// busGrantIn is the second fact, asked of the bus's own module on the machine holding the bus, with the check
|
||||
// main made before it (novox/hq issue 353, PR 186: is the user list sent to the bus's machine the one composed
|
||||
// now) kept for the cases the module's word does not settle.
|
||||
func busGrantIn(open *stores, conn *nats.Conn) grantJudge {
|
||||
ask := func(ctx context.Context) (string, json.RawMessage, error) {
|
||||
if open == nil || open.inventory == nil || conn == nil {
|
||||
return "", nil, errors.New("the mesh's records or the bus are not open")
|
||||
}
|
||||
holders, err := seatHolders(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
h, held := holders[theBrokerSeat]
|
||||
if !held || h.Node == "" || h.Module == "" {
|
||||
return "", nil, nil
|
||||
}
|
||||
a, err := link.AskModuleToolOn(ctx, conn, h.Module, busUserCanTool, h.Node, map[string]any{
|
||||
"user": broker.ControllerName, "action": "publish", "subject": asks.AskSubject(askerName)}, 15*time.Second)
|
||||
if err != nil {
|
||||
return h.Node, nil, err
|
||||
}
|
||||
if a.Error != "" {
|
||||
return h.Node, nil, errors.New(a.Error)
|
||||
}
|
||||
return h.Node, a.Result, nil
|
||||
}
|
||||
list := func(ctx context.Context) (busListState, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return busListState{}, errors.New("the mesh's records are not open")
|
||||
}
|
||||
s, err := brokerListState(ctx, open, nil)
|
||||
if err != nil || s.Machine == "" || !s.Behind {
|
||||
return s, err
|
||||
}
|
||||
step, err := busHeld(ctx, open.inventory, []string{s.Machine})
|
||||
if err != nil {
|
||||
return busListState{}, err
|
||||
}
|
||||
s.BusStep = step[s.Machine]
|
||||
return s, nil
|
||||
}
|
||||
return busGrantBy(ask, list)
|
||||
}
|
||||
|
||||
// busListState is the user list on the bus's machine as the controller's record knows it: whether the list
|
||||
// sent there is behind the one composed now, why it could not be composed, and the bus step a send there waits
|
||||
// on (that machine cannot be pushed until a person's `bus upgrade`).
|
||||
type busListState struct {
|
||||
Machine string
|
||||
Behind bool
|
||||
Uncomposed error
|
||||
BusStep string
|
||||
}
|
||||
|
||||
// remedy is the one thing that sends the bus's machine the list composed now: the plan's own reason when it
|
||||
// cannot be composed, the bus upgrade while sends there wait on it, and otherwise a push of that machine.
|
||||
func (s busListState) remedy(node string) string {
|
||||
if s.Machine != "" {
|
||||
node = s.Machine
|
||||
}
|
||||
switch {
|
||||
case s.Uncomposed != nil:
|
||||
return fmt.Sprintf("the user list for %s cannot be composed (%v); `plan %s` says what to fix first",
|
||||
node, s.Uncomposed, node)
|
||||
case s.BusStep != "":
|
||||
return fmt.Sprintf("sends to %s wait on the bus's planned step, so a push will not carry it: %s", node, s.BusStep)
|
||||
}
|
||||
return fmt.Sprintf("`push %s` carries it", node)
|
||||
}
|
||||
|
||||
// busGrantBy judges the bus's word, and names for each way it is not held the one thing that makes it held:
|
||||
//
|
||||
// - **in force**: held;
|
||||
// - **allowed, from a module older than `in_force`**: held, with the caveat that the reload is not confirmed.
|
||||
// The file allows the publish, and the bus reloads within about 5 s of a change to it, so the worst case is
|
||||
// one ask refused in those seconds, which is asked again (issue 369). The list check is not consulted:
|
||||
// while a bus step is pending the bus's machine cannot be pushed, so any change to the composed list would
|
||||
// read as behind for good and silence every ask until a person's bus upgrade (the review of PR 188);
|
||||
// - **allowed, written, not reloaded** (the module gives both times): wait about 5 s;
|
||||
// - **allowed, in force not known** (the module could not read a time): what is not known, said;
|
||||
// - **not allowed, or the module cannot answer**: the list check decides — the remedy that sends the list
|
||||
// when it is behind (busListState.remedy), and, for a module that cannot answer, held while the list sent
|
||||
// is current, as main judged it.
|
||||
//
|
||||
// ask answers the machine holding the bus and its module's answer ("" when nothing holds the bus); list is the
|
||||
// record's word on the list sent to that machine.
|
||||
func busGrantBy(ask func(ctx context.Context) (string, json.RawMessage, error),
|
||||
list func(ctx context.Context) (busListState, error)) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
node, raw, askErr := ask(ctx)
|
||||
if askErr == nil && node == "" {
|
||||
return false, "no module holds the bus, so nothing says what it enforces", nil
|
||||
}
|
||||
var answer busAnswer
|
||||
if askErr == nil {
|
||||
if answer, askErr = readBusAnswer(raw); askErr != nil {
|
||||
askErr = fmt.Errorf("its answer could not be read: %w", askErr)
|
||||
}
|
||||
}
|
||||
if askErr != nil {
|
||||
if node == "" {
|
||||
node = "the bus's machine"
|
||||
}
|
||||
s, err := list(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the bus module on %s could not answer (%v), and whether the list sent "+
|
||||
"there is the one composed now could not be worked out: %w", node, askErr, err)
|
||||
}
|
||||
if s.Behind {
|
||||
return false, fmt.Sprintf("the bus module on %s could not answer (%v), and the user list sent there is "+
|
||||
"behind what the mesh composes; %s", node, askErr, s.remedy(node)), nil
|
||||
}
|
||||
return true, fmt.Sprintf("the bus module on %s could not answer (%v); judged by the user list sent there, "+
|
||||
"which is the one composed now", node, askErr), nil
|
||||
}
|
||||
notAllowed := func(older string) (bool, string, error) {
|
||||
s, err := list(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the user list on %s does not allow the controller to ask%s, and whether "+
|
||||
"the list sent there is the one composed now could not be worked out: %w", node, older, err)
|
||||
}
|
||||
if s.Behind {
|
||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s: it is behind "+
|
||||
"what the mesh composes; %s", node, older, s.remedy(node)), nil
|
||||
}
|
||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s, though the list "+
|
||||
"sent there is the one composed now; %s again", node, older, s.remedy(node)), nil
|
||||
}
|
||||
switch {
|
||||
case answer.InForce == nil && answer.Allowed:
|
||||
return true, fmt.Sprintf("the user list on %s allows the controller to ask, but its reload is not "+
|
||||
"confirmed: the bus module there is older than the answer whether a grant is in force, and the bus "+
|
||||
"upgrade (`bus upgrade`) carries one that confirms it", node), nil
|
||||
case answer.InForce == nil:
|
||||
return notAllowed(" (the bus module there is older than the answer whether a grant is in force)")
|
||||
case !answer.Allowed:
|
||||
return notAllowed("")
|
||||
case *answer.InForce:
|
||||
return true, "", nil
|
||||
case answer.FileWritten != "" && answer.ServerLoaded != "":
|
||||
return false, fmt.Sprintf("the bus on %s has the controller's grant to ask written (%s) but has not "+
|
||||
"reloaded since (last load %s); it reloads in place within about 5 s, so nothing is to be done but wait",
|
||||
node, answer.FileWritten, answer.ServerLoaded), nil
|
||||
}
|
||||
why := answer.Why
|
||||
if why == "" {
|
||||
why = "it gives no reason"
|
||||
}
|
||||
return false, fmt.Sprintf("the user list on %s allows the controller to ask, but whether the bus enforces it "+
|
||||
"is not known: %s", node, why), nil
|
||||
}
|
||||
}
|
||||
|
||||
// busUserCanTool is the bus module's answer to whether a user may do something, and whether it is in force.
|
||||
const busUserCanTool = "nats_user_can"
|
||||
|
||||
// busAnswer is the bus module's answer to busUserCanTool; InForce is nil from a module older than that field,
|
||||
// and FileWritten and ServerLoaded are given only when it could read both times (mesh-catalog's nats InForce).
|
||||
type busAnswer struct {
|
||||
Allowed bool `json:"allowed"`
|
||||
InForce *bool `json:"in_force"`
|
||||
Why string `json:"why"`
|
||||
FileWritten string `json:"file_written"`
|
||||
ServerLoaded string `json:"server_loaded"`
|
||||
}
|
||||
|
||||
func readBusAnswer(raw json.RawMessage) (busAnswer, error) {
|
||||
var answer busAnswer
|
||||
err := json.Unmarshal(raw, &answer)
|
||||
return answer, err
|
||||
}
|
||||
|
||||
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
||||
const grantLookEvery = 30 * time.Second
|
||||
|
||||
@@ -302,7 +546,19 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
a := &asker{
|
||||
a := servingAsker(open, conn, js, keeper)
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
|
||||
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
||||
// records, and the judgements it asks by — the bus's grant to ask among them (novox/hq issue 353).
|
||||
func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper *conditions.Keeper) *asker {
|
||||
return &asker{
|
||||
open: keeper.Open,
|
||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
||||
@@ -322,7 +578,7 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
||||
},
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open),
|
||||
grantHeld: grantHeldIn(open, conn),
|
||||
channels: channelsIn(open.inventory),
|
||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
||||
@@ -330,10 +586,4 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
||||
now: time.Now,
|
||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
}
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
|
||||
+285
-21
@@ -37,6 +37,11 @@ import (
|
||||
// own commit, the newest first, one walk each, until one is delivered; the older ones are then answered
|
||||
// by that one.
|
||||
//
|
||||
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions
|
||||
// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery
|
||||
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Walks stay one at
|
||||
// a time: such a batch is cut when no walk is open, and queued behind the one that is otherwise.
|
||||
//
|
||||
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
||||
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
||||
// no waiting catalogue walk, and a catalogue walk let go while another walk runs starts once it ended — so no
|
||||
@@ -173,6 +178,12 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
fmt.Printf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit)
|
||||
return nil
|
||||
}
|
||||
// **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk.
|
||||
own := ownPath(touches)
|
||||
waits, err := groupWait(ctx, inv, m, deliverySeatHeld(entries), own)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
@@ -198,7 +209,6 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
// **A merge on the controller's own path never shares a batch with one that waits for the delivery's
|
||||
// word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start
|
||||
// with the controller's and skip their turn. Each kind has a batch of its own.
|
||||
own := ownPath(touches)
|
||||
if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil {
|
||||
return notNow(err)
|
||||
} else if ok {
|
||||
@@ -218,7 +228,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
return cutBatchesHeld(ctx, f.open, now)
|
||||
}
|
||||
|
||||
batch, err := theOpenBatch(ctx, inv, heard, now, own)
|
||||
batch, why, err := batchToJoin(ctx, inv, heard, entries, read, now, own, waits)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
@@ -229,7 +239,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s); it joins %s — %s\n", repository, m.Base, m.Commit, batch.ID,
|
||||
fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID,
|
||||
batchWords(batch, now))
|
||||
return cutBatchesHeld(ctx, f.open, now)
|
||||
}
|
||||
@@ -320,22 +330,211 @@ func ownPath(touches []inventory.Entry) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// theOpenBatch is the batch a merge heard now joins: the one of its kind not yet cut, or a new one.
|
||||
func theOpenBatch(ctx context.Context, inv *inventory.Inventory, first inventory.BatchedMerge, now time.Time, own bool) (inventory.Plan, error) {
|
||||
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, as ADR 0306
|
||||
// decisions 2 and 3 read it). batchToJoin is the batch a merge heard now joins, and why in words:
|
||||
//
|
||||
// - an assembling or queued batch of its kind it is related to (related.go): it joins it and waits for the
|
||||
// window, which its coming restarts. Related to two or more, the newer are folded into the oldest, which
|
||||
// answers all their merges: a merge never waits in two windows, and what it shares with each walks in one.
|
||||
// - none, and its delivery group has a member still to merge, or mesh-delivery could not say (groupWait, waits
|
||||
// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say
|
||||
// why.
|
||||
// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at
|
||||
// once means only that: walks stay one at a time (ADR 0276 decision 4), so with a walk open the batch is
|
||||
// queued behind it as any closed batch is.
|
||||
func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) {
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
}
|
||||
joins, whys, _, err := relatedBatches(ctx, inv, []inventory.BatchedMerge{heard}, own, "", entries, read, edges)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
}
|
||||
if len(joins) > 0 {
|
||||
into := joins[0]
|
||||
why := "related to " + into.ID + " (" + whys[0] + ")"
|
||||
if len(joins) > 1 {
|
||||
plans, moves, err := foldInto(ctx, inv, &into, joins[1:])
|
||||
if err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
}
|
||||
if err := inv.SavePlansMoving(ctx, plans, moves); err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
}
|
||||
var folded []string
|
||||
for i, b := range joins[1:] {
|
||||
folded = append(folded, b.ID+" ("+whys[i+1]+")")
|
||||
}
|
||||
why += ", and to " + strings.Join(folded, ", ") + ", folded into it"
|
||||
}
|
||||
into.Delivery.Batch.CutAtOnce = false
|
||||
return into, why, nil
|
||||
}
|
||||
if waits != "" {
|
||||
// Its own window, beside the batches it shares nothing with: waiting is for its group's members, which
|
||||
// are related to it by their head and join it (ADR 0306 decision 2).
|
||||
b, err := newBatch(ctx, inv, heard, now, own, false, waits)
|
||||
return b, waits + ", it waits for the window in a batch of its own", err
|
||||
}
|
||||
b, err := newBatch(ctx, inv, heard, now, own, true, "")
|
||||
return b, "it shares nothing with a batch assembling and no member of a group is still to merge: it is cut at once", err
|
||||
}
|
||||
|
||||
// relatedBatches are the batches not yet cut of a kind, oldest first, parted into those related to merges (with
|
||||
// the first reason of each) and the others. Each batch is judged by what its cut would move (ADR 0306 decision
|
||||
// 3): its merges' moves as they were kept and as they read now — the catalogue may have moved since a merge was
|
||||
// heard — widened to their dependents, and, for a catalogue batch, every walk waiting for its word, which the
|
||||
// cut folds in (cutBatch's carry). A merge kept with no moves leaves its batch unknown, related to everything.
|
||||
// except is a walk not to count as folded: the one being stopped.
|
||||
func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inventory.BatchedMerge, own bool,
|
||||
except string, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge) (joins []inventory.Plan, whys []string, others []inventory.Plan, err error) {
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
var kind []inventory.Plan
|
||||
for _, b := range batches {
|
||||
if b.OwnPath() == own {
|
||||
return b, nil
|
||||
kind = append(kind, b)
|
||||
}
|
||||
}
|
||||
if len(kind) == 0 {
|
||||
return nil, nil, nil, nil
|
||||
}
|
||||
carry := newFootprint()
|
||||
if !own {
|
||||
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch may fold every one.
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
for _, p := range plans {
|
||||
if p.Release != nil || !p.Waiting() || p.ID == except {
|
||||
continue
|
||||
}
|
||||
kept, err := inv.MergesOf(ctx, p.ID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
carry = unionFootprint(carry, footprintOfWalk(p, kept))
|
||||
}
|
||||
}
|
||||
mine := footprintAsCut(merges, entries, read, edges)
|
||||
for _, b := range kind {
|
||||
held, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
if b.Delivery == nil {
|
||||
b.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
if b.Delivery.Batch == nil {
|
||||
b.Delivery.Batch = &inventory.PlanBatch{Own: own}
|
||||
}
|
||||
if ok, why := related(mine, unionFootprint(footprintAsCut(held, entries, read, edges), carry), edges); ok {
|
||||
joins, whys = append(joins, b), append(whys, why)
|
||||
} else {
|
||||
others = append(others, b)
|
||||
}
|
||||
}
|
||||
return joins, whys, others, nil
|
||||
}
|
||||
|
||||
// footprintAsCut is a batch's footprint as a cut now would read it: footprintOfBatch, widened by the modules each
|
||||
// merge moves as the catalogue reads now (planBatch judges each merge again at the cut, so what it moves may
|
||||
// have changed since it was heard). Both are kept: when in doubt, related.
|
||||
func footprintAsCut(merges []inventory.BatchedMerge, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, edges []inventory.Edge) footprint {
|
||||
f := footprintOfBatch(merges, edges)
|
||||
var moved []string
|
||||
for _, m := range merges {
|
||||
s := announcedOf(m).SourceMoved
|
||||
if s.Owner == "" {
|
||||
continue
|
||||
}
|
||||
s.MergedAt = "" // news, whatever a cut marked seen since (as planBatch reads it)
|
||||
for _, e := range wouldMove(s, entries, read) {
|
||||
moved = append(moved, e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
for _, name := range reachableFrom(moved, edges) {
|
||||
f.modules[name] = true
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// unionFootprint is what two changes reach together.
|
||||
func unionFootprint(a, b footprint) footprint {
|
||||
f := newFootprint()
|
||||
for _, from := range []footprint{a, b} {
|
||||
for k := range from.repos {
|
||||
f.repos[k] = true
|
||||
}
|
||||
for k := range from.modules {
|
||||
f.modules[k] = true
|
||||
}
|
||||
for k := range from.heads {
|
||||
f.heads[k] = true
|
||||
}
|
||||
}
|
||||
f.unknown = a.unknown
|
||||
if f.unknown == "" {
|
||||
f.unknown = b.unknown
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// foldInto folds batches into the oldest one related to a merge: the plans to write (into first, so no merge
|
||||
// names a plan the store does not hold) and the merges to move, for one SavePlansMoving. The folded batches name
|
||||
// the batch that took them over; a stop's closed window and its withheld modules (ADR 0299) go with their merges.
|
||||
func foldInto(ctx context.Context, inv *inventory.Inventory, into *inventory.Plan, from []inventory.Plan) ([]*inventory.Plan,
|
||||
[]inventory.MergeMove, error) {
|
||||
if into.Delivery == nil {
|
||||
into.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
if into.Delivery.Batch == nil {
|
||||
into.Delivery.Batch = &inventory.PlanBatch{Own: into.OwnPath()}
|
||||
}
|
||||
plans := []*inventory.Plan{into}
|
||||
var moves []inventory.MergeMove
|
||||
for i := range from {
|
||||
f := from[i]
|
||||
merges, err := inv.MergesOf(ctx, f.ID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, m := range merges {
|
||||
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: into.ID, Alone: m.Alone})
|
||||
}
|
||||
if f.Delivery == nil {
|
||||
f.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
if w := f.Delivery.Batch; w != nil {
|
||||
into.Delivery.Batch.Closed = into.Delivery.Batch.Closed || w.Closed
|
||||
into.Delivery.Batch.Withheld = union(into.Delivery.Batch.Withheld, w.Withheld)
|
||||
sort.Strings(into.Delivery.Batch.Withheld)
|
||||
}
|
||||
f.State = inventory.PlanSuperseded
|
||||
f.Delivery.Batch = nil
|
||||
f.Delivery.TakenOverBy = into.ID
|
||||
f.Note = fmt.Sprintf("folded into %s before it was cut: a merge related to both joined it, and %s answers "+
|
||||
"its merges", into.ID, into.ID)
|
||||
plans = append(plans, &f)
|
||||
}
|
||||
return plans, moves, nil
|
||||
}
|
||||
|
||||
// newBatch is a new batch of a kind, kept before its first merge names it: a merge never names a plan the store
|
||||
// does not hold. cutAtOnce closes its window at once; waits says why it waits for its group, when it does.
|
||||
func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.BatchedMerge, now time.Time, own,
|
||||
cutAtOnce bool, waits string) (inventory.Plan, error) {
|
||||
b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
||||
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
||||
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}
|
||||
// Kept before its first merge names it: a merge never names a plan the store does not hold.
|
||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own, CutAtOnce: cutAtOnce, Waits: waits}}}
|
||||
if err := inv.SavePlan(ctx, &b); err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
@@ -367,13 +566,16 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan,
|
||||
}
|
||||
closes, latest := windowOf(merges, b.Created, window, atMost)
|
||||
b.Delivery.Merges = named
|
||||
var closed bool
|
||||
var closed, cutAtOnce bool
|
||||
var withheld []string
|
||||
var waits string
|
||||
if was := b.Delivery.Batch; was != nil {
|
||||
closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299)
|
||||
cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1)
|
||||
waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2)
|
||||
}
|
||||
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
|
||||
Closed: closed, Withheld: withheld}
|
||||
Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits}
|
||||
b.State = inventory.PlanAssembling
|
||||
if behind != "" && windowClosed(*b, now) {
|
||||
b.State = inventory.PlanQueued
|
||||
@@ -402,13 +604,14 @@ func windowOf(merges []inventory.BatchedMerge, created time.Time, window, atMost
|
||||
return last.Add(window), created.Add(atMost)
|
||||
}
|
||||
|
||||
// windowClosed says a batch's window is closed: no merge for the window's length, or its maximum reached.
|
||||
// windowClosed says a batch's window is closed: closed by a stop, closed at once for a merge that shares
|
||||
// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached.
|
||||
func windowClosed(b inventory.Plan, now time.Time) bool {
|
||||
if b.Delivery == nil || b.Delivery.Batch == nil {
|
||||
return true
|
||||
}
|
||||
w := b.Delivery.Batch
|
||||
return w.Closed || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
||||
return w.Closed || w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
||||
}
|
||||
|
||||
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
|
||||
@@ -485,7 +688,7 @@ func spelledOf(m inventory.BatchedMerge) string {
|
||||
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
||||
k := announcedOf(m)
|
||||
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
||||
Merged: m.Merged, Heard: m.Heard}
|
||||
Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard}
|
||||
}
|
||||
|
||||
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
||||
@@ -617,8 +820,9 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
return err
|
||||
}
|
||||
// The oldest batch whose window closed is cut, an own-path one before a catalogue one (it goes first, and
|
||||
// the catalogue batch queues behind it rather than being cut and overtaken); one of each kind at most is
|
||||
// open (theOpenBatch).
|
||||
// the catalogue batch queues behind it rather than being cut and overtaken). Several of a kind may be open
|
||||
// since a merge that shares nothing is cut at once (ADR 0304 decision 1): walks stay one at a time, so one
|
||||
// is cut per look, and once its walk started the others are queued behind it.
|
||||
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
||||
for i := range batches {
|
||||
b := &batches[i]
|
||||
@@ -735,11 +939,16 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
commits, named := carriedOf(merges)
|
||||
var moved []string
|
||||
var members []batchMember
|
||||
unchanged := map[string]string{}
|
||||
for _, m := range combinedMerges(merges) {
|
||||
// **News, whatever was seen since** (review of this change): each merge was judged when it was heard, and
|
||||
// a cut, a fold or a failed walk since marks its modules seen — so a merge walked alone after a failed
|
||||
// walk, or folded, would read as history and move nothing. When it was made is not asked again here.
|
||||
m.MergedAt = ""
|
||||
// **Unchanged, said** (novox/hq issue 611): read before the merge is acted on, from the same records.
|
||||
for n, why := range unchangedOfMerge(m, entries, read) {
|
||||
unchanged[n] = why
|
||||
}
|
||||
names, err := movesOfMerge(ctx, inv, m, entries, read)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -791,12 +1000,21 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
plan.Delivery.Merges = named
|
||||
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
||||
plan.Delivery.Withheld = withheld
|
||||
// **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as
|
||||
// the build the mesh holds.
|
||||
for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) {
|
||||
unchanged[n] = why
|
||||
}
|
||||
plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules)
|
||||
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
||||
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
||||
if len(moved) == 0 {
|
||||
if len(moved) == 0 || len(plan.Modules) == 0 {
|
||||
plan.State = inventory.PlanDone
|
||||
plan.Tiers = [][]string{}
|
||||
plan.Note = "moved nothing the mesh holds: nothing to walk"
|
||||
if len(plan.Delivery.Unchanged) > 0 {
|
||||
plan.Note = "moved nothing the mesh holds: nothing to walk; " + unchangedSaid(plan.Delivery.Unchanged)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -831,7 +1049,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
return nil
|
||||
}
|
||||
// Written and its first tier asked as one act on the plans (novox/hq issue 213), under the caller's hold.
|
||||
if err := askTier(ctx, inv, &plan); err != nil {
|
||||
if err := askTier(ctx, inv, &plan, plan.Tiers[plan.Tier]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
@@ -841,6 +1059,31 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
return nil
|
||||
}
|
||||
|
||||
// unchangedSaid names the modules a walk says matched what runs, for its note.
|
||||
func unchangedSaid(unchanged map[string]string) string {
|
||||
names := make([]string, 0, len(unchanged))
|
||||
for n := range unchanged {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return "unchanged, the build the mesh holds is theirs: " + strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// unchangedLeft is what a walk says matched what runs: every module a merge left unchanged that the walk does not
|
||||
// build anyway, as another merge's module or a dependent (novox/hq issue 611). Nil when none is.
|
||||
func unchangedLeft(unchanged map[string]string, builds map[string]*inventory.PlanModule) map[string]string {
|
||||
out := map[string]string{}
|
||||
for n, why := range unchanged {
|
||||
if _, built := builds[n]; !built {
|
||||
out[n] = why
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the
|
||||
// modules taken.
|
||||
func withhold(plan *inventory.Plan, names, moved []string) []string {
|
||||
@@ -886,6 +1129,9 @@ func walkTimesAtCut(batch, walk inventory.Plan, entries []inventory.Entry, now t
|
||||
if !w.AtMost.IsZero() && w.AtMost.Before(closed) {
|
||||
closed = w.AtMost
|
||||
}
|
||||
if w.CutAtOnce && !batch.Created.IsZero() {
|
||||
closed = batch.Created // closed when it was made (ADR 0304 decision 1)
|
||||
}
|
||||
if !closed.IsZero() {
|
||||
if closed.After(now) {
|
||||
closed = now
|
||||
@@ -902,6 +1148,16 @@ const resolverSeat = "mesh-dns-resolver"
|
||||
// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own
|
||||
// path or one holding the mesh's resolver, leaf otherwise.
|
||||
func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
||||
modules := make([]string, 0, len(walk.Modules))
|
||||
for m := range walk.Modules {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
return classOfModules(modules, entries)
|
||||
}
|
||||
|
||||
// classOfModules is the class of modules moved together: core when one is on the controller's own path or holds
|
||||
// the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
|
||||
func classOfModules(modules []string, entries []inventory.Entry) string {
|
||||
resolvers := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
@@ -910,7 +1166,7 @@ func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
for m := range walk.Modules {
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
||||
return inventory.ClassCore
|
||||
}
|
||||
@@ -1142,6 +1398,10 @@ func batchWords(b inventory.Plan, now time.Time) string {
|
||||
if b.State == inventory.PlanQueued {
|
||||
return fmt.Sprintf("queued behind %s; grouped: %s; plan not yet calculated", w.Behind, grouped)
|
||||
}
|
||||
if w.CutAtOnce {
|
||||
return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated",
|
||||
grouped)
|
||||
}
|
||||
left := w.ClosesAt.Sub(now)
|
||||
if w.AtMost.Before(w.ClosesAt) {
|
||||
left = w.AtMost.Sub(now)
|
||||
@@ -1149,8 +1409,12 @@ func batchWords(b inventory.Plan, now time.Time) string {
|
||||
if left < 0 {
|
||||
left = 0
|
||||
}
|
||||
return fmt.Sprintf("assembling: %s left (at the latest %s); grouped: %s; plan not yet calculated",
|
||||
secondsWords(left), w.AtMost.Local().Format("15:04:05"), grouped)
|
||||
waits := ""
|
||||
if w.Waits != "" {
|
||||
waits = " — " + w.Waits
|
||||
}
|
||||
return fmt.Sprintf("assembling: %s left (at the latest %s)%s; grouped: %s; plan not yet calculated",
|
||||
secondsWords(left), w.AtMost.Local().Format("15:04:05"), waits, grouped)
|
||||
}
|
||||
|
||||
// groupedWords is a batch's commits, each with the earlier merges it answers.
|
||||
|
||||
@@ -34,6 +34,7 @@ func windowed(t *testing.T) *stores {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
noGroups(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
@@ -58,19 +59,70 @@ func windowed(t *testing.T) *stores {
|
||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment.
|
||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull
|
||||
// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group
|
||||
// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test
|
||||
// whose merges wait for the window as a group's says so (groupsWaiting).
|
||||
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
||||
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
||||
}
|
||||
|
||||
// repoMerge is a merge of a repository of one module's own.
|
||||
// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge).
|
||||
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
||||
}
|
||||
|
||||
// noGroups stubs mesh-delivery's word on groups for a test: no head names a group.
|
||||
func noGroups(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil }
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// groupsWaiting stubs mesh-delivery's word on groups for a test: each head named is a group with a member of another
|
||||
// repository whose pull request is open (novox/elsewhere, ready), so a merge from it waits for the window (ADR 0306
|
||||
// decision 2); every other head names no group.
|
||||
func groupsWaiting(t *testing.T, heads ...string) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
|
||||
if slices.Contains(heads, head) {
|
||||
return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
|
||||
// before novox/hq ADR 0304: every head is a group with a member of another repository still to merge.
|
||||
func everyHeadWaits(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group,
|
||||
// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share.
|
||||
func fromHead(m link.SourceMoved, head string) link.SourceMoved {
|
||||
m.Head = head
|
||||
return m
|
||||
}
|
||||
|
||||
// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is
|
||||
// related to a batch assembling (ADR 0304 decision 1).
|
||||
func lone(m link.SourceMoved) link.SourceMoved {
|
||||
m.Head = ""
|
||||
return m
|
||||
}
|
||||
|
||||
// hear is a merge heard at a moment.
|
||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||
t.Helper()
|
||||
@@ -108,6 +160,7 @@ func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
||||
// later commit, which names both merges — the earlier carried by the later.
|
||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
@@ -152,9 +205,10 @@ func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
// Merges of three repositories in one window are one walk, one commit for each.
|
||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/three")
|
||||
asksRecorded(t)
|
||||
for i, r := range []string{"one", "two", "three"} {
|
||||
hear(t, open, repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)),
|
||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||
@@ -187,9 +241,10 @@ func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||
// closes at most ten minutes after its first merge however busy.
|
||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(80*time.Second)), t0.Add(80*time.Second))
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||
@@ -200,12 +255,13 @@ func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
}
|
||||
|
||||
busy := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
var last time.Time
|
||||
for i := 0; i < 12; i++ {
|
||||
last = t0.Add(time.Duration(i) * time.Minute)
|
||||
repo := []string{"one", "two", "three"}[i%3]
|
||||
hear(t, busy, repoMerge(repo, "c"+string(rune('a'+i)), last), last)
|
||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||
if i != 10 {
|
||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||
@@ -223,10 +279,11 @@ func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/553b7191claude")
|
||||
asksWithPaths(t)
|
||||
var said []inventory.Plan
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(p inventory.Plan) { said = append(said, p) }
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
@@ -235,7 +292,7 @@ func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||
hear(t, open, claude, now.Add(-19*time.Second))
|
||||
hear(t, open, dunst, now.Add(-time.Second))
|
||||
hear(t, open, repoMerge("one", "a6bc0931one", now), now)
|
||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||
lines := strings.Split(out, "\n")
|
||||
first := lines[0]
|
||||
@@ -360,6 +417,7 @@ func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
@@ -391,6 +449,7 @@ func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
// first; the first delivered answers the older ones, and the search stops.
|
||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||
@@ -459,6 +518,7 @@ func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||
for _, grouped := range []bool{true, false} {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"mesh-host"} {
|
||||
@@ -496,9 +556,10 @@ func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
||||
func TestABatchSurvivesARestart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
hear(t, open, repoMerge("two", "c2", t0.Add(30*time.Second)), t0.Add(30*time.Second))
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
||||
|
||||
// A new controller: nothing of the first one's but the store.
|
||||
again, err := openStores(t.Context())
|
||||
@@ -506,7 +567,7 @@ func TestABatchSurvivesARestart(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(again.Close)
|
||||
hear(t, again, repoMerge("one", "c1", t0), t0.Add(60*time.Second)) // the bus hands it over again
|
||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
||||
cutAt(t, again, t0.Add(119*time.Second))
|
||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||
@@ -563,6 +624,7 @@ func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
||||
// nothing, and the merge joins the next batch.
|
||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||
@@ -696,7 +758,7 @@ func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||
return entries, read, err
|
||||
}
|
||||
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
||||
(following{open}).SourceMoved, t.Logf); err != nil {
|
||||
(following{open: open}).SourceMoved, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
@@ -716,6 +778,7 @@ func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
||||
// still waits for the word, so no catalogue delivery skips its turn.
|
||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
@@ -860,6 +923,7 @@ func sameMerge(a, b inventory.PlanMerge) bool {
|
||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []struct {
|
||||
|
||||
@@ -48,6 +48,10 @@ func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time)
|
||||
if f.atMost.Before(f.closed) {
|
||||
f.closed = f.atMost
|
||||
}
|
||||
if w.Batch.CutAtOnce && !b.Created.IsZero() {
|
||||
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
|
||||
f.atMost, f.closed = b.Created, b.Created
|
||||
}
|
||||
}
|
||||
if f.behind != "" {
|
||||
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -182,8 +183,10 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||
// The module name comes from the manifest, which only exists when the build got that far.
|
||||
if len(kept.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
|
||||
kept.Module = m.Module
|
||||
if list, err := judgedWith(result); err == nil {
|
||||
if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil {
|
||||
kept.Module = m.Module
|
||||
}
|
||||
}
|
||||
}
|
||||
return kept
|
||||
@@ -483,6 +486,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
Seats: seatBases(ctx),
|
||||
DryRun: dryRun,
|
||||
}
|
||||
request.Reuse = reuseAsked(ctx, dryRun)
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
fmt.Printf(" (%s)", repository)
|
||||
@@ -601,6 +605,11 @@ func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module
|
||||
// written with the same values.
|
||||
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
|
||||
catalogue.Manifest, inventory.Build, error) {
|
||||
// The list of predefined keys as the store keeps it now: another controller process may have kept a newer one.
|
||||
if err := loadPredefined(ctx, inv); err != nil {
|
||||
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+
|
||||
"read: %w", err)
|
||||
}
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
@@ -611,7 +620,15 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
||||
result.On, result.Repository, result.Failed)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||
// Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq
|
||||
// ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the
|
||||
// mesh publishes, before it is registered.
|
||||
list, err := judgedWith(result)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
result.Repository, short(result.Commit), err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifestWith(kept.Manifest, list)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
@@ -669,6 +686,12 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
|
||||
// registered only as the walk's copy in the module's own namespace.
|
||||
if scratch := inScratch(result.Made); scratch != "" {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+
|
||||
"namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch)
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
@@ -679,6 +702,23 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
|
||||
short(result.Commit))
|
||||
}
|
||||
// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
|
||||
// catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module
|
||||
// is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list
|
||||
// does not hold, or of another type, is refused here and never enters the catalogue.
|
||||
if carried, err := carriedList(result); err != nil {
|
||||
return manifest, kept, err
|
||||
} else if carried != nil {
|
||||
if err := publishPredefined(ctx, inv, result, carried); err != nil {
|
||||
fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
if manifest.NamesPredefined() {
|
||||
if _, err := catalogue.ParseManifest(kept.Manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+
|
||||
"list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
}
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
||||
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
||||
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
||||
@@ -714,6 +754,16 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, nil
|
||||
}
|
||||
|
||||
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
|
||||
func inScratch(made []link.MadeArtifact) string {
|
||||
for _, m := range made {
|
||||
if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) {
|
||||
return m.Reference
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
||||
@@ -802,6 +852,9 @@ type answers struct {
|
||||
sources map[string]inventory.Source
|
||||
// waiting is every machine not running what the mesh would send it.
|
||||
waiting []inventory.Machine
|
||||
// recordedWaits is, per machine, every move of a recorded module that waits there for a person (novox/hq ADR
|
||||
// 0377 §6): the machine is behind for these, and `push <node>` sends them.
|
||||
recordedWaits map[string][]string
|
||||
// reported is every machine's last word beside when it was last sent a declaration — the
|
||||
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
||||
// recorded at send, not at apply).
|
||||
@@ -881,6 +934,34 @@ type answers struct {
|
||||
//
|
||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||
// runningBy is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module, as a
|
||||
// builder reaches the artifact store (novox/hq issue 721): what a check compares its builds with. Empty when it cannot
|
||||
// be read, said, and then nothing is said unchanged.
|
||||
func runningBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
||||
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
running, err := open.inventory.RegisteredHeld(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
||||
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
if err != nil || address == "" {
|
||||
return running
|
||||
}
|
||||
routed := make(map[string]string, len(running))
|
||||
for key, reference := range running {
|
||||
routed[key] = catalogue.Rerouted(reference, address)
|
||||
}
|
||||
return routed
|
||||
}
|
||||
|
||||
func heldBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Every user's slow consumer is S9's, not the controller's alone** (novox/hq ADR 0357, issue 270). On
|
||||
// 2026-10-07 the bus closed two clients on the laptop as slow consumers and nothing raised a condition:
|
||||
// the controller heard only its own connection. Said by the bus module and read back, a user the bus
|
||||
// closes is one condition naming the module and its machine, raised at the first look and cleared an hour
|
||||
// after the last closure.
|
||||
func TestAUserTheBusClosedAsASlowConsumerIsOneConditionNamingItsModuleAndMachine(t *testing.T) {
|
||||
closed := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
now := closed.Add(20 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
advisories := link.BusClosures([]link.BusClosure{
|
||||
{User: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed.Add(-10 * time.Minute)},
|
||||
{User: "laptop.claude-code", CID: 44, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Minute),
|
||||
Closed: closed},
|
||||
{User: "anchor.desk", CID: 7, Reason: "Maximum Message Payload Exceeded", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed},
|
||||
}, []string{link.AdvisorySlowConsumer, link.AdvisorySlowConsumer, link.AdvisoryConnectionClosed}, closed.Add(-time.Hour))
|
||||
look := func() {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "anchor", advisories: advisories}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
look()
|
||||
|
||||
slow, found, err := conditions.ReadOne(t.Context(), store, "bus.laptop.claude-code.slow-consumer")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("no slow-consumer condition for the user: %v %v", found, err)
|
||||
}
|
||||
if slow.Subject.Machine != "laptop" || slow.Observations != 2 || !slow.LastObserved.Equal(closed) {
|
||||
t.Fatalf("the slow consumer is said on %q, %d closure(s), last %s: %+v", slow.Subject.Machine, slow.Observations,
|
||||
slow.LastObserved, slow)
|
||||
}
|
||||
if !strings.Contains(slow.Summary, "claude-code on laptop") || strings.Contains(slow.Summary, "laptop.claude-code") {
|
||||
t.Fatalf("the summary does not name the module in words, or carries the dotted user: %q", slow.Summary)
|
||||
}
|
||||
if !strings.Contains(slow.Headline, "Claude-code on laptop") {
|
||||
t.Fatalf("the headline is %q", slow.Headline)
|
||||
}
|
||||
cut, found, err := conditions.ReadOne(t.Context(), store, "bus.anchor.desk.connection-closed")
|
||||
if err != nil || !found || cut.Subject.Machine != "anchor" || !strings.Contains(cut.Summary, "maximum message payload exceeded") {
|
||||
t.Fatalf("the user cut off for breaking the bus's rules: %v %v %+v", found, err, cut)
|
||||
}
|
||||
|
||||
// An hour after the newest closure without another, both clear.
|
||||
now = closed.Add(time.Hour + time.Minute)
|
||||
look()
|
||||
for _, key := range []string{"bus.laptop.claude-code.slow-consumer", "bus.anchor.desk.connection-closed"} {
|
||||
if _, found, _ := k.Get(t.Context(), key); found {
|
||||
t.Fatalf("%s is still open an hour after the last closure", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A read of the bus module's events that fails does not blind S9** (review of #326): the controller's own
|
||||
// signals are still said, and the failed read is a condition of its own.
|
||||
func TestABusModuleReadThatFailsKeepsTheControllersOwnSignals(t *testing.T) {
|
||||
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at.Add(time.Minute), host: "anchor",
|
||||
advisories: []link.Advisory{{Kind: link.AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind",
|
||||
First: at, Last: at, Count: 1}},
|
||||
busFaultsErr: errors.New("nats: timeout")}
|
||||
keys := map[string]bool{}
|
||||
for _, o := range watchAdvisories(f) {
|
||||
keys[o.Key()] = true
|
||||
}
|
||||
if !keys["bus.controller.slow-consumer"] || !keys["bus.bus-module.bus-faults-unread"] {
|
||||
t.Fatalf("observed %v: the controller's own slow consumer and the unread events, both", keys)
|
||||
}
|
||||
for _, row := range signalsTable {
|
||||
if row.Row == "S9" && row.needs(f) != nil {
|
||||
t.Fatal("the S9 row went blind for a read of the bus module's events")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -49,10 +49,12 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
consumers, records, err := moduleConsumersAndRecords(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And a grant of a restricted stream that composes nothing, said on every raise and send (novox/hq ADR 0376 §2).
|
||||
sayWithheld(records, " ")
|
||||
// And the work queues of seats that name their caller or their kind, with each holder's worker
|
||||
// (novox/hq ADR 0259 §3): an ask queues until the router takes it, a channel's work until that kind
|
||||
// takes it.
|
||||
@@ -60,9 +62,19 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And every module's own stream (novox/hq ADR 0363), from the catalogue, before the consumers on it.
|
||||
modStreams, err := inv.DeclaredStreams(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, s := range broker.ModuleStreams(modStreams) {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the module stream %s: %w", s.Name, err))
|
||||
}
|
||||
}
|
||||
for _, s := range trafficStreams {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
||||
@@ -73,11 +85,30 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
||||
}
|
||||
}
|
||||
restricted := map[string]bool{}
|
||||
for _, s := range modStreams {
|
||||
restricted[s.Name] = s.Restricted
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
if err := broker.EnsureHeard(r, c.Consumer, restricted[c.Consumer.Stream]); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
}
|
||||
}
|
||||
// **And a consumer no longer granted loses what it reads through** (novox/hq ADR 0363 §6): every consumer
|
||||
// on a module's stream the composition no longer asks for is removed, on this very send.
|
||||
if p, prunes := r.(broker.Pruner); prunes && len(modStreams) > 0 {
|
||||
var wanted []broker.Consumer
|
||||
for _, c := range consumers {
|
||||
wanted = append(wanted, c.Consumer)
|
||||
}
|
||||
removed, err := broker.PruneHeard(p, modStreams, wanted)
|
||||
if err != nil {
|
||||
failed = append(failed, err)
|
||||
}
|
||||
for _, gone := range removed {
|
||||
fmt.Printf(" removed the consumer %s: its module is no longer granted that stream (novox/hq ADR 0363)\n", gone)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return nil, errors.Join(failed...)
|
||||
}
|
||||
@@ -136,15 +167,21 @@ func unassertedObservation(err error) conditions.Observation {
|
||||
|
||||
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||
consumers, _, err := moduleConsumersAndRecords(ctx, inv)
|
||||
return consumers, err
|
||||
}
|
||||
|
||||
// moduleConsumersAndRecords is moduleConsumers with the records they were composed from.
|
||||
func moduleConsumersAndRecords(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, broker.Records, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, broker.Records{}, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, broker.Records{}, err
|
||||
}
|
||||
return broker.ConsumersOf(users), nil
|
||||
return broker.ConsumersOf(users), records, nil
|
||||
}
|
||||
|
||||
// seatTrafficObjects is the work queues and workers of seats that name their caller or their kind, from
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
||||
//
|
||||
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
||||
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
||||
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
||||
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
||||
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
||||
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
||||
// its gate starts as before.
|
||||
|
||||
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
||||
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
||||
var busyGateSlack = 5 * time.Minute
|
||||
|
||||
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
|
||||
// controller's clock from when it first read the wait.
|
||||
const busyEngineBound = 15 * time.Minute
|
||||
|
||||
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
|
||||
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
|
||||
// machine nor a wrong word from it holds a walk longer.
|
||||
func busyGateEnds(b *inventory.GateBusy) time.Time {
|
||||
until := b.Seen.Add(busyEngineBound)
|
||||
if !b.Until.IsZero() && b.Until.Before(until) {
|
||||
until = b.Until
|
||||
}
|
||||
return until.Add(busyGateSlack)
|
||||
}
|
||||
|
||||
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
||||
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
||||
// gate's Since to its end. Pure but for the gate it is handed.
|
||||
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
||||
if g.Busy != nil && g.Busy.Ended != nil {
|
||||
return false
|
||||
}
|
||||
for _, j := range pairs {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
continue
|
||||
}
|
||||
if g.Busy == nil {
|
||||
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
|
||||
Seen: now}
|
||||
}
|
||||
if g.Busy.Seen.IsZero() {
|
||||
g.Busy.Seen = now
|
||||
}
|
||||
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
||||
if now.After(busyGateEnds(g.Busy)) {
|
||||
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
||||
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
||||
end := now
|
||||
g.Busy.Ended, g.Busy.PastBound = &end, true
|
||||
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
|
||||
return false
|
||||
}
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
||||
g.Read(now, false, g.Busy.Words())
|
||||
return true
|
||||
}
|
||||
if g.Busy == nil {
|
||||
return false
|
||||
}
|
||||
// Replaced: the judging starts at the report that said so.
|
||||
end := now
|
||||
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
||||
end = r.At.UTC()
|
||||
}
|
||||
g.Busy.Ended = &end
|
||||
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
|
||||
return false
|
||||
}
|
||||
|
||||
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
||||
// running because it said busy.
|
||||
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
||||
r, said := f.reports[j.node]
|
||||
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
||||
return inventory.BusyWait{}, false
|
||||
}
|
||||
return inventory.WaitingFor(r.Busy, j.module)
|
||||
}
|
||||
|
||||
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
||||
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
||||
func busyStill(j judged, f gateFacts) (string, bool) {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
||||
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
||||
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
||||
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
||||
reported := sent.Add(20 * time.Second)
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
||||
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
||||
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
||||
// As judgeMoves hands it: the report held against the gate's own send.
|
||||
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
||||
}
|
||||
|
||||
now := reported.Add(time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
||||
t.Fatal("a gate whose module's container waits as busy judged it")
|
||||
}
|
||||
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
||||
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
||||
}
|
||||
for _, said := range []string{gateLine(g), g.Last} {
|
||||
if !strings.Contains(said, "waits for a busy container on novox") {
|
||||
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(g.Last, "check of #280") {
|
||||
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
||||
}
|
||||
|
||||
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
||||
now = reported.Add(10 * time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
||||
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
||||
}
|
||||
|
||||
// The machine reports it replaced: the judging starts at that report.
|
||||
replacedAt := reported.Add(12 * time.Minute)
|
||||
now = replacedAt.Add(5 * time.Second)
|
||||
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
||||
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
||||
t.Fatal("the gate still waits after the container was replaced")
|
||||
}
|
||||
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
||||
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
||||
}
|
||||
if strings.Contains(gateLine(g), "waits for a busy container") {
|
||||
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
||||
}
|
||||
// And a later report listing it waiting again does not hold a gate that started.
|
||||
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
||||
t.Error("a gate that started waits again")
|
||||
}
|
||||
}
|
||||
|
||||
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
||||
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
||||
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &sent, Current: true, Busy: waiting}}}
|
||||
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
||||
if busyWait(g, pairs, f, now) {
|
||||
t.Fatal("a gate waited past the engine's bound and the slack")
|
||||
}
|
||||
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
||||
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
||||
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
||||
// report not about the gate's send says nothing of it.
|
||||
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
||||
pairs := []judged{{module: "build-agent", node: "ace"}}
|
||||
at := sent.Add(time.Minute)
|
||||
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
||||
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
||||
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
||||
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
||||
if busyWait(g, pairs, elsewhere, at) {
|
||||
t.Fatal("a wait reported about an earlier declaration held the gate")
|
||||
}
|
||||
}
|
||||
|
||||
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
|
||||
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
|
||||
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
|
||||
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
ahead := seen.Add(time.Hour)
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
|
||||
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
|
||||
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
|
||||
}
|
||||
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
|
||||
t.Fatal("the gate stopped waiting within the bound on its own clock")
|
||||
}
|
||||
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
|
||||
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
|
||||
}
|
||||
}
|
||||
|
||||
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
|
||||
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
|
||||
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
|
||||
{"id": "agent-env", "type": "file"}}}}
|
||||
lines := busyNotSent(modules, false)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
|
||||
!strings.Contains(lines[0], "replaced at once") {
|
||||
t.Fatalf("the plan does not say the busy not sent: %v", lines)
|
||||
}
|
||||
if lines := busyNotSent(modules, true); len(lines) != 0 {
|
||||
t.Fatalf("a busy sent is said as not sent: %v", lines)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -72,6 +73,11 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
|
||||
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
|
||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
||||
case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData):
|
||||
// Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377).
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+
|
||||
"its move recreates none of its containers and moves none of its data, which the walk judges and then sends",
|
||||
name, orNone(u.From)))
|
||||
case waits && len(e.On) > 0:
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
||||
orNone(u.From)))
|
||||
@@ -105,9 +111,32 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
p.Machines = append(p.Machines, *machines[name])
|
||||
}
|
||||
p.Summary = summaryOf(p)
|
||||
p.Class = changeClassOf(p, entries)
|
||||
return p
|
||||
}
|
||||
|
||||
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
|
||||
const classNone = "none"
|
||||
|
||||
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
|
||||
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
|
||||
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
|
||||
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
|
||||
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
|
||||
// resolver claim its manifest makes.
|
||||
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
|
||||
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
|
||||
return classNone
|
||||
}
|
||||
modules := append(append([]string{}, p.Moved...), p.Dependents...)
|
||||
for _, dir := range p.New {
|
||||
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
|
||||
modules = append(modules, name)
|
||||
}
|
||||
}
|
||||
return classOfModules(modules, entries)
|
||||
}
|
||||
|
||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
||||
const grantsStepWord = "grants step"
|
||||
|
||||
@@ -47,6 +47,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
shelf := catalogue.Shelf{}
|
||||
faulted := map[string]bool{}
|
||||
failed := 0
|
||||
// The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read
|
||||
// once per repository: a manifest naming a key is judged against the list its own tree holds.
|
||||
lists := map[string]catalogue.PredefinedList{}
|
||||
listOf := func(path string) (catalogue.PredefinedList, error) {
|
||||
root := catalogue.RepositoryRoot(filepath.Dir(path))
|
||||
if root == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if list, read := lists[root]; read {
|
||||
return list, nil
|
||||
}
|
||||
list, err := catalogue.ReadPredefined(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lists[root] = list
|
||||
return list, nil
|
||||
}
|
||||
for _, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -54,7 +72,18 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
list, err := listOf(path)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.ParseManifestWith(raw, list)
|
||||
if err == nil {
|
||||
// Each template's text read from the tree beside the manifest, and judged as inline content is
|
||||
// (novox/hq ADR 0349 §2): a missing template is refused by name.
|
||||
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
@@ -75,6 +104,16 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
// A module that declares a setting names the keys of its mergeable files (novox/hq ADR 0326 §3,
|
||||
// issue 574): judged where a definition is written, as at registration, never where a stored
|
||||
// build is read again.
|
||||
if wrong := catalogue.WrittenSettingsProblems(m); len(wrong) > 0 {
|
||||
for _, p := range wrong {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
@@ -205,6 +244,12 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
// And its own runtime's, where it runs as its own account (novox/hq ADR 0366).
|
||||
if own, has := catalogue.OwnRuntimeHealth(m); has {
|
||||
if h, _, _ := catalogue.ReadHealth(own); h.Kind != "" {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", own["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
if len(checks) > 0 {
|
||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact
|
||||
// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the
|
||||
// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which
|
||||
// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store,
|
||||
// as the controller's own collection does. The build seat's holders let go of anything a week old on their own,
|
||||
// so a word lost here leaves a week's builds, never more.
|
||||
|
||||
// checkBuildsIn is the scratch namespace as the controller reaches the artifact store.
|
||||
func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if address == "" {
|
||||
return nil, errors.New("the artifact store is not on the mesh's network")
|
||||
}
|
||||
return builder.Registry{Address: address}, nil
|
||||
}
|
||||
|
||||
// forgetChecks lets go of a pull request's check builds and answers what it did.
|
||||
var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||
scratch, err := checkBuildsIn(ctx, inv)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
n, err := scratch.Forget(ctx, pull)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err)
|
||||
}
|
||||
return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil
|
||||
}
|
||||
|
||||
// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge
|
||||
// it answers: the walk took what it reuses. Said, never fatal.
|
||||
func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) {
|
||||
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
|
||||
return
|
||||
}
|
||||
for _, pull := range reuseAskOf(&p).Pulls {
|
||||
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, said)
|
||||
}
|
||||
}
|
||||
|
||||
// forgetChecksCommand is `forget-checks <owner> <repo> <number>`: the verb the delivery's owner asks when the forge
|
||||
// says a pull request closed unmerged (novox/hq ADR 0350 §5).
|
||||
func forgetChecksCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 3 {
|
||||
return errors.New("forget-checks <owner> <repo> <number>")
|
||||
}
|
||||
number, err := strconv.Atoi(args[2])
|
||||
if err != nil || number <= 0 {
|
||||
return fmt.Errorf("%q is not a pull request's number", args[2])
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in.
|
||||
|
||||
// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the
|
||||
// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one
|
||||
// that reads the repository through a context not at all. A change that moves no module builds nothing.
|
||||
func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
|
||||
entries := []inventory.Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}},
|
||||
{Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}},
|
||||
{Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}},
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"}
|
||||
scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}}
|
||||
clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil }
|
||||
|
||||
got := checkBuildsOf(p, scope, entries, clone)
|
||||
want := []link.CheckBuild{
|
||||
{Module: "app", Path: "modules/app", Merged: true, Tier: 0},
|
||||
{Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"},
|
||||
{Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want)
|
||||
}
|
||||
if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil {
|
||||
t.Fatalf("a change that moves no module asks for builds: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
|
||||
// run never does, and neither does an ask that is not a walk's.
|
||||
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
||||
asked := reuseAskOf(p)
|
||||
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
|
||||
t.Fatalf("the walk names %+v", asked.Pulls)
|
||||
}
|
||||
walk := withReuse(t.Context(), asked)
|
||||
if reuseAsked(walk, false) != asked {
|
||||
t.Fatal("a walk's ask does not carry its leave")
|
||||
}
|
||||
if reuseAsked(walk, true) != nil {
|
||||
t.Fatal("a dry run carries a leave to take a check's build")
|
||||
}
|
||||
if reuseAsked(t.Context(), false) != nil {
|
||||
t.Fatal("an ask that is not a walk's carries a leave")
|
||||
}
|
||||
}
|
||||
|
||||
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
|
||||
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
|
||||
var forgot []builder.PullRef
|
||||
was := forgetChecks
|
||||
forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||
forgot = append(forgot, pull)
|
||||
return "", nil
|
||||
}
|
||||
t.Cleanup(func() { forgetChecks = was })
|
||||
p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}}}}
|
||||
|
||||
p.WalkSaid = inventory.WalkStarted
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
p.WalkSaid = ""
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
if len(forgot) != 0 {
|
||||
t.Fatalf("a walk not ended let go of %v, which it may still take", forgot)
|
||||
}
|
||||
p.WalkSaid = inventory.WalkEnded
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) {
|
||||
t.Fatalf("the walk ended let go of %v", forgot)
|
||||
}
|
||||
}
|
||||
|
||||
// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is.
|
||||
func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) {
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" {
|
||||
t.Fatalf("a module's own build reads as a scratch build: %s", got)
|
||||
}
|
||||
scratch := "store.internal:5000/scratch/app/server@sha256:aa"
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch {
|
||||
t.Fatalf("a scratch build was not found: %q", got)
|
||||
}
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" {
|
||||
t.Fatal("a scratch archive was not found")
|
||||
}
|
||||
}
|
||||
|
||||
// The walk says per module what it did: "reused from …" or "built: <why>" — and a build seat from before says
|
||||
// neither, which is said.
|
||||
func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||
if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the
|
||||
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
|
||||
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
gateEvery = 0
|
||||
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
reused := func(id, reference string) link.BuildResult {
|
||||
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
|
||||
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
|
||||
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
|
||||
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
|
||||
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
|
||||
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
|
||||
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
|
||||
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
|
||||
!strings.Contains(err.Error(), "scratch") {
|
||||
t.Fatalf("a build in the scratch namespace was registered: %v", err)
|
||||
}
|
||||
id := link.NewBuildID(time.Now().Add(time.Second))
|
||||
result := reused(id, "store.internal:5000/app/server@"+digest)
|
||||
if _, _, err := takeIn(ctx, inv, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil || current["app"].Commit != "c3" {
|
||||
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(shelf["app"].Resources)
|
||||
if !strings.Contains(string(raw), "app/server@"+digest) {
|
||||
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
|
||||
}
|
||||
|
||||
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
|
||||
asked := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Commit = "c3"
|
||||
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
|
||||
p = g.plan(t)
|
||||
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
|
||||
t.Fatalf("the walk keeps %+v", m)
|
||||
}
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p = g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the gate judged %+v, not the reused build", gate)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
|
||||
t.Fatalf("the first machine was sent %v, not the reused build", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an
|
||||
// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does
|
||||
// not name (novox/hq ADR 0350 §2, issue 703).
|
||||
func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
forgetWalkVerdicts()
|
||||
t.Cleanup(func() { checkedBuildsOf = was; forgetWalkVerdicts() })
|
||||
var asked [][]link.PullRef
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
atHead := link.CheckedBuild{Pull: pull, Check: "build-2", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:2"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{atHead, earlier}, nil
|
||||
}
|
||||
pulls := []link.PullRef{pull}
|
||||
heads := map[link.PullRef]string{pull: "bbbbbbbbbbbb0123456789"}
|
||||
for range 3 {
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, heads); !reflect.DeepEqual(got.Builds,
|
||||
[]link.CheckedBuild{atHead}) {
|
||||
t.Fatalf("the leave names %+v, not only the merged head's build", got.Builds)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(asked, [][]link.PullRef{pulls}) {
|
||||
t.Fatalf("the verdicts were read %d times for one walk: %v", len(asked), asked)
|
||||
}
|
||||
// A merge whose head was not said takes nothing.
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, nil); got.Builds != nil {
|
||||
t.Fatalf("a merge of no said head names %+v", got.Builds)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-2", &link.ReuseAsk{Pulls: pulls}, heads); got.Builds != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got.Builds)
|
||||
}
|
||||
}
|
||||
|
||||
// A check is handed only the builds earlier checks of its own pull request at its own head made: another pull
|
||||
// request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read.
|
||||
func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
t.Cleanup(func() { checkedBuildsOf = was })
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
mine := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-0", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
another := link.CheckedBuild{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 8}, Check: "build-9",
|
||||
Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
var asked [][]link.PullRef
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{mine, earlier, another}, nil
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Commit: "bbbbbbbbbbbb0123456789"}
|
||||
builds := []link.CheckBuild{{Module: "app"}}
|
||||
if got := namedForTheCheck(t.Context(), p, builds); !reflect.DeepEqual(got, []link.CheckedBuild{mine}) ||
|
||||
!reflect.DeepEqual(asked, [][]link.PullRef{{pull}}) {
|
||||
t.Fatalf("the check is handed %+v, read for %v", got, asked)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := namedForTheCheck(t.Context(), p, builds); got != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func forgetWalkVerdicts() {
|
||||
walkVerdicts.Lock()
|
||||
defer walkVerdicts.Unlock()
|
||||
walkVerdicts.read = map[string]verdictsRead{}
|
||||
}
|
||||
|
||||
// A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request
|
||||
// merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703).
|
||||
func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) {
|
||||
merge := func(number int, head string) inventory.PlanMerge {
|
||||
return inventory.PlanMerge{Repository: "novox/shop", Number: number, Head: head}
|
||||
}
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
merge(7, "aaaaaaaaaaaa1"), merge(8, ""), merge(9, "bbbbbbbbbbbb"), merge(9, "cccccccccccc")}}}
|
||||
heads := mergedHeadsOf(p)
|
||||
shop := func(n int) link.PullRef { return link.PullRef{Owner: "novox", Repo: "shop", Number: n} }
|
||||
if heads[shop(7)] != "aaaaaaaaaaaa1" || heads[shop(8)] != "" || heads[shop(9)] != "" {
|
||||
t.Fatalf("merged heads %v", heads)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A delivery group's member is checked beside the other members' heads** (novox/hq ADR 0382, issue 867).
|
||||
//
|
||||
// A mesh-host change and a mesh-controller change that need each other could not both pass their own merge checks:
|
||||
// each was read beside the other's repository at the commit the mesh runs, which holds neither change. So the
|
||||
// controller asks mesh-delivery's `groups` once for every pull request's check it asks (§1), and names, in the ask,
|
||||
// the group's other open pull requests into a base of the same name, in a repository the build seat reads beside the
|
||||
// check — mesh-lab never — each with its whole head. The build seat reads those repositories at the sibling's merge (§2).
|
||||
// A pull request for which mesh-delivery answered and named no sibling is asked exactly as before (§7).
|
||||
|
||||
// wholeCommit is a commit as the forge says a head: forty hexadecimal characters.
|
||||
var wholeCommit = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
// besideRepositories are the repositories (lower case, owner/repo) a check reads beside its own, by the directory each
|
||||
// is found under — the core's, as besideOf clones them, and mesh-sdk beside the controller; never mesh-lab, which is
|
||||
// always read at its trunk.
|
||||
func besideRepositories(entries []inventory.Entry) map[string]string {
|
||||
read := map[string]string{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
read[nameOf(e.Source.Repository)] = dir
|
||||
if dir == "mesh-controller" && e.Source.Seat != "" {
|
||||
read[nameOf(siblingOf(e.Source.Repository, "mesh-sdk"))] = "mesh-sdk"
|
||||
}
|
||||
}
|
||||
delete(read, "")
|
||||
return read
|
||||
}
|
||||
|
||||
// nameOf is a recorded repository as owner/repo in lower case, empty when it names no owner.
|
||||
func nameOf(repository string) string {
|
||||
owner, repo := sourceOwner(repository), sourceName(repository)
|
||||
if owner == "" || repo == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(owner + "/" + repo)
|
||||
}
|
||||
|
||||
// siblingsFor is the pull request's siblings, as mesh-delivery's groups say them (ADR 0382 §1): the other members of
|
||||
// the group its head branch names that are open, unmerged and into a base of its own base's name, in a repository the
|
||||
// check reads beside it (beside: the directories the ask already names). The second answer is why they could not be
|
||||
// learned, empty when they could: the group's members carry no head or base — a mesh-delivery from before the rule.
|
||||
func siblingsFor(p link.PullUpdated, entries []inventory.Entry, beside map[string]link.CheckedOut,
|
||||
lines []groupsLine) ([]link.CheckSibling, string) {
|
||||
own := strings.ToLower(p.Owner + "/" + p.Repo)
|
||||
var group *groupsLine
|
||||
if p.Head == "" {
|
||||
// A recheck (mesh-delivery's `recheck` through `delivery check`) carries no head branch name: the group is the one
|
||||
// whose members hold this very pull request, by repository and number.
|
||||
learnable := false
|
||||
for i := range lines {
|
||||
for _, m := range lines[i].Details {
|
||||
learnable = true
|
||||
if p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own {
|
||||
group = &lines[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if group == nil {
|
||||
for _, l := range lines {
|
||||
if len(l.Details) == 0 && len(l.Members) > 0 && !learnable {
|
||||
return nil, "mesh-delivery named the groups' members without their pull requests (a mesh-delivery from before novox/hq ADR 0382): " +
|
||||
"the group of a pull request announced without its head branch cannot be found"
|
||||
}
|
||||
}
|
||||
return nil, "" // a lone pull request
|
||||
}
|
||||
} else {
|
||||
for i := range lines {
|
||||
if lines[i].ID == p.Head {
|
||||
group = &lines[i]
|
||||
}
|
||||
}
|
||||
if group == nil {
|
||||
return nil, ""
|
||||
}
|
||||
}
|
||||
if len(group.Details) == 0 && len(group.Members) > 0 {
|
||||
return nil, "mesh-delivery named the group's members without their heads and bases (a mesh-delivery from before novox/hq ADR 0382)"
|
||||
}
|
||||
read := besideRepositories(entries)
|
||||
var siblings []link.CheckSibling
|
||||
for _, m := range group.Details {
|
||||
repository := strings.ToLower(m.Repository)
|
||||
if repository == own {
|
||||
continue
|
||||
}
|
||||
if !m.Open {
|
||||
continue
|
||||
}
|
||||
if m.Head == "" || m.Base == "" {
|
||||
return nil, "mesh-delivery named " + m.ID + " without its head or base (a mesh-delivery from before novox/hq ADR 0382)"
|
||||
}
|
||||
dir, readBeside := read[repository]
|
||||
if m.Base != p.Base || !readBeside || dir == "mesh-lab" {
|
||||
continue
|
||||
}
|
||||
if _, there := beside[dir]; !there {
|
||||
continue
|
||||
}
|
||||
head := strings.ToLower(m.Head)
|
||||
if !wholeCommit.MatchString(head) {
|
||||
return nil, fmt.Sprintf("mesh-delivery named %s with a head that is not a whole commit (%q)", m.ID, m.Head)
|
||||
}
|
||||
siblings = append(siblings, link.CheckSibling{Repository: m.Repository, Number: m.Number, Head: head, Base: m.Base, Dir: dir})
|
||||
}
|
||||
sort.Slice(siblings, func(i, j int) bool { return siblings[i].Dir < siblings[j].Dir })
|
||||
return siblings, ""
|
||||
}
|
||||
|
||||
// withSiblings asks mesh-delivery's `groups` once and says in the request the pull request's siblings, or why they
|
||||
// could not be learned (ADR 0382 §1): a check never waits on mesh-delivery past groupAskWithin and never fails for it,
|
||||
// so a fix to mesh-delivery is still checked without it.
|
||||
func withSiblings(ctx context.Context, request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry) {
|
||||
if request.Check == nil {
|
||||
return
|
||||
}
|
||||
lines, err := groupsAsked(ctx, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
request.Check.SiblingsUnknown = fmt.Sprintf("mesh-delivery did not say the group's pull requests (%v): "+
|
||||
"the check reads no siblings", err)
|
||||
return
|
||||
}
|
||||
request.Check.Siblings, request.Check.SiblingsUnknown = siblingsFor(p, entries, request.Check.Beside, lines)
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A delivery group's member is checked beside the other members' heads** (novox/hq ADR 0382 §1, §7, issue 867): the
|
||||
// controller asks mesh-delivery's `groups` once for every check it asks, and names in the ask the group's other open
|
||||
// pull requests into a base of the same name, in a repository the build seat reads beside it — mesh-lab never.
|
||||
|
||||
const (
|
||||
headHost = "1111111111111111111111111111111111111111"
|
||||
headController = "2222222222222222222222222222222222222222"
|
||||
headSDK = "3333333333333333333333333333333333333333"
|
||||
headLab = "4444444444444444444444444444444444444444"
|
||||
headHq = "5555555555555555555555555555555555555555"
|
||||
)
|
||||
|
||||
func theCoreOnTheGitSeat() []inventory.Entry {
|
||||
onSeat := func(module, repository string) inventory.Entry {
|
||||
e := fromRepo(module, repository, "")
|
||||
e.Source.Seat = "git"
|
||||
return e
|
||||
}
|
||||
nats := onSeat("nats", "novox/mesh-catalog")
|
||||
nats.Source.Path = "modules/nats"
|
||||
held := catalogue.Manifest{Module: "mesh-delivery", Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
|
||||
return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"),
|
||||
onSeat("node-tools", "novox/mesh-tools"), nats, {Manifest: held, On: []string{"anchor"}}}
|
||||
}
|
||||
|
||||
func besideTheCore() map[string]link.CheckedOut {
|
||||
return map[string]link.CheckedOut{"mesh-controller": {Repository: "c"}, "mesh-host": {Repository: "h"},
|
||||
"mesh-tools": {Repository: "t"}, "mesh-catalog": {Repository: "k"}, "mesh-sdk": {Repository: "s"}, "mesh-lab": {Repository: "l"}}
|
||||
}
|
||||
|
||||
func member(repository string, number int, head, base string, open bool) memberLine {
|
||||
return memberLine{ID: repository + "@" + head[:12], Repository: repository, State: "checked", Number: number, Head: head,
|
||||
Base: base, Open: open}
|
||||
}
|
||||
|
||||
func group(id string, members ...memberLine) groupsLine {
|
||||
line := groupsLine{ID: id, Details: members}
|
||||
for _, m := range members {
|
||||
line.Members = append(line.Members, m.ID+" ("+m.State+")")
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func TestASiblingIsTheGroupsOtherOpenPullRequestIntoABaseOfTheSameName(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
pull := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
lines []groupsLine
|
||||
want []string
|
||||
unknown string
|
||||
}{
|
||||
{"the other member, open, same base", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
[]string{"mesh-controller=novox/mesh-controller#346@" + headController + " onto main"}, ""},
|
||||
{"a member into a base of another name is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "release", true))},
|
||||
nil, ""},
|
||||
{"a member not open (merged or closed) is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", false))},
|
||||
nil, ""},
|
||||
{"mesh-lab is never one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-lab", 5, headLab, "main", true))},
|
||||
nil, ""},
|
||||
{"a repository the check reads nothing of is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/hq", 9, headHq, "main", true))},
|
||||
nil, ""},
|
||||
{"mesh-sdk, read beside the controller, is one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-sdk", 7, headSDK, "main", true))},
|
||||
[]string{"mesh-sdk=novox/mesh-sdk#7@" + headSDK + " onto main"}, ""},
|
||||
{"another group is none of its business", []groupsLine{group("feat/y",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
nil, ""},
|
||||
{"a group of its own alone has none", []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true))},
|
||||
nil, ""},
|
||||
{"an older mesh-delivery names no heads", []groupsLine{{ID: "feat/x",
|
||||
Members: []string{"novox/mesh-host@111111111111 (checked)", "novox/mesh-controller@222222222222 (checked)"}}},
|
||||
nil, "without their heads and bases"},
|
||||
{"a member without its base", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "", true))},
|
||||
nil, "without its head or base"},
|
||||
{"a head that is no commit", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController[:12]+"zzzzzzzzzzzzzzzzzzzzzzzzzzzz", "main", true))},
|
||||
nil, "not a whole commit"},
|
||||
} {
|
||||
got, unknown := siblingsFor(pull, entries, besideTheCore(), c.lines)
|
||||
var said []string
|
||||
for _, s := range got {
|
||||
said = append(said, s.Dir+"="+s.Repository+"#"+itoa(s.Number)+"@"+s.Head+" onto "+s.Base)
|
||||
}
|
||||
if strings.Join(said, ";") != strings.Join(c.want, ";") {
|
||||
t.Errorf("%s: siblings %v, wanted %v", c.what, said, c.want)
|
||||
}
|
||||
if (c.unknown == "") != (unknown == "") || !strings.Contains(unknown, c.unknown) {
|
||||
t.Errorf("%s: could not learn %q, wanted %q", c.what, unknown, c.unknown)
|
||||
}
|
||||
}
|
||||
// A pull request into a release branch has siblings into the release branch of the same name (ADR 0358 §1).
|
||||
release := pull
|
||||
release.Base = "feat/57"
|
||||
lines := []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "feat/57", true),
|
||||
member("novox/mesh-controller", 346, headController, "feat/57", true), member("novox/mesh-tools", 8, headLab, "main", true))}
|
||||
if got, _ := siblingsFor(release, entries, besideTheCore(), lines); len(got) != 1 || got[0].Dir != "mesh-controller" {
|
||||
t.Errorf("into a release branch: %+v", got)
|
||||
}
|
||||
// A recheck announces the pull request without its head branch: its group is the one holding it (repository, number).
|
||||
pull.Head = ""
|
||||
got, unknown := siblingsFor(pull, entries, besideTheCore(), lines)
|
||||
if len(got) != 1 || got[0].Dir != "mesh-tools" || unknown != "" {
|
||||
t.Errorf("a recheck of a group member: %+v %q", got, unknown)
|
||||
}
|
||||
// A pull request in no group, rechecked, is alone: no sibling, and nothing said.
|
||||
pull.Number = 999
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), lines); got != nil || unknown != "" {
|
||||
t.Errorf("a lone pull request: %+v %q", got, unknown)
|
||||
}
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), nil); got != nil || unknown != "" {
|
||||
t.Errorf("no groups at all: %+v %q", got, unknown)
|
||||
}
|
||||
// A mesh-delivery that names members without their pull requests cannot find the group of a recheck: it says so.
|
||||
older := []groupsLine{{ID: "feat/x", Members: []string{"novox/mesh-host@111111111111 (checked)"}}}
|
||||
pull.Number = 114
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), older); got != nil || unknown == "" {
|
||||
t.Errorf("an older mesh-delivery: %+v %q", got, unknown)
|
||||
}
|
||||
}
|
||||
|
||||
func itoa(n int) string { b, _ := json.Marshal(n); return string(b) }
|
||||
|
||||
// Every check asks mesh-delivery's `groups` once; the ask names the sibling's repository, head and base.
|
||||
func TestEveryCheckAsksMeshDeliveryOnceAndNamesItsSiblings(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
asked := 0
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
groupsAsked = func(_ context.Context, held bool) ([]groupsLine, error) {
|
||||
asked++
|
||||
if !held {
|
||||
t.Error("the seat is held, yet it was asked as not held")
|
||||
}
|
||||
return []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true),
|
||||
member("novox/mesh-controller", 346, headController, "main", true))}, nil
|
||||
}
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", Beside: besideTheCore()}}
|
||||
withSiblings(t.Context(), &request, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
if asked != 1 {
|
||||
t.Fatalf("mesh-delivery was asked %d times for one check", asked)
|
||||
}
|
||||
want := link.CheckSibling{Repository: "novox/mesh-controller", Number: 346, Head: headController, Base: "main", Dir: "mesh-controller"}
|
||||
if got := request.Check.Siblings; len(got) != 1 || got[0] != want || request.Check.SiblingsUnknown != "" {
|
||||
t.Fatalf("the ask names siblings %+v, unknown %q", got, request.Check.SiblingsUnknown)
|
||||
}
|
||||
body, _ := json.Marshal(request.Check)
|
||||
if !strings.Contains(string(body), `"siblings":[{"repository":"novox/mesh-controller","number":346,"head":"`+headController+
|
||||
`","base":"main","dir":"mesh-controller"}]`) {
|
||||
t.Fatalf("on the wire: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// When mesh-delivery does not answer in time (or is not held, or answered without the fields), the check reads no
|
||||
// sibling and its request says it could not learn them, so a fix to mesh-delivery is still checked without it.
|
||||
func TestWhenMeshDeliveryDoesNotAnswerTheCheckReadsNoSiblingAndSaysSo(t *testing.T) {
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
err error
|
||||
held bool
|
||||
}{{"it did not answer", context.DeadlineExceeded, true}, {"nothing serves it", errors.New("nothing serves mesh-delivery.groups"), true},
|
||||
{"no holder on record", errors.New("no holder of mesh-delivery is on record to say it"), false}} {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
if !c.held {
|
||||
entries = entries[:4]
|
||||
}
|
||||
groupsAsked = func(_ context.Context, held bool) ([]groupsLine, error) {
|
||||
if held != c.held {
|
||||
t.Errorf("%s: held %v", c.what, held)
|
||||
}
|
||||
return nil, c.err
|
||||
}
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", Beside: besideTheCore()}}
|
||||
withSiblings(t.Context(), &request, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
if len(request.Check.Siblings) != 0 || !strings.Contains(request.Check.SiblingsUnknown, c.err.Error()) ||
|
||||
!strings.Contains(request.Check.SiblingsUnknown, "reads no siblings") {
|
||||
t.Errorf("%s: siblings %+v, unknown %q", c.what, request.Check.Siblings, request.Check.SiblingsUnknown)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A pull request for which mesh-delivery answered and named no sibling is asked exactly as before** (ADR 0382 §7):
|
||||
// byte for byte the request it was without the rule.
|
||||
func TestAPullRequestWithNoSiblingIsAskedByteForByteAsBefore(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
for _, answer := range [][]groupsLine{nil, {group("feat/other", member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/hq", 3, headHq, "main", true))}} {
|
||||
groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, nil }
|
||||
build := func() link.BuildRequest {
|
||||
return link.BuildRequest{ID: "build-1", Repository: "r", Ref: headHost, Check: &link.CheckRequest{Owner: "novox",
|
||||
Repo: "mesh-host", Number: 114, Base: "main", Beside: besideTheCore(), Modules: []string{"mesh-host"}, Judge: "validator"}}
|
||||
}
|
||||
before, _ := json.Marshal(build())
|
||||
after := build()
|
||||
withSiblings(t.Context(), &after, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
got, _ := json.Marshal(after)
|
||||
if string(before) != string(got) {
|
||||
t.Fatalf("the request changed:\nbefore %s\nafter %s", before, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the build seat's verdict says of its siblings reaches the forge's holder in the controller's `checked`.
|
||||
func TestTheVerdictCarriesTheSiblingsAndWhereTheJudgeCameFrom(t *testing.T) {
|
||||
read := []link.CheckSibling{{Repository: "novox/mesh-controller", Number: 346, Head: headController, Base: "main", BaseCommit: headHost}}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: headHost, Checked: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114},
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Summary: "ok", Siblings: read, JudgeFrom: "sibling novox/mesh-controller#346 merge abc",
|
||||
Validator: "the change's own validator"}})
|
||||
if len(c.Siblings) != 1 || c.Siblings[0] != read[0] || c.JudgeFrom == "" || c.Validator == "" {
|
||||
t.Fatalf("the verdict says %+v", c)
|
||||
}
|
||||
body, _ := json.Marshal(c)
|
||||
for _, want := range []string{`"siblings":[{"repository":"novox/mesh-controller","number":346,"head":"` + headController +
|
||||
`","base":"main","base_commit":"` + headHost + `"}]`, `"judge_from":"sibling novox/mesh-controller#346 merge abc"`} {
|
||||
if !strings.Contains(string(body), want) {
|
||||
t.Fatalf("the verdict lacks %s:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
unknown := checkedOf(link.BuildResult{ID: "b", Check: &link.CheckOutcome{Verdict: "pass", SiblingsUnknown: "mesh-delivery did not answer"}})
|
||||
if body, _ := json.Marshal(unknown); !strings.Contains(string(body), `"siblings_unknown":"mesh-delivery did not answer"`) ||
|
||||
strings.Contains(string(body), `"siblings":`) {
|
||||
t.Fatalf("the verdict that could not learn them says %s", body)
|
||||
}
|
||||
}
|
||||
+310
-32
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
@@ -161,9 +162,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Print(planText(plan))
|
||||
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
|
||||
direct := directVerdict(p, plan)
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
@@ -181,6 +180,8 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return err
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
// **The group's other open pull requests are read beside it** (novox/hq ADR 0382 §1).
|
||||
withSiblings(ctx, &request, p, entries)
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
@@ -190,6 +191,8 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
// A new head ends the check of the old one (novox/hq issue 726).
|
||||
endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot)
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
@@ -201,6 +204,24 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// directVerdict is the verdict of a change no build judges — it touches no module, and either the repository is not
|
||||
// the mesh's or the head holds no merge-check.sh. **It names the trunk commit it holds for** (novox/hq ADR 0323 §3):
|
||||
// the base branch's head the forge announced with the head, so the merge tool can weigh it against the trunk's head
|
||||
// and does not refuse it for ever (a verdict naming no trunk is never the trunk's head's). Nothing was built to
|
||||
// judge: the trunk commit is the one the forge said the change was announced against.
|
||||
func directVerdict(p link.PullUpdated, plan link.ChangePlan) link.Checked {
|
||||
trunk := p.BaseSHA
|
||||
if len(trunk) > 12 {
|
||||
trunk = trunk[:12]
|
||||
}
|
||||
if trunk != "" {
|
||||
plan.Trunk = trunk
|
||||
}
|
||||
return link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan, Trunk: trunk}
|
||||
}
|
||||
|
||||
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
|
||||
// and what is read beside it.
|
||||
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
|
||||
@@ -232,31 +253,16 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
if dir == "mesh-controller" {
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358): asked of the forge once.
|
||||
release, err := releaseBranchOf(ctx, p, scope, entries)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone, release)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
builds := checkBuildsOf(p, scope, entries, clone)
|
||||
return link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
@@ -266,10 +272,237 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge},
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||
Release: release, Running: runningBy(ctx)},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||
// by the module the mesh builds from it, at besideRefs — or, for a pull request into a release branch (novox/hq ADR
|
||||
// 0358 §3), at its branch of that name where it has one and at its trunk otherwise, the controller's main kept for
|
||||
// its judge, and mesh-lab at its trunk whatever the base: its replays run with the container runtime's socket. running
|
||||
// is the commit the mesh runs of a module.
|
||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
clone func(inventory.Source) (string, error), release string) (map[string]link.CheckedOut, error) {
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs := besideRefs(dir, running(e.Manifest.Module))
|
||||
at := func(dir, trunk string) link.CheckedOut {
|
||||
if release == "" {
|
||||
return link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
}
|
||||
if trunk == "" {
|
||||
trunk = "main"
|
||||
}
|
||||
return link.CheckedOut{Repository: url, Ref: release, Else: trunk}
|
||||
}
|
||||
beside[dir] = at(dir, followedBranch(e.Source.Ref))
|
||||
if dir == "mesh-controller" {
|
||||
// The controller's main stays its main: it judges only a check the running controller predates.
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if release != "" {
|
||||
// mesh-controller has the release branch: that is what makes it one (ADR 0358 §1).
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: release}
|
||||
}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
b := link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
// **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container
|
||||
// runtime's socket, so they are only ever reviewed code — never a branch anyone may create.
|
||||
if release != "" && sibling != "mesh-lab" {
|
||||
b = link.CheckedOut{Repository: url, Ref: release, Else: "main"}
|
||||
}
|
||||
beside[sibling] = b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return beside, nil
|
||||
}
|
||||
|
||||
// releaseBranchOf is the release branch a pull request merges into (novox/hq ADR 0358 §1): its base, when that is not
|
||||
// its repository's trunk and mesh-controller has a branch of that name; empty otherwise, and the pull request is
|
||||
// checked as before it. **A base that is a trunk asks nothing**: the branch a module built from the repository follows
|
||||
// (main when it records none), or mesh-controller's own trunk. Any other base is asked of the forge once, and an
|
||||
// answer it cannot give is an error, never read as "no release branch".
|
||||
func releaseBranchOf(ctx context.Context, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (string, error) {
|
||||
base := p.Base
|
||||
if base == "" {
|
||||
return "", nil
|
||||
}
|
||||
if scope.From != nil && base == trunkOf(scope.From.Source) {
|
||||
return "", nil
|
||||
}
|
||||
var controller *inventory.Entry
|
||||
for i, e := range entries {
|
||||
if e.Manifest.Module == "mesh-controller" && !e.Provided && e.Source.Repository != "" {
|
||||
controller = &entries[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if controller == nil || base == trunkOf(controller.Source) {
|
||||
return "", nil
|
||||
}
|
||||
if !tryRef.MatchString(base) || strings.Contains(base, "..") {
|
||||
return "", nil // no branch the forge could hold: checked as before
|
||||
}
|
||||
owner, repo := sourceOwner(controller.Source.Repository), sourceName(controller.Source.Repository)
|
||||
if owner == "" || repo == "" {
|
||||
return "", nil
|
||||
}
|
||||
has, err := forgeHasBranch(ctx, owner, repo, base)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("whether %s/%s#%d merges into a release branch cannot be told — the forge did not say "+
|
||||
"whether %s/%s has a branch %s (novox/hq ADR 0358): %w", p.Owner, p.Repo, p.Number, owner, repo, base, err)
|
||||
}
|
||||
if !has {
|
||||
return "", nil
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// trunkOf is the branch a module's source follows: main when it records none, or records a commit.
|
||||
func trunkOf(s inventory.Source) string {
|
||||
if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" {
|
||||
return t
|
||||
}
|
||||
return "main"
|
||||
}
|
||||
|
||||
// sourceName is the name of a recorded repository, a path on the git seat or a URL: novox/mesh-host → mesh-host.
|
||||
func sourceName(repository string) string {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
|
||||
// forgeHasBranch asks the forge, through its module's tools on the bus, whether a repository has a branch. A
|
||||
// variable so a test needs no forge.
|
||||
var forgeHasBranch = func(ctx context.Context, owner, repo, branch string) (bool, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(map[string]any{"path": "/repos/" + owner + "/" + repo + "/branches/" + branch})
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return branchAnswered(answer, branch)
|
||||
}
|
||||
|
||||
// branchAnswered reads the forge's answer about a branch: a 404 is "no such branch"; an answer naming the branch is
|
||||
// "it has it"; anything else — another error, an unreadable answer, one naming no branch or another — is an error,
|
||||
// never read as "no release branch".
|
||||
func branchAnswered(answer link.Answer, branch string) (bool, error) {
|
||||
if answer.Error != "" {
|
||||
if strings.Contains(answer.Error, " 404 ") {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.New(answer.Error)
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &found); err != nil {
|
||||
return false, fmt.Errorf("the forge's answer about branch %s cannot be read: %w", branch, err)
|
||||
}
|
||||
if found.Result.Name != branch {
|
||||
return false, fmt.Errorf("the forge answered about branch %s naming %q", branch, found.Result.Name)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the
|
||||
// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key
|
||||
// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be
|
||||
// read, and the check builds, as before.
|
||||
func namedForTheCheck(ctx context.Context, p link.PullUpdated, builds []link.CheckBuild) []link.CheckedBuild {
|
||||
if len(builds) == 0 || p.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
pull := link.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}
|
||||
read, err := checkedBuildsOf(ctx, []link.PullRef{pull})
|
||||
if err != nil {
|
||||
fmt.Printf(" the earlier checks' builds of %s/%s#%d could not be read, so the check builds: %v\n",
|
||||
p.Owner, p.Repo, p.Number, err)
|
||||
// empty-on-error: said above; a check handed no named build builds every module, as before issue 703
|
||||
return nil
|
||||
}
|
||||
var named []link.CheckedBuild
|
||||
for _, b := range read {
|
||||
if b.Pull == pull && link.SameHead(b.Head, p.Commit) {
|
||||
named = append(named, b)
|
||||
}
|
||||
}
|
||||
return named
|
||||
}
|
||||
|
||||
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
|
||||
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
|
||||
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
|
||||
// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context,
|
||||
// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that
|
||||
// moves no module builds nothing.
|
||||
func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry,
|
||||
clone func(inventory.Source) (string, error)) []link.CheckBuild {
|
||||
if len(scope.Reach.Plan.Tiers) == 0 {
|
||||
return nil
|
||||
}
|
||||
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
skip := map[string]string{}
|
||||
for _, e := range scope.Reach.Packaging {
|
||||
skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " +
|
||||
"cannot hand the merge"
|
||||
}
|
||||
for _, e := range scope.Reach.Deleted {
|
||||
skip[e.Manifest.Module] = "the merge deletes it"
|
||||
}
|
||||
var out []link.CheckBuild
|
||||
for tier, names := range scope.Reach.Plan.Tiers {
|
||||
for _, name := range names {
|
||||
e, known := byName[name]
|
||||
b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]}
|
||||
switch {
|
||||
case !known:
|
||||
b.Skip = "the catalogue does not hold it"
|
||||
case b.Skip != "":
|
||||
case sourceIs(e.Source, m):
|
||||
b.Merged, b.Path = true, e.Source.Path
|
||||
default:
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
b.Skip = "its repository cannot be cloned: " + err.Error()
|
||||
break
|
||||
}
|
||||
b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref)
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||
@@ -363,6 +596,13 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
case result.Check != nil:
|
||||
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
|
||||
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
|
||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
||||
// repositories read beside it.
|
||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
||||
c.Built = result.Check.Built
|
||||
c.Release, c.JudgedBy, c.BesideAt = result.Check.Release, result.Check.JudgedBy, result.Check.BesideAt
|
||||
c.Validator = result.Check.Validator
|
||||
c.Siblings, c.SiblingsUnknown, c.JudgeFrom = result.Check.Siblings, result.Check.SiblingsUnknown, result.Check.JudgeFrom
|
||||
if c.Gate == nil {
|
||||
// A build seat from before the layers: its verdict is the gate's.
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
@@ -388,14 +628,20 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
}
|
||||
if result.Checked != nil {
|
||||
c.Plan = result.Checked.Plan
|
||||
if c.Plan != nil && c.Trunk != "" {
|
||||
// The kept delivery plan records the trunk commit with its other inputs (ADR 0323 §3).
|
||||
plan := *c.Plan
|
||||
plan.Trunk = c.Trunk
|
||||
c.Plan = &plan
|
||||
}
|
||||
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
|
||||
if g := result.Checked; g.Group != "" {
|
||||
c.Group = g.Group
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
|
||||
Commit: result.Ref})
|
||||
Commit: result.Ref, Trunk: c.Trunk})
|
||||
for _, m := range g.Members {
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
|
||||
Commit: m.Ref})
|
||||
Commit: m.Ref, Trunk: memberTrunk(result.Check, m)})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -404,12 +650,40 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
l.Verdict = "error"
|
||||
}
|
||||
}
|
||||
// **A check run again says so** (novox/hq issue 594). The summaries are the lines the pull request
|
||||
// shows, clipped by the forge to a few words, so they get only a short marker after the verdict's own
|
||||
// words; the report gets the whole sentence. Layers copied, so the build's own outcome is left as it came.
|
||||
if mark := link.RunMarker(result.Delivery); mark != "" {
|
||||
c.Summary += " " + mark
|
||||
for _, l := range []**link.CheckLayer{&c.Gate, &c.RepoCheck} {
|
||||
if *l != nil {
|
||||
layer := **l
|
||||
layer.Summary += " " + mark
|
||||
*l = &layer
|
||||
}
|
||||
}
|
||||
c.Report = link.Restarted(result.Delivery) + "\n\n" + c.Report
|
||||
}
|
||||
if len(c.Report) > maxCheckReport {
|
||||
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// memberTrunk is the trunk commit the build seat merged one of a group's heads into (novox/hq ADR 0323 §6), empty
|
||||
// when it said none.
|
||||
func memberTrunk(outcome *link.CheckOutcome, m link.GroupMember) string {
|
||||
if outcome == nil {
|
||||
return ""
|
||||
}
|
||||
for _, j := range outcome.Members {
|
||||
if strings.EqualFold(j.Owner, m.Owner) && strings.EqualFold(j.Repo, m.Repo) && strings.HasPrefix(m.Ref, j.Commit) {
|
||||
return j.Trunk
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func prefixedAll(prefix string, items []string) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, i := range items {
|
||||
@@ -424,8 +698,12 @@ func sayChecked(ctx context.Context, c link.Checked) {
|
||||
if c.RepoCheck != nil {
|
||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||
}
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
judged := ""
|
||||
if c.Trunk != "" {
|
||||
judged = " merged into trunk " + c.Trunk
|
||||
}
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
if checkEvents == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -210,3 +210,81 @@ func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
|
||||
t.Errorf("a build seat that said nothing was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verdict names the trunk commit it judged** (novox/hq ADR 0323 §3, issue 341): the controller's `checked`
|
||||
// carries the head and the trunk commit the build seat merged it into, the commits of the repositories read beside
|
||||
// it, how long it took, and the kept delivery plan records the trunk commit with its other inputs. A group's
|
||||
// composed check names, per member, the trunk commit that head was merged into (§6).
|
||||
func TestAVerdictNamesTheTrunkCommitItJudged(t *testing.T) {
|
||||
plan := &link.ChangePlan{Repository: "novox/mesh-host", Base: "main", Head: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Summary: "builds mesh-host"}
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 66, Base: "main", Plan: plan}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Checked: asked,
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Took: "4m12s", Head: "dfb1415aaaaa", Trunk: "d28ca95bbbbb",
|
||||
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "pass"},
|
||||
Beside: map[string]string{"mesh-catalog": "0123456789ab", "mesh-lab": "ba9876543210"}}})
|
||||
if c.Trunk != "d28ca95bbbbb" || c.Commit != "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
||||
t.Fatalf("the verdict names head %s and trunk %q", c.Commit, c.Trunk)
|
||||
}
|
||||
if c.Beside["mesh-catalog"] != "0123456789ab" || c.Beside["mesh-lab"] != "ba9876543210" {
|
||||
t.Fatalf("the verdict names the repositories read beside it as %v", c.Beside)
|
||||
}
|
||||
if c.Took != "4m12s" {
|
||||
t.Fatalf("the verdict says it took %q", c.Took)
|
||||
}
|
||||
if c.Plan == nil || c.Plan.Trunk != "d28ca95bbbbb" || c.Plan.Head != plan.Head {
|
||||
t.Fatalf("the kept plan records %+v", c.Plan)
|
||||
}
|
||||
if plan.Trunk != "" {
|
||||
t.Fatal("the plan the controller asked with was changed: the verdict's is a copy")
|
||||
}
|
||||
|
||||
group := &link.CheckRequest{Owner: "novox", Repo: "mesh-controller", Number: 7, Base: "main", Group: "feat/x",
|
||||
Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-host", Number: 8, Ref: "cccccccccccc0000", Base: "main"}}}
|
||||
c = checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{
|
||||
Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"},
|
||||
Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "cccccccccccc", Trunk: "222222222222"}}}})
|
||||
if len(c.Members) != 2 || c.Members[0].Trunk != "111111111111" || c.Members[1].Trunk != "222222222222" ||
|
||||
c.Members[1].Commit != "cccccccccccc0000" {
|
||||
t.Fatalf("the composed check names its members %+v", c.Members)
|
||||
}
|
||||
}
|
||||
|
||||
// **The merge made for a check is never registered** (novox/hq ADR 0323 §2): a build of it — a commit on no branch
|
||||
// of the forge — is refused, naming the trunk. The rule of ADR 0238 §1 stands unchanged; this holds it for the
|
||||
// merge commit a check makes.
|
||||
func TestABuildOfACheckMergeIsRefusedNamingTheTrunk(t *testing.T) {
|
||||
merge := link.BuildResult{ID: "b", Commit: "4b5c6d7e8f90aabbccddeeff0011223344556677", Trunk: "main"}
|
||||
err := publishable(merge, "")
|
||||
if !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "not on main") {
|
||||
t.Fatalf("a build of a check's merge was answered %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verdict no build judges names its trunk commit too** (novox/hq ADR 0323 §3): a change touching no module of a
|
||||
// repository with no merge-check.sh (novox/mesh-tools-go) is answered by the controller itself. Naming no trunk, the
|
||||
// merge tool refused it, asked again, got the same verdict, and refused for ever.
|
||||
func TestADirectVerdictNamesTheTrunkTheForgeAnnounced(t *testing.T) {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-tools-go", Number: 3, Base: "main", Commit: "1a2b3c4d5e6f7a8b9c0d",
|
||||
BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
|
||||
c := directVerdict(p, link.ChangePlan{Repository: "novox/mesh-tools-go", Base: "main", Head: p.Commit})
|
||||
if c.Trunk != "7a8b9c0d1e2f" || c.Plan == nil || c.Plan.Trunk != "7a8b9c0d1e2f" {
|
||||
t.Fatalf("the direct verdict names trunk %q, its plan %+v", c.Trunk, c.Plan)
|
||||
}
|
||||
if c.Verdict != "pass" || c.Commit != p.Commit {
|
||||
t.Fatalf("the direct verdict is %+v", c)
|
||||
}
|
||||
if c := directVerdict(link.PullUpdated{Owner: "novox", Repo: "x", Commit: "abc"}, link.ChangePlan{}); c.Trunk != "" {
|
||||
t.Fatalf("a forge that said no base head gave the trunk %q", c.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
// A head checked again through mesh-delivery (`delivery check --members`, a recheck) reaches the same direct verdict:
|
||||
// it names the base head the forge last announced, kept on the member, or the merge tool would refuse it for ever.
|
||||
func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
|
||||
m := orderMember{ID: "novox/mesh-tools-go@1a2b3c4d5e6f", Repository: "novox/mesh-tools-go", Base: "main",
|
||||
Head: "1a2b3c4d5e6f7a8b9c0d", Number: 3, BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
|
||||
if c := directVerdict(m.pull(), link.ChangePlan{}); c.Trunk != "7a8b9c0d1e2f" {
|
||||
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,616 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **The move of config** (novox/hq ADR 0348 §9 and its consequences, design 57 K5, issue 664): every config value a
|
||||
// settings layer holds is set in mesh-config, one of the moves of data a release runs once after its delivery, after
|
||||
// the move of placement. A module's layer on a node becomes its own value on that node (`module.<node>.<module>.<key>`).
|
||||
// A whole-mesh layer has no module value for the whole mesh (ADR 0348 §3): its value becomes mesh config for the
|
||||
// predefined key it falls back to where that module alone falls back to the key (claude-code's managed settings
|
||||
// become `agent-managed-settings`), and otherwise the module's own value on each node it is assigned to, laid under
|
||||
// that node's layer as composition lays it. A key a layer still says by its file's name (nodered's `timeZone`) is
|
||||
// set under its config key (`time-zone`, issue 751).
|
||||
//
|
||||
// **Every node composes byte for byte as before.** The move sets the values, reads every module's resolved values
|
||||
// back through mesh-config and compares them with what the layers lay today; only when they agree does it empty the
|
||||
// layers it moved (each kept whole in its history first) and turn on the switch (inventory.FinishConfigMove), after
|
||||
// which the controller reads mesh-config where it read the layers. A module still in the settings form keeps its
|
||||
// layers and is said so. Anything that cannot move as it composes refuses the whole move, by name, before anything
|
||||
// is set. Asked again, it sets nothing that is set already.
|
||||
|
||||
// configLevel is where mesh-config keeps a value: mesh config, node config, or a module's own value on a node.
|
||||
type configLevel struct {
|
||||
Level string `json:"level"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
}
|
||||
|
||||
func (l configLevel) valueKey(key string) string {
|
||||
switch l.Level {
|
||||
case "node":
|
||||
return "node." + l.Node + "." + key
|
||||
case "module":
|
||||
return "module." + l.Node + "." + l.Module + "." + key
|
||||
}
|
||||
return "mesh." + key
|
||||
}
|
||||
|
||||
func (l configLevel) String() string {
|
||||
switch l.Level {
|
||||
case "node":
|
||||
return "node config on " + l.Node
|
||||
case "module":
|
||||
return l.Module + "'s own value on " + l.Node
|
||||
}
|
||||
return "mesh config"
|
||||
}
|
||||
|
||||
// configSet is one value the move sets in mesh-config, and the layers it comes from.
|
||||
type configSet struct {
|
||||
configLevel
|
||||
Key string `json:"key"`
|
||||
Value any `json:"value"`
|
||||
Trusted bool `json:"trusted,omitempty"`
|
||||
// From is the layers it comes from, as `<module> on <where>: <key as the layer says it>`.
|
||||
From []string `json:"from"`
|
||||
// Already is true when mesh-config holds this value at this level already: nothing is set.
|
||||
Already bool `json:"already,omitempty"`
|
||||
}
|
||||
|
||||
// configMoveAnswer is what the move of config says it did, or would do.
|
||||
type configMoveAnswer struct {
|
||||
// DryRun is true when nothing was set: asked without --confirm.
|
||||
DryRun bool `json:"dry-run"`
|
||||
Sets []configSet `json:"sets"`
|
||||
// Moved is each layer whose values the move sets, emptied once they read back.
|
||||
Moved []string `json:"moved"`
|
||||
// Stays is each layer of a module still in the settings form: kept, and composed from as before.
|
||||
Stays []string `json:"stays,omitempty"`
|
||||
// Unread is each layer that composes nothing (its module is not assigned where it says): kept in the history,
|
||||
// emptied, and set nowhere.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// Problems is why the move refuses, each by name; nothing is set while there is one.
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
// Switched is true once the controller reads mesh-config where it read the layers.
|
||||
Switched bool `json:"switched"`
|
||||
}
|
||||
|
||||
// configMovePlan is the move worked out from the layers, before anything is asked of mesh-config.
|
||||
type configMovePlan struct {
|
||||
answer configMoveAnswer
|
||||
// moved is the layers the move empties once the values read back.
|
||||
moved []inventory.LayerToMove
|
||||
// expected is, for each module the move reaches, keyed `<node>.<module>`, every value composition lays over
|
||||
// its files today: what mesh-config must resolve for it from mesh config, node config or its own value.
|
||||
expected map[string]map[string]any
|
||||
// modules is each module in the config form, by the nodes it is assigned to: whose resolved values are read back.
|
||||
modules map[string][]string
|
||||
}
|
||||
|
||||
func whereOf(node string) string {
|
||||
if node == "" {
|
||||
return "the whole mesh"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// planConfigMove works out the move from every layer that holds a value, the catalogue, and where each module is
|
||||
// assigned.
|
||||
func planConfigMove(layers []inventory.LayerToMove, shelf map[string]catalogue.Manifest,
|
||||
on map[string][]string) configMovePlan {
|
||||
p := configMovePlan{answer: configMoveAnswer{Sets: []configSet{}, Moved: []string{}},
|
||||
expected: map[string]map[string]any{}, modules: map[string][]string{}}
|
||||
problem := func(format string, a ...any) {
|
||||
p.answer.Problems = append(p.answer.Problems, fmt.Sprintf(format, a...))
|
||||
}
|
||||
for name, m := range shelf {
|
||||
if m.DeclaresConfig() {
|
||||
p.modules[name] = on[name]
|
||||
}
|
||||
}
|
||||
|
||||
byModule := map[string]*layersOfModule{}
|
||||
var names []string
|
||||
for _, l := range layers {
|
||||
said := fmt.Sprintf("%s on %s", l.Module, whereOf(l.Node))
|
||||
m, known := shelf[l.Module]
|
||||
if !known || !m.DeclaresConfig() {
|
||||
p.answer.Stays = append(p.answer.Stays, said+" (its module is in the settings form)")
|
||||
continue
|
||||
}
|
||||
if _, placement := catalogue.SplitPlacement(l.Values); len(placement) > 0 {
|
||||
problem("%s still holds placement (%s): run move-placement first", said, strings.Join(movedKeys(placement), ", "))
|
||||
continue
|
||||
}
|
||||
assigned := on[l.Module]
|
||||
if (l.Node == "" && len(assigned) == 0) || (l.Node != "" && !containsString(assigned, l.Node)) {
|
||||
p.answer.Unread = append(p.answer.Unread, said+" (the module is not assigned there, so it composes nothing)")
|
||||
p.moved = append(p.moved, l)
|
||||
continue
|
||||
}
|
||||
// The layer's keys as config keys: a key the file names otherwise is set under its config key.
|
||||
as := catalogue.TakenAs(m)
|
||||
values := map[string]any{}
|
||||
from := map[string]string{}
|
||||
for _, k := range movedKeys(l.Values) {
|
||||
key := k
|
||||
if _, declared := m.Settings[k]; !declared {
|
||||
if renamed, ok := as[k]; ok {
|
||||
key = renamed
|
||||
} else {
|
||||
problem("%s says %q, which %s does not declare: declare it, or clear it from the layer", said, k, l.Module)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, both := values[key]; both {
|
||||
// The layer says the key and the file's name for it: the config key wins, as composition has it (issue 751).
|
||||
if key != k {
|
||||
continue
|
||||
}
|
||||
}
|
||||
values[key] = l.Values[k]
|
||||
from[key] = said + ": " + k
|
||||
}
|
||||
om := byModule[l.Module]
|
||||
if om == nil {
|
||||
om = &layersOfModule{nodes: map[string]map[string]any{}, saidAt: map[string]map[string]string{}}
|
||||
byModule[l.Module] = om
|
||||
names = append(names, l.Module)
|
||||
}
|
||||
om.saidAt[l.Node] = from
|
||||
if l.Node == "" {
|
||||
om.mesh = values
|
||||
} else {
|
||||
om.nodes[l.Node] = values
|
||||
}
|
||||
p.moved = append(p.moved, l)
|
||||
p.answer.Moved = append(p.answer.Moved, said)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
// Which predefined key each module falls back to, so a whole-mesh value goes to mesh config only where no other
|
||||
// module's value moves with it.
|
||||
fallingBack := map[string][]string{}
|
||||
for name, m := range shelf {
|
||||
for _, d := range m.Settings {
|
||||
if d.Predefined != "" && !containsString(fallingBack[d.Predefined], name) {
|
||||
fallingBack[d.Predefined] = append(fallingBack[d.Predefined], name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
m, om := shelf[name], byModule[name]
|
||||
asked := catalogue.AskedKeys(m)
|
||||
keys := map[string]bool{}
|
||||
for k := range om.mesh {
|
||||
keys[k] = true
|
||||
}
|
||||
for _, values := range om.nodes {
|
||||
for k := range values {
|
||||
keys[k] = true
|
||||
}
|
||||
}
|
||||
for _, key := range movedKeys(keys) {
|
||||
d := m.Settings[key]
|
||||
meshValue, meshSays := om.mesh[key]
|
||||
alone := d.Predefined != "" && len(fallingBack[d.Predefined]) == 1
|
||||
if meshSays && alone {
|
||||
p.set(configLevel{Level: "mesh"}, d.Predefined, meshValue, d, om.said("", key))
|
||||
}
|
||||
for _, node := range on[name] {
|
||||
nodeValue, nodeSays := om.nodes[node][key]
|
||||
value := meshValue
|
||||
switch {
|
||||
case nodeSays && meshSays:
|
||||
_, under := meshValue.(map[string]any)
|
||||
_, over := nodeValue.(map[string]any)
|
||||
if under && over && asked[key] {
|
||||
problem("%s on %s lays an object for %q over the whole mesh's, and a placeholder takes it whole "+
|
||||
"where its file merges the two: say the value on %s whole, then move", name, node, key, node)
|
||||
continue
|
||||
}
|
||||
value = catalogue.LaidOver(meshValue, nodeValue)
|
||||
case nodeSays:
|
||||
value = nodeValue
|
||||
case !meshSays:
|
||||
continue
|
||||
}
|
||||
p.expect(node, name, key, value)
|
||||
if nodeSays || !alone {
|
||||
saidBy := om.said(node, key)
|
||||
if meshSays {
|
||||
saidBy = append(om.said("", key), saidBy...)
|
||||
}
|
||||
p.set(configLevel{Level: "module", Node: node, Module: name}, key, value, d, saidBy)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(p.answer.Problems)
|
||||
return p
|
||||
}
|
||||
|
||||
// layersOfModule is one module's layers as the move reads them: its whole-mesh layer's config values, each node's,
|
||||
// and where each key was said, by node ("" for the whole mesh).
|
||||
type layersOfModule struct {
|
||||
mesh map[string]any
|
||||
nodes map[string]map[string]any
|
||||
saidAt map[string]map[string]string
|
||||
}
|
||||
|
||||
func (om *layersOfModule) said(node, key string) []string {
|
||||
if s, ok := om.saidAt[node][key]; ok {
|
||||
return []string{s}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// set is one value the move sets, judged against its declaration as mesh-config will judge it.
|
||||
func (p *configMovePlan) set(level configLevel, key string, value any, d catalogue.SettingDeclaration, from []string) {
|
||||
p.answer.Sets = append(p.answer.Sets, configSet{configLevel: level, Key: key, Value: value, Trusted: d.Trusted,
|
||||
From: from})
|
||||
for _, wrong := range catalogue.ValueProblems(d, value) {
|
||||
p.answer.Problems = append(p.answer.Problems, fmt.Sprintf("%s at %s: %s", key, level, wrong))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *configMovePlan) expect(node, module, key string, value any) {
|
||||
k := node + "." + module
|
||||
if p.expected[k] == nil {
|
||||
p.expected[k] = map[string]any{}
|
||||
}
|
||||
p.expected[k][key] = value
|
||||
}
|
||||
|
||||
func movedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// configService is mesh-config as the move asks it: what it holds at each level, what it resolved, and a value set
|
||||
// as the controller carries it from its terminal.
|
||||
type configService interface {
|
||||
// Values is every value set, by its key in mesh-config_values (`mesh.<key>`, `node.<node>.<key>`,
|
||||
// `module.<node>.<module>.<key>`).
|
||||
Values(ctx context.Context) (map[string]json.RawMessage, error)
|
||||
// Resolved is every resolved value.
|
||||
Resolved(ctx context.Context) ([]resolvedEntry, error)
|
||||
// Set sets one value at its level, by whom and why.
|
||||
Set(ctx context.Context, level configLevel, key string, value any, by, why string) error
|
||||
}
|
||||
|
||||
// runConfigMove is the move against a store and mesh-config: a dry run says what it would set.
|
||||
func runConfigMove(ctx context.Context, inv *inventory.Inventory, config configService, apply bool, by,
|
||||
why string) (configMoveAnswer, error) {
|
||||
layers, holding, err := inv.LayersToMove(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
on := map[string][]string{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
on[e.Manifest.Module] = e.On
|
||||
}
|
||||
p := planConfigMove(layers, shelf, on)
|
||||
if len(holding) > 0 && len(p.answer.Problems) == 0 {
|
||||
p.answer.Problems = append(p.answer.Problems, "a layer still holds placement ("+strings.Join(holding, "; ")+
|
||||
"): run move-placement first")
|
||||
}
|
||||
moved, err := inv.ConfigMoved(ctx)
|
||||
if err != nil {
|
||||
return configMoveAnswer{}, err
|
||||
}
|
||||
p.answer.Switched = moved.Moved
|
||||
if config == nil {
|
||||
return p.answer, errors.New("mesh-config cannot be asked, so nothing was moved")
|
||||
}
|
||||
held, err := config.Values(ctx)
|
||||
if err != nil {
|
||||
return p.answer, fmt.Errorf("what mesh-config holds cannot be read, so nothing was moved: %w", err)
|
||||
}
|
||||
for i := range p.answer.Sets {
|
||||
s := &p.answer.Sets[i]
|
||||
if was, ok := held[s.valueKey(s.Key)]; ok && sameJSON(was, s.Value) {
|
||||
s.Already = true
|
||||
}
|
||||
}
|
||||
p.answer.DryRun = !apply
|
||||
if !apply || len(p.answer.Problems) > 0 {
|
||||
if apply {
|
||||
return p.answer, fmt.Errorf("the move of config refuses, and nothing was set: %s",
|
||||
strings.Join(p.answer.Problems, "; "))
|
||||
}
|
||||
return p.answer, nil
|
||||
}
|
||||
if len(p.moved) == 0 && moved.Moved {
|
||||
return p.answer, nil
|
||||
}
|
||||
for _, s := range p.answer.Sets {
|
||||
if s.Already {
|
||||
continue
|
||||
}
|
||||
said := "moved from the settings layer of " + strings.Join(s.From, " and ") + " (issue 664): " + why
|
||||
if err := config.Set(ctx, s.configLevel, s.Key, s.Value, by, said); err != nil {
|
||||
return p.answer, fmt.Errorf("%s at %s was not set, and no layer was emptied; ask the move again: %w",
|
||||
s.Key, s.configLevel, err)
|
||||
}
|
||||
}
|
||||
// Read back through mesh-config before anything points at it (design 57): every module the move reaches, and
|
||||
// every other in the config form, must resolve exactly what its layers lay today.
|
||||
if differs, err := readsBack(ctx, config, p); err != nil {
|
||||
return p.answer, err
|
||||
} else if len(differs) > 0 {
|
||||
return p.answer, fmt.Errorf("mesh-config does not resolve what the layers lay today, so no layer was emptied and "+
|
||||
"the controller still composes from the layers: %s", strings.Join(differs, "; "))
|
||||
}
|
||||
if err := inv.FinishConfigMove(ctx, p.moved, by, why); err != nil {
|
||||
return p.answer, err
|
||||
}
|
||||
p.answer.Switched = true
|
||||
return p.answer, nil
|
||||
}
|
||||
|
||||
// readsBack compares what mesh-config resolved with what the layers lay, for every module in the config form on
|
||||
// every node it is assigned to, and says each difference. A value the layers do not lay and mesh-config resolves from
|
||||
// a level would be laid once the switch is on: a difference too.
|
||||
func readsBack(ctx context.Context, config configService, p configMovePlan) ([]string, error) {
|
||||
var resolved []resolvedEntry
|
||||
var err error
|
||||
// mesh-config writes the resolved values with each set; a pass that missed one is put right by its next
|
||||
// resolution, so a difference is read again for a little while before it is said.
|
||||
for wait := 0; ; wait++ {
|
||||
if resolved, err = config.Resolved(ctx); err != nil {
|
||||
return nil, fmt.Errorf("mesh-config's resolved values cannot be read back, so no layer was emptied: %w", err)
|
||||
}
|
||||
differs, err := differences(resolved, p)
|
||||
if err != nil || len(differs) == 0 || wait >= readBackTries {
|
||||
return differs, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(readBackEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// How often, and how many times, the move reads mesh-config's resolved values back before it says they differ.
|
||||
var (
|
||||
readBackEvery = 2 * time.Second
|
||||
readBackTries = 15
|
||||
)
|
||||
|
||||
func differences(resolved []resolvedEntry, p configMovePlan) ([]string, error) {
|
||||
var differs []string
|
||||
for _, module := range movedKeys(p.modules) {
|
||||
for _, node := range p.modules[module] {
|
||||
laid, err := configLaid(resolved, node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
want := p.expected[node+"."+module]
|
||||
if want == nil {
|
||||
want = map[string]any{}
|
||||
}
|
||||
for _, k := range movedKeys(mergedKeys(laid, want)) {
|
||||
got, has := laid[k]
|
||||
w, wants := want[k]
|
||||
switch {
|
||||
case !wants:
|
||||
differs = append(differs, fmt.Sprintf("%s on %s resolves %s from a level no layer says; clear it "+
|
||||
"there, or set it in a layer first", module, node, k))
|
||||
case !has:
|
||||
differs = append(differs, fmt.Sprintf("%s on %s does not resolve %s from the value moved", module, node, k))
|
||||
case !reflect.DeepEqual(normalJSON(got), normalJSON(w)):
|
||||
differs = append(differs, fmt.Sprintf("%s on %s resolves %s as %s, and its layers lay %s", module,
|
||||
node, k, compactJSON(got), compactJSON(w)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return differs, nil
|
||||
}
|
||||
|
||||
func mergedKeys(a, b map[string]any) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for k := range a {
|
||||
out[k] = true
|
||||
}
|
||||
for k := range b {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalJSON(v any) any {
|
||||
raw, _ := json.Marshal(v)
|
||||
var out any
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
return out
|
||||
}
|
||||
|
||||
func compactJSON(v any) string {
|
||||
raw, _ := json.Marshal(v)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func sameJSON(raw json.RawMessage, v any) bool {
|
||||
var held any
|
||||
if err := json.Unmarshal(raw, &held); err != nil {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(held, normalJSON(v))
|
||||
}
|
||||
|
||||
// busConfig is mesh-config on this controller's bus: its buckets read directly, a value set through its verb in
|
||||
// the controller's own name, so mesh-config takes a trusted value as given at the controller's terminal (ADR 0348 §8).
|
||||
type busConfig struct{}
|
||||
|
||||
func (busConfig) Values(ctx context.Context) (map[string]json.RawMessage, error) {
|
||||
out := map[string]json.RawMessage{}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, broker.BucketName("mesh-config", "values"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var v struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal(e.Value(), &v); err != nil {
|
||||
return fmt.Errorf("mesh-config holds %q, which is not a value: %w", e.Key(), err)
|
||||
}
|
||||
out[e.Key()] = v.Value
|
||||
}
|
||||
return errors.New("mesh-config's values stopped answering")
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (busConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { return readResolved(ctx) }
|
||||
|
||||
func (busConfig) Set(ctx context.Context, level configLevel, key string, value any, by, why string) error {
|
||||
args := map[string]any{"action": "set", "key": key, "value": value, "why": why, "by": by}
|
||||
if level.Node != "" {
|
||||
args["node"] = level.Node
|
||||
}
|
||||
if level.Module != "" {
|
||||
args["module"] = level.Module
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// moveConfigCommand is `move-config`: the move of config (issue 664).
|
||||
func moveConfigCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("move-config", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
confirm := set.Bool("confirm", false, "set the values and switch, rather than only say what would move")
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("move-config [--json] [--confirm --why <text>]")
|
||||
}
|
||||
if !*confirm && f.given() {
|
||||
return errors.New("move-config takes --why only with --confirm: without it nothing is moved. Nothing was done")
|
||||
}
|
||||
if *confirm {
|
||||
if err := f.require("move-config"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if *confirm {
|
||||
f.record(ctx, "move-config", nil)
|
||||
}
|
||||
why := ""
|
||||
if *confirm {
|
||||
why = strings.TrimSpace(*f.why)
|
||||
}
|
||||
answer, err := runConfigMove(ctx, open.inventory, busConfig{}, *confirm, "the move of config, run by "+link.Caller(), why)
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if encErr := encoder.Encode(answer); encErr != nil {
|
||||
return encErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Print(describeConfigMove(answer))
|
||||
return err
|
||||
}
|
||||
|
||||
// describeConfigMove says the answer as a person reads it.
|
||||
func describeConfigMove(a configMoveAnswer) string {
|
||||
var b strings.Builder
|
||||
verb := "set"
|
||||
if a.DryRun {
|
||||
verb = "would set"
|
||||
}
|
||||
for _, s := range a.Sets {
|
||||
state := verb
|
||||
if s.Already {
|
||||
state = "holds already"
|
||||
}
|
||||
fmt.Fprintf(&b, "%s %s = %s at %s (from %s)\n", state, s.Key, compactJSON(s.Value), s.configLevel,
|
||||
strings.Join(s.From, " and "))
|
||||
}
|
||||
for _, s := range a.Unread {
|
||||
fmt.Fprintf(&b, "emptied, set nowhere: %s\n", s)
|
||||
}
|
||||
for _, s := range a.Stays {
|
||||
fmt.Fprintf(&b, "stays: %s\n", s)
|
||||
}
|
||||
for _, s := range a.Problems {
|
||||
fmt.Fprintf(&b, "refused: %s\n", s)
|
||||
}
|
||||
switch {
|
||||
case len(a.Problems) > 0:
|
||||
b.WriteString("nothing was set: the move refuses until each is put right\n")
|
||||
case a.DryRun:
|
||||
fmt.Fprintf(&b, "%d layer(s), %d value(s); nothing was set — with --confirm --why <text> they are\n",
|
||||
len(a.Moved), len(a.Sets))
|
||||
case a.Switched:
|
||||
fmt.Fprintf(&b, "%d layer(s) moved into mesh-config, each kept whole in its history first; the controller "+
|
||||
"reads mesh-config where it read them\n", len(a.Moved))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,785 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
snapshot "github.com/novox/mesh-controller/internal/facts"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four
|
||||
// machines as anchor, laptop, workstation and desktop, the whole mesh's with no node; personal values stood in for,
|
||||
// and the placement words left out, which move-placement moves first — with each module in the config form shaped
|
||||
// as its definition on the feature branch: its config, its files (templates inlined) and directories, placeholders
|
||||
// of kinds this test's mesh does not hold stood in for. A module still in the settings form (the media catalogue's,
|
||||
// and kernel) stands in as a mergeable file that takes every key. `on` is where each is assigned, as live.
|
||||
type configuredMesh struct {
|
||||
Modules []json.RawMessage `json:"modules"`
|
||||
Layers []configuredLayer `json:"layers"`
|
||||
On map[string][]string `json:"on"`
|
||||
// Predefined is the catalogue's list of predefined keys, which a module falls back to.
|
||||
Predefined json.RawMessage `json:"predefined"`
|
||||
}
|
||||
|
||||
type configuredLayer struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Values map[string]any `json:"values"`
|
||||
}
|
||||
|
||||
var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"}
|
||||
|
||||
// aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and
|
||||
// its layers held as the store holds them once move-placement has run.
|
||||
func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for i, name := range []string{"workstation", "desktop"} {
|
||||
record, err := inv.AddNode(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "container-runtime", "present": true},
|
||||
map[string]any{"name": "systemd", "present": true}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w configuredMesh
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := catalogue.ParsePredefined(w.Predefined)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalogue.UsePredefined(list)
|
||||
served := map[string]map[string]any{}
|
||||
for _, text := range w.Modules {
|
||||
var shape map[string]json.RawMessage
|
||||
if err := json.Unmarshal(text, &shape); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var name string
|
||||
_ = json.Unmarshal(shape["module"], &name)
|
||||
if s, ok := shape["serves-checked"]; ok {
|
||||
var serves map[string]any
|
||||
if err := json.Unmarshal(s, &serves); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
served[name] = serves
|
||||
delete(shape, "serves-checked")
|
||||
}
|
||||
text, err := json.Marshal(shape)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(text)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, l := range w.Layers {
|
||||
values, err := json.Marshal(l.Values)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l.Node == "" {
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values))
|
||||
continue
|
||||
}
|
||||
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
|
||||
l.Node, l.Module, string(values))
|
||||
}
|
||||
for _, module := range movedKeys(w.On) {
|
||||
for _, node := range w.On[module] {
|
||||
if _, err := assign(ctx, open, node, module); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", node, module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return open, w, served
|
||||
}
|
||||
|
||||
// standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue
|
||||
// the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node
|
||||
// config then mesh config for the predefined key it falls back to, else its default; a required value nothing sets is
|
||||
// unset.
|
||||
type standInMeshConfig struct {
|
||||
open *stores
|
||||
values map[string]any
|
||||
sets []string
|
||||
// asked is each call of its verb: the action and the level's key.
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Values(context.Context) (map[string]json.RawMessage, error) {
|
||||
out := map[string]json.RawMessage{}
|
||||
for k, v := range c.values {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[k] = raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Set(_ context.Context, level configLevel, key string, value any, by, why string) error {
|
||||
if by == "" || why == "" {
|
||||
return fmt.Errorf("%s at %s is set without who or why", key, level)
|
||||
}
|
||||
c.values[level.valueKey(key)] = normalJSON(value)
|
||||
c.sets = append(c.sets, level.valueKey(key))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *standInMeshConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) {
|
||||
entries, err := c.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []resolvedEntry
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
if !m.DeclaresConfig() {
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
for _, key := range movedKeys(m.Settings) {
|
||||
d := m.Settings[key]
|
||||
r := resolvedEntry{Node: node, Module: m.Module, Key: key, Source: "default"}
|
||||
value, has := c.values["module."+node+"."+m.Module+"."+key]
|
||||
switch {
|
||||
case has:
|
||||
r.Source = "module"
|
||||
case d.Predefined != "" && c.values["node."+node+"."+d.Predefined] != nil:
|
||||
value, r.Source = c.values["node."+node+"."+d.Predefined], "node"
|
||||
case d.Predefined != "" && c.values["mesh."+d.Predefined] != nil:
|
||||
value, r.Source = c.values["mesh."+d.Predefined], "mesh"
|
||||
case d.Required:
|
||||
r.Source = "unset"
|
||||
default:
|
||||
value = d.Default
|
||||
}
|
||||
if r.Source != "unset" {
|
||||
if r.Value, err = json.Marshal(value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// laid is the stand-in as composition reads it once the switch is on (inventory.ConfigValues).
|
||||
func (c *standInMeshConfig) laid(ctx context.Context, node, module string) (map[string]any, error) {
|
||||
resolved, err := c.Resolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return configLaid(resolved, node, module)
|
||||
}
|
||||
|
||||
// snapshotFacts is the facts snapshot as gatherFacts gives it.
|
||||
type snapshotFacts = snapshot.Facts
|
||||
|
||||
// toldEveryMachine is what each of the four machines is told — every resource and the filter's rules — and what
|
||||
// each module that serves something serves, settled from its layers, as JSON a person can diff.
|
||||
func toldEveryMachine(t *testing.T, open *stores, served map[string]map[string]any, on map[string][]string) map[string]string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
told := map[string]string{}
|
||||
for _, node := range fourMachines {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
composed, err := plan.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
rules, err := plan.Rules(with)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
settles := map[string]any{}
|
||||
for module, serves := range served {
|
||||
if !containsString(on[module], node) {
|
||||
continue
|
||||
}
|
||||
layers, err := open.inventory.SettingsFor(ctx, node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, s := range serves {
|
||||
fact, _ := s.(map[string]any)
|
||||
settled, err := catalogue.Settle(fact, catalogue.WithDefaults(shelf[module], layers))
|
||||
if err != nil {
|
||||
settles[module+" serves "+name] = "error: " + err.Error()
|
||||
continue
|
||||
}
|
||||
settles[module+" serves "+name] = settled
|
||||
}
|
||||
}
|
||||
got, err := json.MarshalIndent(map[string]any{"resources": composed.Resources, "left-out": composed.LeftOut,
|
||||
"rules": rules, "served": settles}, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
told[node] = string(got)
|
||||
}
|
||||
return told
|
||||
}
|
||||
|
||||
// effectiveEverywhere is each module's every value on each machine it is assigned to, as `settings show` and
|
||||
// `settings preferences` say them (catalogue.Effective): the value, never where it came from, which the move
|
||||
// changes by design; a key a layer says by its file's name is read as its config key.
|
||||
func effectiveEverywhere(t *testing.T, open *stores, w configuredMesh) map[string]any {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]any{}
|
||||
for _, module := range movedKeys(w.On) {
|
||||
if !shelf[module].DeclaresConfig() {
|
||||
continue
|
||||
}
|
||||
for _, node := range w.On[module] {
|
||||
layers, err := open.inventory.SettingsFor(ctx, node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A layer's old spelling of a key (nodered's timeZone) is listed beside the key it is the file's name
|
||||
// for, and reaches the file as that key does (issue 751): read as that key, which the move sets.
|
||||
as := catalogue.TakenAs(shelf[module])
|
||||
for _, v := range catalogue.Effective(shelf[module], layers) {
|
||||
if key, old := as[v.Key]; old {
|
||||
out[node+"."+module+"."+key] = normalJSON(v.Value)
|
||||
continue
|
||||
}
|
||||
if _, said := out[node+"."+module+"."+v.Key]; said && v.FromDefault {
|
||||
continue
|
||||
}
|
||||
out[node+"."+module+"."+v.Key] = normalJSON(v.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func setAt(a configMoveAnswer, key string) []configSet {
|
||||
var out []configSet
|
||||
for _, s := range a.Sets {
|
||||
if s.valueKey(s.Key) == key {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **The settings layers move into mesh-config and no machine is told anything new** (novox/hq ADR 0348 §9, design
|
||||
// 57 K5, issue 664). Every live layer is seeded as the store holds it once move-placement has run, on modules shaped
|
||||
// as their definitions, and every machine composed. The move, asked without confirming, says each value it would set
|
||||
// and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it
|
||||
// moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same,
|
||||
// and `settings` lists the same values `config` resolves. Asked again, it sets nothing.
|
||||
func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
before := toldEveryMachine(t, open, served, w.On)
|
||||
listedBefore := effectiveEverywhere(t, open, w)
|
||||
factsBefore, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "")
|
||||
if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 {
|
||||
t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets)
|
||||
}
|
||||
// nodered's live layer says the file's name for its time zone: set under the config key (issue 751).
|
||||
if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" ||
|
||||
!strings.HasSuffix(s[0].From[0], ": timeZone") {
|
||||
t.Errorf("nodered's time zone moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Key == "timeZone" {
|
||||
t.Errorf("a key is set under the file's name: %+v", s)
|
||||
}
|
||||
}
|
||||
// claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's
|
||||
// consequences), and its whole-mesh layer sets no module value.
|
||||
if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" {
|
||||
t.Errorf("claude-code's managed settings move as %+v", s)
|
||||
}
|
||||
for _, node := range fourMachines {
|
||||
if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 {
|
||||
t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s)
|
||||
}
|
||||
}
|
||||
// A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that
|
||||
// machine's layer: power's on the workstation is the workstation's, its own elsewhere.
|
||||
if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" {
|
||||
t.Errorf("power's key on the workstation moves as %+v", s)
|
||||
}
|
||||
if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" {
|
||||
t.Errorf("power's key on the anchor moves as %+v", s)
|
||||
}
|
||||
for _, s := range asked.Sets {
|
||||
if s.Level == "node" {
|
||||
t.Errorf("a machine's layer became node config: %+v", s)
|
||||
}
|
||||
}
|
||||
var stays []string
|
||||
for _, s := range asked.Stays {
|
||||
stays = append(stays, strings.Fields(s)[0])
|
||||
}
|
||||
sort.Strings(stays)
|
||||
if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi",
|
||||
"plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") {
|
||||
t.Errorf("the layers that stay are %v, want those of %v", stays, want)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("a dry run changed what a machine is told")
|
||||
}
|
||||
|
||||
moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered")
|
||||
if err != nil || moved.DryRun || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if len(config.sets) != len(asked.Sets) {
|
||||
t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets))
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select module from settings s where values <> '{}'
|
||||
and module in (select name from module where manifest ? 'config')`); len(held) != 0 {
|
||||
t.Fatalf("a layer of a module in the config form still holds values: %v", held)
|
||||
}
|
||||
after := toldEveryMachine(t, open, served, w.On)
|
||||
for _, node := range fourMachines {
|
||||
if after[node] != before[node] {
|
||||
t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node])
|
||||
}
|
||||
}
|
||||
if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) {
|
||||
t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore)
|
||||
}
|
||||
// And config lists what settings listed: each module's resolved value on each machine is the value settings
|
||||
// said before the move.
|
||||
resolved, err := config.Resolved(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range resolved {
|
||||
said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key]
|
||||
if r.Source == "unset" {
|
||||
if listed && said != nil {
|
||||
t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var v any
|
||||
_ = json.Unmarshal(r.Value, &v)
|
||||
if !listed || !reflect.DeepEqual(v, said) {
|
||||
t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said)
|
||||
}
|
||||
}
|
||||
|
||||
// The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and
|
||||
// each machine's declaration composes to the digest it did before the move.
|
||||
factsAfter, err := gatherFacts(ctx, open, "2.11.17")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Machines are named in a snapshot by stand-ins of its own; each is found by what it composes.
|
||||
digests := func(f snapshotFacts) ([]string, string) {
|
||||
var out []string
|
||||
zone := ""
|
||||
for _, m := range f.Machines {
|
||||
out = append(out, m.Declaration.Digest)
|
||||
for _, l := range m.Settings {
|
||||
if l.Module == "nodered" {
|
||||
zone = compactJSON(l.Values["time-zone"])
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, zone
|
||||
}
|
||||
was, _ := digests(factsBefore)
|
||||
now, zone := digests(factsAfter)
|
||||
if zone != `"Europe/Brussels"` {
|
||||
t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone)
|
||||
}
|
||||
if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) {
|
||||
t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was)
|
||||
}
|
||||
for _, s := range factsAfter.Settings {
|
||||
if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig &&
|
||||
len(s.Values) > 0 {
|
||||
t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values)
|
||||
}
|
||||
}
|
||||
|
||||
config.sets = nil
|
||||
again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again")
|
||||
if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 {
|
||||
t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err)
|
||||
}
|
||||
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("the move asked again changed what a machine is told")
|
||||
}
|
||||
}
|
||||
|
||||
// **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds
|
||||
// placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses —
|
||||
// each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch
|
||||
// is on) leaves every layer and the switch as they were.
|
||||
func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}'
|
||||
where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`)
|
||||
answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || answer.Switched || len(config.sets) > 0 {
|
||||
t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets)
|
||||
}
|
||||
for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first",
|
||||
`unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`)
|
||||
inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`)
|
||||
inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`)
|
||||
|
||||
// Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it.
|
||||
config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration {
|
||||
shelf, _ := inv.Catalogue(ctx)
|
||||
return shelf["grafana"].Settings
|
||||
}())[0]] = "set before the move"
|
||||
answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
|
||||
if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched {
|
||||
t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err)
|
||||
}
|
||||
if moved, _ := inv.ConfigMoved(ctx); moved.Moved {
|
||||
t.Fatal("the switch is on after a refused move")
|
||||
}
|
||||
if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" {
|
||||
t.Fatal("a refused move emptied the layers")
|
||||
}
|
||||
}
|
||||
|
||||
// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of
|
||||
// the release, never a repair (issue 664).
|
||||
func TestTheMoveOfConfigIsADryRunUnlessConfirmed(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "move-config --json"},
|
||||
{map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-config --json --confirm --why release 57 is delivered"},
|
||||
} {
|
||||
argv, err := argvFor("move-config", c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{{"confirm": "true"}, {"why": "tidy"}, {"node": "anchor"}} {
|
||||
if argv, err := argvFor("move-config", args); err == nil {
|
||||
t.Errorf("%v was composed as %q", args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"move-config", "--json", "--confirm", "--why", "x"}) != "move-config" {
|
||||
t.Error("a confirmed move through the generic verb would go unrecorded")
|
||||
}
|
||||
if !personsDecision(link.HandAct{Verb: "move-config"}) {
|
||||
t.Error("a move of the release would count toward a healer the mesh lacks")
|
||||
}
|
||||
}
|
||||
|
||||
// ask is mesh-config's verb as the stand-in answers it: list, set, clear and propose.
|
||||
func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
str := func(k string) string { s, _ := args[k].(string); return s }
|
||||
level := configLevel{Level: "mesh"}
|
||||
switch {
|
||||
case str("module") != "" && str("node") != "":
|
||||
level = configLevel{Level: "module", Node: str("node"), Module: str("module")}
|
||||
case str("node") != "":
|
||||
level = configLevel{Level: "node", Node: str("node")}
|
||||
}
|
||||
c.asked = append(c.asked, str("action")+" "+level.valueKey(str("key")))
|
||||
switch str("action") {
|
||||
case "list":
|
||||
resolved, err := c.Resolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]any{"resolved": []resolvedEntry{}, "set": []any{}}
|
||||
var listed []resolvedEntry
|
||||
for _, r := range resolved {
|
||||
if r.Module == str("module") && (str("node") == "" || r.Node == str("node")) {
|
||||
listed = append(listed, r)
|
||||
}
|
||||
}
|
||||
out["resolved"] = listed
|
||||
var set []any
|
||||
for _, k := range movedKeys(c.values) {
|
||||
parts := strings.SplitN(k, ".", 4)
|
||||
if len(parts) == 4 && parts[0] == "module" && parts[2] == str("module") {
|
||||
set = append(set, map[string]any{"level": configLevel{Level: "module", Node: parts[1], Module: parts[2]},
|
||||
"key": parts[3], "value": c.values[k], "by": "a test"})
|
||||
}
|
||||
}
|
||||
out["set"] = set
|
||||
return json.Marshal(out)
|
||||
case "set":
|
||||
if str("why") == "" || str("by") == "" {
|
||||
return nil, fmt.Errorf("a set without why or by: %v", args)
|
||||
}
|
||||
return nil, c.Set(ctx, level, str("key"), args["value"], str("by"), str("why"))
|
||||
case "clear":
|
||||
if _, had := c.values[level.valueKey(str("key"))]; !had {
|
||||
return nil, fmt.Errorf("nothing is set for %s", str("key"))
|
||||
}
|
||||
delete(c.values, level.valueKey(str("key")))
|
||||
return json.RawMessage(`{}`), nil
|
||||
case "propose":
|
||||
return json.RawMessage(`{"proposal":"p-1"}`), nil
|
||||
}
|
||||
return nil, fmt.Errorf("the stand-in does not %s", str("action"))
|
||||
}
|
||||
|
||||
// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664): once the move
|
||||
// has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the
|
||||
// machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a
|
||||
// verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb.
|
||||
func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) {
|
||||
open, w, served := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
savedRead, savedAsk := readConfigValues, configAsk
|
||||
readConfigValues, configAsk = config.laid, config.ask
|
||||
t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk })
|
||||
if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) })
|
||||
for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`,
|
||||
"mesh-config.config", `"mqtt": {`} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("settings show does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
if strings.Contains(shown, "timeZone") {
|
||||
t.Errorf("settings show still says the file's name:\n%s", shown)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
said := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") ||
|
||||
containsString(config.asked, "set module.workstation.dunst.width") {
|
||||
t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said)
|
||||
}
|
||||
told := toldEveryMachine(t, open, served, w.On)["workstation"]
|
||||
if !strings.Contains(told, "12") {
|
||||
t.Error("the workstation is not told the font size set through settings")
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "would no longer set width") {
|
||||
t.Fatalf("a set dropping a value without --replace: %v", err)
|
||||
}
|
||||
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil ||
|
||||
!strings.Contains(err.Error(), "no own value for the whole mesh") {
|
||||
t.Fatalf("a module value for the whole mesh: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv(verbVar, "settings")
|
||||
for _, c := range []struct{ module, node, values, key, held, why string }{
|
||||
// A file root reads: refused as before the move (novox/hq issue 339).
|
||||
{"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"},
|
||||
{"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`,
|
||||
"module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"},
|
||||
} {
|
||||
err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node})
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held {
|
||||
t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key])
|
||||
}
|
||||
}
|
||||
// And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's
|
||||
// word for it (ADR 0348 §8).
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "ADR 0348 §8") {
|
||||
t.Fatalf("a trusted value through a verb: %v", err)
|
||||
}
|
||||
if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil {
|
||||
t.Fatalf("a value nobody trusts through a verb: %v", err)
|
||||
}
|
||||
stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"})
|
||||
})
|
||||
if config.values["module.workstation.dunst.width"] != float64(600) {
|
||||
t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"])
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
|
||||
stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) })
|
||||
for _, k := range []string{"font-size", "width"} {
|
||||
if _, held := config.values["module.workstation.dunst."+k]; held {
|
||||
t.Errorf("settings clear left dunst's %s on the workstation", k)
|
||||
}
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
proposed := stdoutOf(t, func() error {
|
||||
return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"})
|
||||
})
|
||||
if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") {
|
||||
t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed)
|
||||
}
|
||||
|
||||
config.asked = nil
|
||||
kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) })
|
||||
if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 {
|
||||
t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked)
|
||||
}
|
||||
}
|
||||
|
||||
// **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348):
|
||||
// a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and
|
||||
// approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module
|
||||
// would compose from emptied layers everywhere.
|
||||
func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0]
|
||||
values := map[string]any{key: "raised before the move"}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Fatalf("grafana does not read config after the move: %v %v", reads, err)
|
||||
}
|
||||
_, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator")
|
||||
if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") ||
|
||||
!strings.Contains(err.Error(), "propose it again") {
|
||||
t.Fatalf("the stale approval: %v", err)
|
||||
}
|
||||
if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 {
|
||||
t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the
|
||||
// old path on purpose, so refusing it would loop the operator (refused, proposed again, refused).
|
||||
func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) {
|
||||
open, _, _ := aConfiguredMesh(t)
|
||||
ctx := t.Context()
|
||||
readBackTries = 0
|
||||
t.Cleanup(func() { readBackTries = 15 })
|
||||
inv := open.inventory
|
||||
config := &standInMeshConfig{open: open, values: map[string]any{}}
|
||||
inv.ReadsConfigFrom(config.laid)
|
||||
values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}}
|
||||
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
|
||||
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
|
||||
}
|
||||
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
|
||||
t.Fatalf("the move says %+v (%v)", moved, err)
|
||||
}
|
||||
if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil {
|
||||
t.Fatalf("the placement-only approval after the move: %v", err)
|
||||
}
|
||||
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has {
|
||||
t.Errorf("the approval did not set the layer: %v %v", has, err)
|
||||
}
|
||||
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
|
||||
t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Where the controller reads config values once the move has run** (novox/hq ADR 0348 §9, design 57 K5, issue 664):
|
||||
// mesh-config_resolved, the bucket mesh-config alone writes, keyed `<node>.<module>.<key>`, each value with the
|
||||
// source it came from. Read only for a module the switch reaches (inventory.ReadsConfig), so a controller before the
|
||||
// move never asks the bus for config. The exception to ADR 0348 §9 ("the controller holds no config value") that
|
||||
// ADR 0349 clause 17's old way needs, until design 57's R1 hands every module to the provisioner.
|
||||
|
||||
// resolvedEntry is one key of mesh-config_resolved as the controller reads it: the value and where it came from.
|
||||
type resolvedEntry struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// The sources a resolved value is laid over a module's files from: mesh config, node config and the module's own
|
||||
// value. Its manifest's default is composition's already, and a taken key's default is never laid over its file;
|
||||
// a required value nothing sets (`unset`) is refused by composition by name, as it is today.
|
||||
var laidSources = map[string]bool{"mesh": true, "node": true, "module": true}
|
||||
|
||||
// configLaid is the values of one module on one node that are laid over its files, from the resolved entries.
|
||||
func configLaid(entries []resolvedEntry, node, module string) (map[string]any, error) {
|
||||
out := map[string]any{}
|
||||
for _, e := range entries {
|
||||
if e.Node != node || e.Module != module || !laidSources[e.Source] {
|
||||
continue
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(e.Value, &v); err != nil {
|
||||
return nil, fmt.Errorf("mesh-config's value %s of %s on %s cannot be read: %w", e.Key, module, node, err)
|
||||
}
|
||||
out[e.Key] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// resolvedRead is what one read of the whole bucket held, and when it was read.
|
||||
type resolvedRead struct {
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
entries []resolvedEntry
|
||||
}
|
||||
|
||||
// resolvedFresh is how long one read of the bucket answers: a plan reads every module of a machine, and one read
|
||||
// serves them all; a value set since is read by the next composition.
|
||||
const resolvedFresh = 5 * time.Second
|
||||
|
||||
var theResolved resolvedRead
|
||||
|
||||
// readConfigValues is where an opened inventory reads config values once the switch is on; a test replaces it.
|
||||
var readConfigValues inventory.ConfigValues = busConfigValues
|
||||
|
||||
// busConfigValues reads one module's config values on one node from mesh-config_resolved (inventory.ConfigValues).
|
||||
func busConfigValues(ctx context.Context, node, module string) (map[string]any, error) {
|
||||
theResolved.mu.Lock()
|
||||
defer theResolved.mu.Unlock()
|
||||
if theResolved.entries == nil || time.Since(theResolved.at) > resolvedFresh {
|
||||
entries, err := readResolved(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
theResolved.entries, theResolved.at = entries, time.Now()
|
||||
}
|
||||
return configLaid(theResolved.entries, node, module)
|
||||
}
|
||||
|
||||
// readResolved is every entry of mesh-config_resolved, read on the bus.
|
||||
func readResolved(ctx context.Context) ([]resolvedEntry, error) {
|
||||
entries := []resolvedEntry{}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, broker.ConfigResolvedBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s cannot be opened: %w", broker.ConfigResolvedBucket, err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%s was not read whole: %w", broker.ConfigResolvedBucket, ctx.Err())
|
||||
case e, open := <-w.Updates():
|
||||
if !open {
|
||||
return errors.New(broker.ConfigResolvedBucket + " stopped answering")
|
||||
}
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var r resolvedEntry
|
||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
||||
return fmt.Errorf("%s holds %q, which is not a resolved value: %w", broker.ConfigResolvedBucket,
|
||||
e.Key(), err)
|
||||
}
|
||||
if r.Node == "" || r.Module == "" || r.Key == "" || !strings.HasPrefix(e.Key(), r.Node+"."+r.Module+".") {
|
||||
return fmt.Errorf("%s holds %q, whose value names another key", broker.ConfigResolvedBucket, e.Key())
|
||||
}
|
||||
entries = append(entries, r)
|
||||
}
|
||||
}
|
||||
})
|
||||
return entries, err
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdh"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The core's own credentials, sealed by the vault (novox/hq ADR 0349 §16, issue 755; the lead's decision on issue 755,
|
||||
// notes 4 and 5).
|
||||
//
|
||||
// For a machine whose engine opens the vault's box (link.VaultBoxContract), each own secret of a module on the core's
|
||||
// list (catalogue.CoreCredentialModules) travels as the vault sealed it to that machine's key, which the controller
|
||||
// asks for on the send path (`get`, answered to the controller alone and sealed only to the key that machine's engine
|
||||
// stated: mesh-vault's core.go) and carries without being able to open it. A machine below that contract is sent the
|
||||
// controller's own sealed value, exactly as before.
|
||||
//
|
||||
// **The vault seals every answer afresh**, so the box last got is kept (inventory.CoreBox) and carried again while the
|
||||
// vault's generation and the machine's key are the same: a declaration that changed with every answer would never be
|
||||
// current. Every question but a send reads the kept box and asks nothing.
|
||||
//
|
||||
// **When the vault does not answer, the controller never falls back to its own seal.** A credential with a kept box
|
||||
// keeps it — the machine keeps the file it has — and one with none leaves its module out of the declaration (the
|
||||
// machine keeps what it holds of it, and a module not there yet is not placed). Either way the send raises a
|
||||
// condition naming the vault and the machine, cleared by the first send the vault answers whole.
|
||||
|
||||
// engineOpensVaultBox says whether a machine's node-engine opens the vault's box, by its own newest statement.
|
||||
func engineOpensVaultBox(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.VaultBoxContract, nil
|
||||
}
|
||||
|
||||
// coreAnswer is the vault's `get` answer as the controller reads it: the name, the generation and the box, never
|
||||
// opened.
|
||||
type coreAnswer struct {
|
||||
Name string `json:"name"`
|
||||
Generation int `json:"generation"`
|
||||
Sealed json.RawMessage `json:"sealed"`
|
||||
}
|
||||
|
||||
// coreAsk asks the vault for one secret sealed to the given key.
|
||||
type coreAsk func(name, publicKey string) (coreAnswer, error)
|
||||
|
||||
// coreBoxes is what the vault's boxes make of one machine's composition.
|
||||
type coreBoxes struct {
|
||||
// sealed is each core credential's box, by module and name (catalogue.Rendering.VaultSealed).
|
||||
sealed map[string]map[string]json.RawMessage
|
||||
// notPlaced is each core module left out, with why (catalogue.Rendering.NotPlaced).
|
||||
notPlaced map[string]string
|
||||
// keep is each box newly answered, to keep.
|
||||
keep []inventory.CoreBox
|
||||
// held is each credential the vault did not answer for whose kept box is carried again; silent is why the vault
|
||||
// did not answer, per credential.
|
||||
held, silent []string
|
||||
}
|
||||
|
||||
// coreModulesOf is every module of the plan on the core's list whose own secrets this machine's declaration carries,
|
||||
// with their plain own secret names, sorted: one handed to the provisioner has its secrets from the vault through the
|
||||
// client, and one issued outside the mesh is never the vault's to seal here.
|
||||
func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest) bool) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, m := range modules {
|
||||
if !catalogue.CarriesVaultSealed(m.Module) || (handed != nil && handed(m)) {
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
if own.IssuedBy != catalogue.IssuedOutside {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
sort.Strings(names)
|
||||
out[m.Module] = names
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sealingKeyPEM is a machine's sealing key, recorded raw in base64, as the SPKI PEM the vault seals to.
|
||||
func sealingKeyPEM(raw string) (string, error) {
|
||||
b, err := base64.StdEncoding.DecodeString(strings.TrimSpace(raw))
|
||||
if err != nil || len(b) != 32 {
|
||||
return "", errors.New("the machine's sealing key is not a raw X25519 key")
|
||||
}
|
||||
pub, err := ecdh.X25519().NewPublicKey(b)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
der, err := x509.MarshalPKIXPublicKey(pub)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil
|
||||
}
|
||||
|
||||
// coreBoxesOf decides each core credential of a machine: on a send, the vault's answer (the kept box while its
|
||||
// generation and key are the same), the kept box where the vault did not answer, or the module left out where nothing
|
||||
// was kept; on any other question, the kept box, or a stand-in the next send asks for. Pure: what to keep and what to
|
||||
// say is answered, not done.
|
||||
func coreBoxesOf(node, sealingKey string, core map[string][]string, kept map[string]map[string]inventory.CoreBox,
|
||||
choosing Choosing, ask coreAsk) coreBoxes {
|
||||
out := coreBoxes{sealed: map[string]map[string]json.RawMessage{}, notPlaced: map[string]string{}}
|
||||
modules := make([]string, 0, len(core))
|
||||
for m := range core {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
if sealingKey == "" {
|
||||
out.notPlaced[module] = fmt.Sprintf("%s has no sealing key, so the vault can seal %s's credentials to "+
|
||||
"nothing there (novox/hq ADR 0349 §16)", node, module)
|
||||
continue
|
||||
}
|
||||
key, err := sealingKeyPEM(sealingKey)
|
||||
if err != nil {
|
||||
out.notPlaced[module] = fmt.Sprintf("%s's credentials cannot be asked of the vault: %v", module, err)
|
||||
continue
|
||||
}
|
||||
boxes := map[string]json.RawMessage{}
|
||||
for _, name := range core[module] {
|
||||
k, has := kept[module][name]
|
||||
valid := has && k.SealedTo == sealingKey && len(k.Box) > 0
|
||||
if choosing != Allocating {
|
||||
if valid {
|
||||
boxes[name] = k.Box
|
||||
} else {
|
||||
boxes[name] = catalogue.ForeseenVaultBox
|
||||
}
|
||||
continue
|
||||
}
|
||||
secret := "own/" + node + "/" + module + "/" + name
|
||||
got, err := ask(secret, key)
|
||||
if err == nil && (got.Generation <= 0 || canonicalBox(got.Sealed) == nil) {
|
||||
err = errors.New("it answered no box")
|
||||
}
|
||||
if err != nil {
|
||||
out.silent = append(out.silent, fmt.Sprintf("%s: %v", secret, err))
|
||||
if valid {
|
||||
boxes[name] = k.Box
|
||||
out.held = append(out.held, module+"/"+name)
|
||||
continue
|
||||
}
|
||||
out.notPlaced[module] = fmt.Sprintf("the vault did not give %s's credential %s for %s (%v), and the "+
|
||||
"controller never seals it itself (novox/hq ADR 0349 §16): not placed until the vault answers",
|
||||
module, name, node, err)
|
||||
break
|
||||
}
|
||||
if valid && k.Generation == got.Generation {
|
||||
boxes[name] = k.Box
|
||||
continue
|
||||
}
|
||||
box := canonicalBox(got.Sealed)
|
||||
boxes[name] = box
|
||||
out.keep = append(out.keep, inventory.CoreBox{Module: module, Name: name, Generation: got.Generation,
|
||||
SealedTo: sealingKey, Box: box})
|
||||
}
|
||||
if _, left := out.notPlaced[module]; !left {
|
||||
out.sealed[module] = boxes
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// canonicalBox is the vault's box in the one form the declaration carries — the sealed box's fields, in their order,
|
||||
// and nothing else — or nil when it is not one.
|
||||
func canonicalBox(raw json.RawMessage) json.RawMessage {
|
||||
var b struct {
|
||||
V int `json:"v"`
|
||||
Eph string `json:"eph"`
|
||||
IV string `json:"iv"`
|
||||
Tag string `json:"tag"`
|
||||
Ct string `json:"ct"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &b); err != nil || b.V <= 0 || b.Eph == "" || b.Ct == "" {
|
||||
return nil
|
||||
}
|
||||
out, _ := json.Marshal(b)
|
||||
return out
|
||||
}
|
||||
|
||||
// coreVault is how a send asks the vault for a core credential: its `get`, in the controller's name. A test stands
|
||||
// in for it.
|
||||
var coreVault = func(ctx context.Context) coreAsk {
|
||||
vault := theVault(ctx)
|
||||
return func(name, publicKey string) (coreAnswer, error) {
|
||||
raw, err := vault("get", map[string]any{"name": name, "public_key": publicKey})
|
||||
if err != nil {
|
||||
return coreAnswer{}, err
|
||||
}
|
||||
var a coreAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return coreAnswer{}, fmt.Errorf("the vault's answer is not one: %w", err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// coreBoxesFor is coreBoxesOf for one machine, read from and kept in the store, and said on a send.
|
||||
func coreBoxesFor(ctx context.Context, inv *inventory.Inventory, node string, modules []catalogue.Manifest,
|
||||
handed func(catalogue.Manifest) bool, choosing Choosing) (coreBoxes, error) {
|
||||
core := coreModulesOf(modules, handed)
|
||||
if len(core) == 0 {
|
||||
if choosing == Allocating {
|
||||
settleVaultSilent(ctx, node)
|
||||
}
|
||||
return coreBoxes{}, nil
|
||||
}
|
||||
key, err := inv.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
kept, err := inv.KeptCoreBoxes(ctx, node)
|
||||
if err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
var ask coreAsk
|
||||
if choosing == Allocating {
|
||||
ask = coreVault(ctx)
|
||||
}
|
||||
out := coreBoxesOf(node, key, core, kept, choosing, ask)
|
||||
for _, b := range out.keep {
|
||||
if err := inv.KeepCoreBox(ctx, node, b); err != nil {
|
||||
return coreBoxes{}, err
|
||||
}
|
||||
}
|
||||
if choosing == Allocating {
|
||||
sayVaultSilent(ctx, node, out)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// kindVaultSilent is a send that did not have the vault's box for a core credential of a machine.
|
||||
const kindVaultSilent = "vault-silent"
|
||||
|
||||
// vaultSilentObservation is that, as a condition: `machine.<machine>.vault-silent`.
|
||||
func vaultSilentObservation(node string, b coreBoxes) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent,
|
||||
Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource}
|
||||
var said []string
|
||||
if len(b.notPlaced) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
var left []string
|
||||
for m := range b.notPlaced {
|
||||
left = append(left, m)
|
||||
}
|
||||
sort.Strings(left)
|
||||
said = append(said, "not placed: "+strings.Join(left, ", "))
|
||||
}
|
||||
if len(b.held) > 0 {
|
||||
said = append(said, "kept as last sent: "+strings.Join(b.held, ", "))
|
||||
}
|
||||
o.Summary = fmt.Sprintf("the vault did not answer for the core's credentials on %s; %s", node, strings.Join(said, "; "))
|
||||
o.Said = o.Summary + " — " + strings.Join(b.silent, "; ")
|
||||
return o
|
||||
}
|
||||
|
||||
// sayVaultSilent raises the condition for a send the vault did not answer whole, and clears it for one it did.
|
||||
func sayVaultSilent(ctx context.Context, node string, b coreBoxes) {
|
||||
if len(b.silent) == 0 && len(b.notPlaced) == 0 {
|
||||
settleVaultSilent(ctx, node)
|
||||
return
|
||||
}
|
||||
fmt.Printf("HELD: %s\n", vaultSilentObservation(node, b).Said)
|
||||
if conditionsFrom == nil {
|
||||
return
|
||||
}
|
||||
if _, err := conditionsFrom.Observe(ctx, vaultSilentObservation(node, b)); err != nil {
|
||||
fmt.Printf("the condition for the vault's silence on %s could not be raised: %v\n", node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// settleVaultSilent clears the condition once a send had every core credential of the machine from the vault.
|
||||
func settleVaultSilent(ctx context.Context, node string) {
|
||||
if conditionsFrom == nil {
|
||||
return
|
||||
}
|
||||
key := vaultSilentObservation(node, coreBoxes{}).Key()
|
||||
if _, err := conditionsFrom.Clear(ctx, key, "the vault answered for every core credential of "+node); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's
|
||||
// decision, issue 755 notes 4 and 5): to an engine that opens the vault's box, the send carries the box the vault
|
||||
// answered the controller and never the controller's own seal; when the vault does not answer, a kept box is carried
|
||||
// again and a module with none is not placed, with a condition naming the vault and the machine. An engine below the
|
||||
// contract composes exactly as before, and the vault is not asked.
|
||||
|
||||
// coreStandIn answers `get` with a box per name and generation, or its silence.
|
||||
type coreStandIn struct {
|
||||
asked []string
|
||||
keys []string
|
||||
generation int
|
||||
silent error
|
||||
}
|
||||
|
||||
func (v *coreStandIn) ask(name, key string) (coreAnswer, error) {
|
||||
v.asked, v.keys = append(v.asked, name), append(v.keys, key)
|
||||
if v.silent != nil {
|
||||
return coreAnswer{}, v.silent
|
||||
}
|
||||
box := `{"v":1,"eph":"ZQ==","iv":"aQ==","tag":"dA==","ct":"` + strings.Repeat("c", len(v.asked)) + `"}`
|
||||
return coreAnswer{Name: name, Generation: v.generation, Sealed: json.RawMessage(box)}, nil
|
||||
}
|
||||
|
||||
func withVault(t *testing.T, v *coreStandIn) {
|
||||
t.Helper()
|
||||
before := coreVault
|
||||
coreVault = func(context.Context) coreAsk { return v.ask }
|
||||
t.Cleanup(func() { coreVault = before })
|
||||
}
|
||||
|
||||
func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
client := aClientModule()
|
||||
register(t, open, client)
|
||||
if _, err := assign(ctx, open, "laptop", client.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"},
|
||||
"10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract,
|
||||
SaidAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &coreStandIn{generation: 1}
|
||||
withVault(t, v)
|
||||
return open, v
|
||||
}
|
||||
|
||||
func clientsBroker(resources []map[string]any) map[string]any {
|
||||
for _, r := range resources {
|
||||
if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
c, found, err := conditionsFrom.Get(t.Context(), vaultSilentObservation(node, coreBoxes{}).Key())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, found
|
||||
}
|
||||
|
||||
func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
key, err := open.inventory.SealingKeyOf(t.Context(), "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil {
|
||||
t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f)
|
||||
}
|
||||
if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) {
|
||||
t.Fatalf("the vault was asked %v", vault.asked)
|
||||
}
|
||||
if want, _ := sealingKeyPEM(key); vault.keys[0] != want {
|
||||
t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0])
|
||||
}
|
||||
first, _ := json.Marshal(f[catalogue.VaultSealedField])
|
||||
|
||||
// The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable.
|
||||
again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(again) != string(first) {
|
||||
t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again)
|
||||
}
|
||||
// A rotation in the vault (a newer generation) is carried.
|
||||
vault.generation = 2
|
||||
rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
if string(rotated) == string(first) {
|
||||
t.Fatal("a newer generation was not carried")
|
||||
}
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("a send the vault answered raised its silence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
|
||||
vault.silent = errors.New("no responders")
|
||||
held := clientsBroker(sentTo(t, open, "laptop"))
|
||||
got, _ := json.Marshal(held[catalogue.VaultSealedField])
|
||||
if held == nil || held["sealed"] != nil || string(got) != string(first) {
|
||||
t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") {
|
||||
t.Fatalf("no condition names the vault and the machine: %+v", c)
|
||||
}
|
||||
// The vault answers again: the condition clears.
|
||||
vault.silent = nil
|
||||
sentTo(t, open, "laptop")
|
||||
if _, found := vaultSilence(t, "laptop"); found {
|
||||
t.Fatal("the condition stayed after the vault answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsNotPlaced(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.VaultBoxContract)
|
||||
vault.silent = errors.New("no responders")
|
||||
plan, settings, err := planFor(t.Context(), open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range declared.Resources {
|
||||
if strings.HasPrefix(r["id"].(string), "mesh-provisioner-client.") {
|
||||
t.Fatalf("a core module was placed though the vault gave nothing for it: %v", r)
|
||||
}
|
||||
}
|
||||
if why := declared.leftOutWhy["mesh-provisioner-client"]; !strings.Contains(why, "never seals it itself") {
|
||||
t.Fatalf("left out %v", declared.leftOutWhy)
|
||||
}
|
||||
c, found := vaultSilence(t, "laptop")
|
||||
if !found || c.Severity != conditions.Urgent || !strings.Contains(c.Summary, "not placed") {
|
||||
t.Fatalf("the condition does not say why: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked.
|
||||
func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) {
|
||||
open, vault := aCoreMesh(t, link.HandedContract)
|
||||
f := clientsBroker(sentTo(t, open, "laptop"))
|
||||
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
|
||||
t.Fatalf("an older engine is sent %v", f)
|
||||
}
|
||||
if len(vault.asked) != 0 {
|
||||
t.Fatalf("the vault was asked %v for an older engine", vault.asked)
|
||||
}
|
||||
}
|
||||
|
||||
// Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for.
|
||||
func TestOnlyASendAsksTheVault(t *testing.T) {
|
||||
core := map[string][]string{"nats": {"broker"}}
|
||||
key := aPublicKey(t)
|
||||
kept := map[string]map[string]inventory.CoreBox{"nats": {"broker": {Module: "nats", Name: "broker",
|
||||
Generation: 1, SealedTo: key, Box: json.RawMessage(`{"v":1}`)}}}
|
||||
never := func(string, string) (coreAnswer, error) {
|
||||
t.Fatal("the vault was asked by a question that is not a send")
|
||||
return coreAnswer{}, nil
|
||||
}
|
||||
for _, choosing := range []Choosing{Reading, Foreseeing} {
|
||||
if got := coreBoxesOf("anchor", key, core, kept, choosing, never); string(got.sealed["nats"]["broker"]) != `{"v":1}` {
|
||||
t.Errorf("read %s", got.sealed["nats"]["broker"])
|
||||
}
|
||||
if got := coreBoxesOf("anchor", key, core, nil, choosing, never); string(got.sealed["nats"]["broker"]) !=
|
||||
string(catalogue.ForeseenVaultBox) {
|
||||
t.Errorf("with none kept, read %s", got.sealed["nats"]["broker"])
|
||||
}
|
||||
}
|
||||
// A box kept for another key of the machine is never carried.
|
||||
other := coreBoxesOf("anchor", aPublicKey(t), core, kept, Allocating,
|
||||
func(string, string) (coreAnswer, error) { return coreAnswer{}, errors.New("silent") })
|
||||
if _, left := other.notPlaced["nats"]; !left {
|
||||
t.Fatalf("a box sealed to the machine's old key was carried: %+v", other)
|
||||
}
|
||||
}
|
||||
|
||||
// The core's modules on a machine: on the list, not handed, with their own secrets but one issued outside.
|
||||
func TestTheCoresModulesAreTheListsWithTheirOwnSecrets(t *testing.T) {
|
||||
modules := []catalogue.Manifest{
|
||||
{Module: "nats", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/a"}, "given": {Path: "/b",
|
||||
IssuedBy: catalogue.IssuedOutside}}},
|
||||
{Module: "postgres", OwnSecrets: catalogue.OwnSecrets{"superuser": {Path: "/c"}}},
|
||||
{Module: "mesh-provisioner", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/d"}}},
|
||||
}
|
||||
got := coreModulesOf(modules, func(m catalogue.Manifest) bool { return m.Module == "mesh-provisioner" })
|
||||
if want := map[string][]string{"nats": {"broker"}}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
|
||||
// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
|
||||
// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
|
||||
// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
|
||||
// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
|
||||
// sent the control node the controller build before it. The older binary read the row it could not serve and the
|
||||
// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
|
||||
// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
|
||||
// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
|
||||
//
|
||||
// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
|
||||
// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
|
||||
// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
|
||||
// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
|
||||
// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
|
||||
// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
|
||||
// newer merge, or `rebuild mesh-controller`.
|
||||
//
|
||||
// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
|
||||
// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
|
||||
// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
|
||||
// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
|
||||
// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
|
||||
// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
|
||||
// schema already (issue 352).
|
||||
|
||||
// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
|
||||
const kindCoreDowngrade = "core-downgrade-kept"
|
||||
|
||||
// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
|
||||
const coreDowngradeSource = "sends"
|
||||
|
||||
// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
|
||||
// store and its seat's row, and the node-engine, which reads what the controller declares.
|
||||
var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
|
||||
|
||||
// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
|
||||
type askedBuild struct {
|
||||
asked time.Time
|
||||
failedGate bool
|
||||
}
|
||||
|
||||
// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
|
||||
// the request time is not known), and whether it failed its gate — and false when the records hold none. A
|
||||
// variable, so a test reads its own order.
|
||||
var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
|
||||
builds, err := inv.Builds(ctx, module, askedLookBack)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Worked() && sameCommit(b.Commit, commit) {
|
||||
failed, err := inv.GateFailed(ctx, b.ID)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
|
||||
}
|
||||
}
|
||||
return askedBuild{}, false, nil
|
||||
}
|
||||
|
||||
// askedLookBack is how many of a module's newest builds are read for when a build was asked.
|
||||
const askedLookBack = 50
|
||||
|
||||
// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
|
||||
type coreDowngrade struct {
|
||||
module, node, component string
|
||||
// sent is the build the machine was last sent, record the one the mesh holds.
|
||||
sent, record string
|
||||
sentAsked, recordAsked time.Time
|
||||
}
|
||||
|
||||
func (d coreDowngrade) String() string {
|
||||
return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
|
||||
"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
|
||||
"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
|
||||
d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
|
||||
d.module)
|
||||
}
|
||||
|
||||
// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
|
||||
// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
|
||||
// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
|
||||
// but for asked.
|
||||
func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
|
||||
asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
|
||||
var out []coreDowngrade
|
||||
for module, was := range sent {
|
||||
component := coreComponent(module)
|
||||
now := record[module]
|
||||
if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
|
||||
continue
|
||||
}
|
||||
if scope.puttingBack && scope.modules[module] {
|
||||
continue
|
||||
}
|
||||
from, known, err := asked(module, was)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
|
||||
}
|
||||
if !known || from.failedGate {
|
||||
continue
|
||||
}
|
||||
to, known, err := asked(module, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
|
||||
}
|
||||
if known && to.asked.Before(from.asked) {
|
||||
out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
|
||||
sentAsked: from.asked, recordAsked: to.asked})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func guarded(component string) bool {
|
||||
for _, c := range downgradeGuarded {
|
||||
if c == component {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
|
||||
// was last sent, module → commit, each said and raised as a condition.
|
||||
func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
|
||||
current map[string]inventory.CurrentBuild) (map[string]string, error) {
|
||||
record := map[string]string{}
|
||||
for m, c := range current {
|
||||
record[m] = c.Commit
|
||||
}
|
||||
found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
|
||||
return buildAsked(ctx, inv, module, commit)
|
||||
})
|
||||
if err != nil || len(found) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, d := range found {
|
||||
out[d.module] = d.sent
|
||||
sayCoreDowngrade(ctx, d)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
|
||||
func coreDowngradeObservation(component, node, summary string) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
|
||||
Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
|
||||
Summary: summary}
|
||||
}
|
||||
|
||||
func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
|
||||
fmt.Printf("KEPT: %s\n", d)
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil {
|
||||
return
|
||||
}
|
||||
if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
|
||||
fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
|
||||
// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
|
||||
// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
|
||||
func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil || carried == nil {
|
||||
return
|
||||
}
|
||||
var open []string
|
||||
for _, c := range downgradeGuarded {
|
||||
if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
|
||||
open = append(open, c)
|
||||
}
|
||||
}
|
||||
if len(open) == 0 {
|
||||
return
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
|
||||
return
|
||||
}
|
||||
for module, now := range current {
|
||||
component := coreComponent(module)
|
||||
if !guarded(component) {
|
||||
continue
|
||||
}
|
||||
if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
key := coreDowngradeObservation(component, node, "").Key()
|
||||
if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A dead letter from a restricted stream ages out with its stream (novox/hq ADR 0376 §3, §6).
|
||||
//
|
||||
// DEAD_LETTERS lets such a message go when its stream's maximum age runs out, so a consumer's `max-deliveries`
|
||||
// condition can clear with nobody having delivered it again or dropped it. That is said: the watchdogs remember,
|
||||
// from one look to the next, when each restricted dead letter ages out, count those gone once their time passed,
|
||||
// say each in the controller's log as it goes, and clear the condition with words saying how many aged out
|
||||
// undelivered and from which stream.
|
||||
|
||||
// agingTally is what the watchdogs remember of restricted dead letters between looks.
|
||||
type agingTally struct {
|
||||
// seen is each consumer's restricted dead letters at the last look, by id, with when each ages out.
|
||||
seen map[string]map[uint64]time.Time
|
||||
// agedOut is how many of each consumer's aged out undelivered while its condition was open.
|
||||
agedOut map[string]int
|
||||
}
|
||||
|
||||
// look compares this look's restricted dead letters with the last: newly is how many of each consumer's aged out
|
||||
// since, and cleared how many aged out in all for each consumer DEAD_LETTERS no longer holds anything of, when any
|
||||
// did. held is every consumer's count this look; aging its restricted dead letters.
|
||||
func (t *agingTally) look(now time.Time, held map[string]int, aging map[string][]link.Aging) (newly, cleared map[string]int) {
|
||||
if t.seen == nil {
|
||||
t.seen, t.agedOut = map[string]map[uint64]time.Time{}, map[string]int{}
|
||||
}
|
||||
newly, cleared = map[string]int{}, map[string]int{}
|
||||
for key, before := range t.seen {
|
||||
still := map[uint64]bool{}
|
||||
for _, a := range aging[key] {
|
||||
still[a.ID] = true
|
||||
}
|
||||
for id, agesOut := range before {
|
||||
if !still[id] && !agesOut.After(now) {
|
||||
newly[key]++
|
||||
}
|
||||
}
|
||||
if newly[key] == 0 {
|
||||
delete(newly, key)
|
||||
}
|
||||
t.agedOut[key] += newly[key]
|
||||
}
|
||||
for key, n := range t.agedOut {
|
||||
if held[key] > 0 {
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
cleared[key] = n
|
||||
}
|
||||
delete(t.agedOut, key)
|
||||
}
|
||||
t.seen = map[string]map[uint64]time.Time{}
|
||||
for key, letters := range aging {
|
||||
t.seen[key] = map[uint64]time.Time{}
|
||||
for _, a := range letters {
|
||||
t.seen[key][a.ID] = a.AgesOut
|
||||
}
|
||||
}
|
||||
return newly, cleared
|
||||
}
|
||||
|
||||
// agedOutResolved is the line a consumer's condition clears with when messages it gave up on aged out undelivered.
|
||||
func agedOutResolved(key string, n int) string {
|
||||
stream, _, _ := strings.Cut(key, ".")
|
||||
messages := "a message"
|
||||
if n > 1 {
|
||||
messages = fmt.Sprintf("%d messages", n)
|
||||
}
|
||||
return fmt.Sprintf("Resolved: %s of the stream %s aged out undelivered", messages, stream)
|
||||
}
|
||||
|
||||
// sayAgedOut says, in the log, what aged out since the last look, and clears the condition of every consumer whose
|
||||
// dead letters are all gone with the words saying how many aged out. Before the rows are reconciled, so the
|
||||
// condition is cleared with these words and not its own.
|
||||
func (w *watchdogs) sayAgedOut(ctx context.Context, f *signalFacts) []string {
|
||||
if f.advisoriesErr != nil || f.deadLettersAging == nil {
|
||||
return nil
|
||||
}
|
||||
newly, cleared := w.aging.look(f.now, f.deadLetters, f.deadLettersAging)
|
||||
var problems []string
|
||||
for _, key := range countedKeys(newly) {
|
||||
stream, consumer, _ := strings.Cut(key, ".")
|
||||
fmt.Printf("%d message(s) %s gave up on aged out of %s undelivered: %s keeps them only for its maximum age "+
|
||||
"(novox/hq ADR 0376)\n", newly[key], link.ConsumerInWords(stream, consumer), "DEAD_LETTERS", stream)
|
||||
}
|
||||
for _, key := range countedKeys(cleared) {
|
||||
stream, consumer, _ := strings.Cut(key, ".")
|
||||
fmt.Printf("%s no longer has a message kept: %d aged out of %s undelivered, so its condition clears "+
|
||||
"(novox/hq ADR 0376)\n", link.ConsumerInWords(stream, consumer), cleared[key], stream)
|
||||
if _, err := w.keeper.ClearSaying(ctx, conditions.Key(conditions.ScopeBus, key, link.AdvisoryMaxDeliveries),
|
||||
fmt.Sprintf("its %d dead letter(s) of %s aged out undelivered", cleared[key], stream),
|
||||
agedOutResolved(key, cleared[key])); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// countedKeys is a count's keys in order.
|
||||
func countedKeys(m map[string]int) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A dead letter that ages out clears its condition and says so** (novox/hq ADR 0376 §6), on a fake clock: while
|
||||
// DEAD_LETTERS holds a restricted stream's messages for a consumer its max-deliveries condition is open; once they
|
||||
// have aged out it clears, with words saying how many aged out undelivered and from which stream. One dropped before
|
||||
// its age is not counted as aged out, and a consumer of EVENTS clears as before.
|
||||
func TestADeadLetterThatAgesOutClearsItsConditionAndSaysSo(t *testing.T) {
|
||||
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
||||
Now: func() time.Time { return now }})
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
|
||||
look := func(at time.Time, held map[string]int, aging map[string][]link.Aging) {
|
||||
f := calm(at)
|
||||
f.deadLetters, f.deadLettersAging = held, aging
|
||||
w.see(t.Context(), f)
|
||||
}
|
||||
mail := "MAIL.ace_mailer"
|
||||
look(now, map[string]int{mail: 3, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: {
|
||||
{ID: 1, AgesOut: now.Add(time.Hour)}, {ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
if open, _ := k.Open(t.Context()); len(open) != 2 {
|
||||
t.Fatalf("open: %+v", open)
|
||||
}
|
||||
// One dropped by a person before its age: not aged out.
|
||||
look(now.Add(30*time.Minute), map[string]int{mail: 2, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: {
|
||||
{ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
look(now.Add(150*time.Minute), map[string]int{mail: 1}, map[string][]link.Aging{mail: {{ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
|
||||
open, _ := k.Open(t.Context())
|
||||
if len(open) != 1 || open[0].Key != conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries) {
|
||||
t.Fatalf("after one aged out: %+v", open)
|
||||
}
|
||||
look(now.Add(4*time.Hour), map[string]int{}, map[string][]link.Aging{})
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("after all aged out: %+v", open)
|
||||
}
|
||||
k.Close(context.Background()) // everything said is said once it closes
|
||||
var mailCleared, sonarrCleared string
|
||||
for _, e := range told.Said() {
|
||||
if e.Change != conditions.ChangeCleared {
|
||||
continue
|
||||
}
|
||||
switch e.Condition.Key {
|
||||
case conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries):
|
||||
mailCleared = e.Condition.Resolved
|
||||
case conditions.Key(conditions.ScopeBus, "EVENTS.media_sonarr", link.AdvisoryMaxDeliveries):
|
||||
sonarrCleared = e.Condition.Resolved
|
||||
}
|
||||
}
|
||||
if mailCleared != "Resolved: 2 messages of the stream MAIL aged out undelivered" {
|
||||
t.Fatalf("MAIL's condition cleared with %q", mailCleared)
|
||||
}
|
||||
if sonarrCleared != "Resolved: the messages it gave up on were delivered again or dropped" {
|
||||
t.Fatalf("an EVENTS consumer's condition cleared with %q", sonarrCleared)
|
||||
}
|
||||
if why, ok := conditions.Plain(mailCleared); !ok {
|
||||
t.Fatalf("not plain: %s", why)
|
||||
}
|
||||
}
|
||||
+118
-37
@@ -10,6 +10,7 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -240,7 +241,7 @@ func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why
|
||||
}
|
||||
return p, said, inv.SavePlan(ctx, &p)
|
||||
}
|
||||
target, err := batchToWalkAgain(ctx, inv, again, now)
|
||||
target, folded, foldMoves, err := batchToWalkAgain(ctx, inv, p.ID, again, now)
|
||||
if err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
|
||||
}
|
||||
@@ -261,8 +262,10 @@ func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why
|
||||
if len(said.Kept) > 0 {
|
||||
p.Note += "; " + keptWords(said.Kept)
|
||||
}
|
||||
// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7).
|
||||
if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&target, &p}, moves); err != nil {
|
||||
// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7) — the
|
||||
// batches folded into it too.
|
||||
if err := inv.SavePlansMoving(ctx, append([]*inventory.Plan{&target, &p}, folded...),
|
||||
append(moves, foldMoves...)); err != nil {
|
||||
return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
|
||||
}
|
||||
// What the batch holds and names now: written again by the cutter's next look should this fail.
|
||||
@@ -317,36 +320,42 @@ func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Pla
|
||||
return nil
|
||||
}
|
||||
|
||||
// batchToWalkAgain is the batch a stopped walk's merges join: the open batch of their kind, or a new one, not yet
|
||||
// written. The kind is read from what the merges moved.
|
||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, again []inventory.BatchedMerge,
|
||||
now time.Time) (inventory.Plan, error) {
|
||||
// batchToWalkAgain is the batch a stopped walk's merges join (novox/hq ADR 0299): a batch of their kind not yet cut
|
||||
// that they are related to (ADR 0304 decision 1, ADR 0306 decision 3), or a new one, not yet written. Related to
|
||||
// two or more, the newer are folded into the oldest: the plans and merge moves that fold them, to be written in
|
||||
// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds.
|
||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge,
|
||||
now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) {
|
||||
own := false
|
||||
for _, n := range movesOf(again) {
|
||||
if _, on := onTheControllersPath[n]; on {
|
||||
own = true
|
||||
}
|
||||
}
|
||||
batches, err := inv.Batches(ctx)
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
}
|
||||
for _, b := range batches {
|
||||
if b.OwnPath() == own {
|
||||
if b.Delivery == nil {
|
||||
b.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
if b.Delivery.Batch == nil {
|
||||
b.Delivery.Batch = &inventory.PlanBatch{Own: own}
|
||||
}
|
||||
return b, nil
|
||||
joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
}
|
||||
if len(joins) > 0 {
|
||||
into := joins[0]
|
||||
if len(joins) == 1 {
|
||||
return into, nil, nil, nil
|
||||
}
|
||||
plans, moves, err := foldInto(ctx, inv, &into, joins[1:])
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
}
|
||||
return into, plans[1:], moves, nil
|
||||
}
|
||||
first := again[0]
|
||||
return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
||||
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
||||
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil
|
||||
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil, nil, nil
|
||||
}
|
||||
|
||||
// leaveOut splits a walk's merges by the merges named to leave out, each `<repository>@<commit>` (a commit's
|
||||
@@ -571,6 +580,9 @@ type orderMember struct {
|
||||
CloneURL string `json:"clone_url,omitempty"`
|
||||
// After are the repositories its pull request says it goes after (`after: <repository>`).
|
||||
After []string `json:"after,omitempty"`
|
||||
// BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit
|
||||
// a verdict no build judges names when the head is checked again through mesh-delivery.
|
||||
BaseSHA string `json:"base_sha,omitempty"`
|
||||
}
|
||||
|
||||
// pull is the member as the forge announced it.
|
||||
@@ -580,7 +592,7 @@ func (m orderMember) pull() link.PullUpdated {
|
||||
if base == "" {
|
||||
base = "main"
|
||||
}
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head,
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA,
|
||||
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
||||
}
|
||||
@@ -940,7 +952,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
// own path for a pull request, run because the delivery's owner asked.
|
||||
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open: open, oneShot: true}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"rechecked": members[0].ID})
|
||||
@@ -958,7 +970,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
@@ -973,7 +985,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
var p inventory.Plan
|
||||
var said stopAnswer
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without),
|
||||
time.Now().UTC())
|
||||
return err
|
||||
@@ -1030,16 +1042,18 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||
}
|
||||
|
||||
// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
|
||||
// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
||||
// appliedFrom answers a machine's own report of the send a walk made to it from the controller's `apply` durations
|
||||
// (novox/hq issue 582); a lookup that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
||||
func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
|
||||
return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
|
||||
return func(node string, send inventory.SentDeclaration, walkSent time.Time) (inventory.AppliedReport, bool) {
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, node, send, walkSent)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
|
||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err)
|
||||
return inventory.AppliedReport{}, false
|
||||
}
|
||||
return r, ok
|
||||
// And, for each module the send handed to the provisioner, its client's report on that build (novox/hq ADR
|
||||
// 0362): applied only once both said it, at the later; failed if the client failed.
|
||||
return inventory.ProvisionedSend(node, send, r, ok, provLookup(ctx))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1176,7 +1190,7 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
|
||||
}
|
||||
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths})
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base})
|
||||
}
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
@@ -1215,15 +1229,19 @@ func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
||||
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
||||
// comparison. Records arriving out of order are told apart by the plan's revision.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, p)
|
||||
//
|
||||
// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and,
|
||||
// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what
|
||||
// the walk recorded.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, facts, p)
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
// Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are
|
||||
// read by whoever asks for the walk (`delivery walks`).
|
||||
p.Phases = p.WalkPhases(time.Now(), nil)
|
||||
body, err := json.Marshal(p)
|
||||
body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(ctx, facts, p)})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -1238,7 +1256,7 @@ func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
|
||||
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
|
||||
// still runs; what it did is found by comparison.
|
||||
func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error {
|
||||
if checkEvents != nil {
|
||||
return f()
|
||||
}
|
||||
@@ -1249,9 +1267,10 @@ func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
return err
|
||||
}
|
||||
bus := link.OverNATS{Conn: conn, JS: js}
|
||||
was := inventory.PlanSaved
|
||||
checkEvents = bus
|
||||
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { publishPlanMoved(ctx, bus, facts, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, was }()
|
||||
ran = true
|
||||
return f()
|
||||
})
|
||||
@@ -1261,3 +1280,65 @@ func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sayingWhatAVerbKept is PlanSaved for a process that is not the serving controller: each walk the verb keeps is
|
||||
// said as `plan-moved` before the save returns, with the facts of the inventory that kept it, on the bus the verb
|
||||
// reaches. It reaches the bus on the first save and never for a verb that keeps no walk. When the bus cannot be
|
||||
// reached, a save that started or ended a walk says so on the verb's error output: its mark is kept and no later
|
||||
// save says it, so the router holds what the walk touched until its bound, or not at all for a start. done lets
|
||||
// the bus go and takes the hook back.
|
||||
func sayingWhatAVerbKept(ctx context.Context) (said func(*inventory.Inventory, inventory.Plan), done func()) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
reached bool
|
||||
bus link.Bus
|
||||
closing func()
|
||||
failed error
|
||||
)
|
||||
said = func(inv *inventory.Inventory, p inventory.Plan) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !reached {
|
||||
reached = true
|
||||
if handActConn != nil {
|
||||
bus, failed = overNATS(handActConn)
|
||||
} else if js, err := aBus(); err != nil {
|
||||
failed = err
|
||||
} else if bus, failed = overNATS(js.Conn()); failed == nil {
|
||||
closing = js.Close
|
||||
} else {
|
||||
js.Close()
|
||||
}
|
||||
}
|
||||
if bus == nil {
|
||||
switch p.WalkSaid {
|
||||
case inventory.WalkStarted:
|
||||
fmt.Fprintf(os.Stderr, "%s: its start is kept and could NOT be said as plan-moved (%v): the router "+
|
||||
"does not hold what it touches\n", p.ID, failed)
|
||||
case inventory.WalkEnded:
|
||||
fmt.Fprintf(os.Stderr, "%s: its end is kept and could NOT be said as plan-moved (%v): the router holds "+
|
||||
"what it touched until its bound\n", p.ID, failed)
|
||||
}
|
||||
return
|
||||
}
|
||||
publishPlanMoved(ctx, bus, inv, p)
|
||||
}
|
||||
done = func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
inventory.PlanSaved = nil
|
||||
if closing != nil {
|
||||
closing()
|
||||
}
|
||||
}
|
||||
return said, done
|
||||
}
|
||||
|
||||
// overNATS is the bus a connection carries, with its JetStream.
|
||||
func overNATS(conn *nats.Conn) (link.Bus, error) {
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.OverNATS{Conn: conn, JS: js}, nil
|
||||
}
|
||||
|
||||
@@ -47,10 +47,14 @@ type stalledLine struct {
|
||||
// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
|
||||
// says none, and Waiting is said as it reads.
|
||||
Checks []waitingCheck `json:"checks,omitempty"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
// Held is the state of the delivery of a line of the state `unanswered`, ready or rejected, when one is held
|
||||
// (novox/hq issues 441, 451): its check answered once, so a recheck is the step. Empty for a head with no
|
||||
// delivery, and from a mesh-delivery that does not say it yet; the words are never read for it.
|
||||
Held string `json:"held,omitempty"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
||||
|
||||
@@ -119,6 +119,13 @@ func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
// A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353).
|
||||
if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) {
|
||||
t.Fatal("a list nobody could compose reads as current")
|
||||
}
|
||||
if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") {
|
||||
t.Fatal("listBehind does not read the digest as userListBehind does")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -54,6 +55,10 @@ type deskGive struct {
|
||||
trusted func(module string) (bool, error)
|
||||
// ask asks one machine's node-launcher.secret and answers its result, or the holder's refusal.
|
||||
ask func(machine string, args map[string]any) (json.RawMessage, error)
|
||||
// promptFor is the one line the secret's declaration asks for it as (novox/hq issue 789): read from the
|
||||
// controller's own catalogue, never from the call or the module's runtime. Nil, or an empty answer, asks by
|
||||
// name alone, as the desk does today.
|
||||
promptFor func(module, name string) (string, error)
|
||||
// accept seals the value as `secret accept` does, and says whether it lives until the module's start.
|
||||
accept func(value string) (untilStart bool, err error)
|
||||
// record writes the act in the hand-act log.
|
||||
@@ -67,6 +72,9 @@ type deskGive struct {
|
||||
// nil keeps no record (a test that does not look). end closes it with how it ended.
|
||||
open func(node, module, name, desk string) (int64, error)
|
||||
end func(id int64, outcome string) error
|
||||
// vault, when set, is the path of a module whose secrets are filled the new way (vault_secrets.go): the vault
|
||||
// records the ask, holds its bounds and opens what is typed; open, end and accept are not used.
|
||||
vault vaultAsker
|
||||
}
|
||||
|
||||
// askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters
|
||||
@@ -133,6 +141,9 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
"bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name)
|
||||
}
|
||||
}
|
||||
if d.vault != nil {
|
||||
return d.giveToVault(node, module, name, desk)
|
||||
}
|
||||
// The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret,
|
||||
// few an hour. How the ask ends is recorded whatever happens below.
|
||||
outcome := "failed"
|
||||
@@ -153,15 +164,28 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
|
||||
}
|
||||
// By name, never by words: the holder writes the prompt from these, and says the controller asks, which
|
||||
// the bus alone makes true (broker.ControllerOnly). Who asked is the bus's word on the caller, cut to a
|
||||
// name's characters — never an argument of the call.
|
||||
raw, err := d.ask(desk, map[string]any{
|
||||
// name's characters — never an argument of the call. `what` is the catalogue's own words for the secret
|
||||
// (novox/hq issue 789), never a word the caller chose; omitted when the module declares none, so an old
|
||||
// desk and an old module behave as today.
|
||||
args := map[string]any{
|
||||
"module": module,
|
||||
"secret": name,
|
||||
"node": node,
|
||||
"asked_by": askedByName(d.askedBy),
|
||||
"seal_to": public,
|
||||
"timeout_seconds": deskPromptWithin,
|
||||
})
|
||||
}
|
||||
if d.promptFor != nil {
|
||||
what, err := d.promptFor(module, name)
|
||||
if err != nil {
|
||||
outcome = "refused"
|
||||
return "", fmt.Errorf("nobody was asked to type anything: %w", err)
|
||||
}
|
||||
if what != "" {
|
||||
args["what"] = what
|
||||
}
|
||||
}
|
||||
raw, err := d.ask(desk, args)
|
||||
if err != nil {
|
||||
outcome = "refused"
|
||||
return "", fmt.Errorf("the desk on %s could not be asked: %w", desk, err)
|
||||
@@ -236,8 +260,9 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
askedBy := link.Caller()
|
||||
d := deskGive{
|
||||
askedBy: link.Caller(),
|
||||
askedBy: askedBy,
|
||||
open: func(node, module, name, desk string) (int64, error) {
|
||||
return open.inventory.OpenSecretAsk(ctx, node, module, name, askedByName(link.Caller()), desk)
|
||||
},
|
||||
@@ -247,7 +272,8 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, machine)
|
||||
return err
|
||||
},
|
||||
trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) },
|
||||
trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) },
|
||||
promptFor: func(module, name string) (string, error) { return open.inventory.OwnSecretPrompt(ctx, module, name) },
|
||||
ask: func(machine string, args map[string]any) (json.RawMessage, error) {
|
||||
var result json.RawMessage
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
@@ -273,7 +299,18 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error {
|
||||
return err
|
||||
})
|
||||
},
|
||||
announce: func(node, module, name, how string) error { return announceSecretGiven(ctx, node, module, name, how) },
|
||||
announce: func(node, module, name, how string) error {
|
||||
return announceSecretGiven(ctx, node, module, name, how, askedBy)
|
||||
},
|
||||
}
|
||||
// A module whose secrets are filled the new way is asked for by the vault (ADR 0349 clause 9); every other
|
||||
// keeps today's path (clause 17).
|
||||
ours, err := filledTheNewWay(module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
if ours {
|
||||
d.vault = theVault(ctx)
|
||||
}
|
||||
words, err := d.give(node, module, name, desk)
|
||||
if err != nil {
|
||||
@@ -324,10 +361,85 @@ func secretNameWords(name string) string {
|
||||
return strings.Join(strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' || r == '.' }), " ")
|
||||
}
|
||||
|
||||
// announceSecretGiven raises it on this controller's keeper.
|
||||
func announceSecretGiven(ctx context.Context, node, module, name, how string) error {
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
_, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, time.Now()))
|
||||
return err
|
||||
// tellWithin is how long the operator channel's `tell` is given to answer: well under a human's patience,
|
||||
// and short because nothing in the call waits on the operator — it asks nothing (novox/hq ADR 0380).
|
||||
const tellWithin = 10 * time.Second
|
||||
|
||||
// askOperatorChannel asks the operator channel's seat (the holder issue 797 gives it) its `tell` verb,
|
||||
// through the controller's grant (novox/hq ADR 0380 §8), over this controller's own bus connection; a seam
|
||||
// a test replaces whole. The holder's own refusal is an error naming it; an absent holder or a refused
|
||||
// grant answer as link.ErrNothingServes or link.ErrGrantRefused (checked with errors.Is), so the caller can
|
||||
// tell the two apart from any other failure.
|
||||
var askOperatorChannel = func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
var result json.RawMessage
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell", args, tellWithin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("operator-channel.tell refused: %s", answer.Error)
|
||||
}
|
||||
result = answer.Result
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
// announceSecretGiven tells the operator, once, that a module's own secret was given (novox/hq ADR 0380):
|
||||
// one plain message through the operator channel's `tell`, naming the secret, the module, the machine,
|
||||
// where and when it was given and who asked — asking nothing, and opening no condition. **The controller's
|
||||
// change waits for that verb at run time, not at merge** (clause 9): while nothing on the bus yet serves it,
|
||||
// or the controller's own grant is refused, the secret-given condition is raised exactly as before ADR 0380
|
||||
// — any other failure (the holder itself refusing, or answering something unreadable) is the announcement's
|
||||
// own failure, said to the caller as it always was.
|
||||
func announceSecretGiven(ctx context.Context, node, module, name, how, askedBy string) error {
|
||||
at := time.Now()
|
||||
body := map[string]any{
|
||||
"headline": fmt.Sprintf("A password was entered for %s's %s on %s", module, secretNameWords(name), node),
|
||||
"subject": node,
|
||||
"lines": []string{
|
||||
fmt.Sprintf("%s at %s, asked by %s.", how, at.Local().Format("15:04"), askedByName(askedBy)),
|
||||
"If that was not you, give it again yourself at the controller's terminal.",
|
||||
},
|
||||
}
|
||||
_, told := askOperatorChannel(ctx, body)
|
||||
switch {
|
||||
case told == nil:
|
||||
clearSecretGivenConditions(ctx)
|
||||
return nil
|
||||
case errors.Is(told, link.ErrNothingServes), errors.Is(told, link.ErrGrantRefused):
|
||||
log.Printf("secret-given: %s's %s on %s could not be told through the operator channel (%v); the "+
|
||||
"secret-given condition is raised as it was before novox/hq ADR 0380 (its clause 9)", module, name,
|
||||
node, told)
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
_, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, at))
|
||||
return err
|
||||
})
|
||||
default:
|
||||
return told
|
||||
}
|
||||
}
|
||||
|
||||
// clearSecretGivenConditions retires every open secret-given condition once a plain message has replaced
|
||||
// it (novox/hq ADR 0380 §10): called after the first time a secret-given message is told successfully. No
|
||||
// resolved line is sent — the condition is simply gone — and why is kept in the keeper's own history.
|
||||
func clearSecretGivenConditions(ctx context.Context) {
|
||||
if err := withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Kind != kindSecretGiven {
|
||||
continue
|
||||
}
|
||||
if _, err := k.ClearSaying(ctx, c.Key, "retired by novox/hq ADR 0380", ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
log.Printf("secret-given: the open secret-given conditions, retired by ADR 0380, could not be cleared: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -456,6 +456,52 @@ func TestThePromptNamesWhoAskedFromTheBussWordAlone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The desk's prompt names which account of a family is asked for, from the catalogue's own words (novox/hq
|
||||
// issue 789): a family declaring `prompt` with `{member}` sends `what` with the member's suffix filled in.
|
||||
func TestThePromptNamesWhichAccountOfAFamilyIsAsked(t *testing.T) {
|
||||
d, _, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.declares = func(module, name string) error {
|
||||
if module != "mesh-mail" || name != "mail-credential-gmail" {
|
||||
return errors.New(module + " does not declare " + name + " as an own secret")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
d.promptFor = func(module, name string) (string, error) {
|
||||
if module == "mesh-mail" && name == "mail-credential-gmail" {
|
||||
return "the password of the mail account gmail", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if _, err := d.give("anchor", "mesh-mail", "mail-credential-gmail", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := (*asked)[0]["what"]; got != "the password of the mail account gmail" {
|
||||
t.Errorf(`"what" = %v`, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Without a declared prompt, no `what` is sent: an old module, or one that declares none, is asked by name alone,
|
||||
// as the desk does today (novox/hq issue 789).
|
||||
func TestWithNoDeclaredPromptNoWhatIsSent(t *testing.T) {
|
||||
d, _, _, asked := aDesk(t, sealedTo(t, typed))
|
||||
d.promptFor = func(module, name string) (string, error) { return "", nil }
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, there := (*asked)[0]["what"]; there {
|
||||
t.Errorf(`"what" was sent with no declared prompt: %v`, (*asked)[0])
|
||||
}
|
||||
// And with no promptFor wired at all (an older caller of deskGive), same thing: `give` still works, asking
|
||||
// nothing more than it does today.
|
||||
d2, _, _, asked2 := aDesk(t, sealedTo(t, typed))
|
||||
if _, err := d2.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, there := (*asked2)[0]["what"]; there {
|
||||
t.Errorf(`"what" was sent with promptFor nil: %v`, (*asked2)[0])
|
||||
}
|
||||
}
|
||||
|
||||
// An ask for a secret is bounded before anybody is asked to type (ADR 0277): the record refuses it, nothing is
|
||||
// asked; and how every ask ends is recorded.
|
||||
func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) {
|
||||
@@ -510,3 +556,172 @@ func TestASecretAskIsNeverASecretRead(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// askOperatorChannelFake replaces askOperatorChannel for the test and records every call it was given.
|
||||
func askOperatorChannelFake(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) *[]map[string]any {
|
||||
t.Helper()
|
||||
var calls []map[string]any
|
||||
was := askOperatorChannel
|
||||
askOperatorChannel = func(_ context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
calls = append(calls, args)
|
||||
return answer(args)
|
||||
}
|
||||
t.Cleanup(func() { askOperatorChannel = was })
|
||||
return &calls
|
||||
}
|
||||
|
||||
// announceWithRealTell wires a deskGive's announce to the real announceSecretGiven, as askAtDesk does,
|
||||
// rather than the fakes the tests above use to look at what `give` does around it.
|
||||
func announceWithRealTell(ctx context.Context, askedBy string) func(node, module, name, how string) error {
|
||||
return func(node, module, name, how string) error {
|
||||
return announceSecretGiven(ctx, node, module, name, how, askedBy)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0380 clauses 1-2: a secret given at the desk is told to the operator once, in a plain
|
||||
// message through the operator channel's `tell`, naming the secret, the module, where and when and who
|
||||
// asked, and opens no condition — the kind the review of 2026-10-09 (M4) raised is retired.
|
||||
func TestASecretGivenAtTheDeskIsToldOnceAndOpensNoCondition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
calls := askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, nil })
|
||||
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||
d.announce = announceWithRealTell(ctx, "mail-seats-835@novox")
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*calls) != 1 {
|
||||
t.Fatalf("told the operator %d times, want exactly 1: %v", len(*calls), *calls)
|
||||
}
|
||||
call := (*calls)[0]
|
||||
if call["headline"] != "A password was entered for telegram's telegram token on anchor" {
|
||||
t.Errorf("headline %v", call["headline"])
|
||||
}
|
||||
if call["subject"] != "anchor" {
|
||||
t.Errorf("subject %v", call["subject"])
|
||||
}
|
||||
lines, _ := call["lines"].([]string)
|
||||
if len(lines) != 2 || !strings.Contains(lines[0], "at the desk on laptop at") ||
|
||||
!strings.Contains(lines[0], "asked by mail-seats-835@novox") ||
|
||||
lines[1] != "If that was not you, give it again yourself at the controller's terminal." {
|
||||
t.Errorf("lines %v", lines)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 0 {
|
||||
t.Errorf("a condition was raised though the operator was told: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0380 §9: until the operator channel's `tell` is delivered — nothing yet serves it, or the
|
||||
// controller's own grant is refused — the secret-given condition is raised exactly as before.
|
||||
func TestASecretGivenAtTheDeskRaisesTheConditionWhileNothingServesTell(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, failure := range []error{link.ErrNothingServes, link.ErrGrantRefused} {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, failure })
|
||||
d, _, _, _ := aDesk(t, sealedTo(t, typed))
|
||||
d.announce = announceWithRealTell(ctx, "mesh-mail-835, novox")
|
||||
if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Kind != kindSecretGiven {
|
||||
t.Fatalf("%v: the condition was not raised as before: %+v", failure, open)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0380 §10: once the operator channel is told successfully, every secret-given condition
|
||||
// still open from before this change is retired, without a resolved line of its own.
|
||||
func TestASuccessfulTellRetiresEveryOpenSecretGivenCondition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
for _, module := range []string{"telegram", "mesh-mail"} {
|
||||
if _, err := k.Observe(ctx, secretGivenObservation("anchor", module, "a-secret", "at the desk on laptop",
|
||||
time.Now())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, nil })
|
||||
if err := announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the desk on laptop", "me"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 0 {
|
||||
t.Fatalf("the open secret-given conditions were not retired: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0380 clause 7: a trusted party's secret given at the controller's terminal is announced
|
||||
// before it is kept, and not kept when the announcement fails. It still fails when telling the operator
|
||||
// AND raising the fallback condition both fail — nothing silently keeps a trusted party's value unheard.
|
||||
func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes })
|
||||
// conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be
|
||||
// raised either: both of ADR 0380's ways to tell the operator fail.
|
||||
kept := false
|
||||
if _, _, err := keepGiven(true,
|
||||
func() error {
|
||||
return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me")
|
||||
},
|
||||
func() (bool, error) { kept = true; return false, nil }); err == nil {
|
||||
t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed")
|
||||
}
|
||||
if kept {
|
||||
t.Error("the value was kept despite the failure")
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2 of mesh-controller (feat/mail-seats, head 5f24a47d): ADR 0380 §9's fallback — raising the
|
||||
// secret-given condition — answers only link.ErrNothingServes and link.ErrGrantRefused. Any other failure of
|
||||
// the operator channel's `tell` (the holder itself refusing, or answering something unreadable) is the
|
||||
// announcement's own failure, surfaced to the caller, and opens no condition.
|
||||
func TestAnOperatorChannelFailureOtherThanNothingServesOrGrantRefusedSurfacesAndRaisesNoCondition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
refused := errors.New("operator-channel.tell refused: too many lines")
|
||||
askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, refused })
|
||||
if err := announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the desk on laptop", "me"); err != refused {
|
||||
t.Fatalf("announceSecretGiven = %v, want the refusal itself surfaced", err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 0 {
|
||||
t.Errorf("a condition was raised though the failure was neither ErrNothingServes nor ErrGrantRefused: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2 of mesh-controller: a secret accepted at the controller's terminal writes a hand act naming
|
||||
// who asked, where and when it was given — and never the value (novox/hq ADR 0380 §6).
|
||||
func TestASecretAcceptedAtTheTerminalWritesAHandActNamingWhoWhereAndNeverTheValue(t *testing.T) {
|
||||
act := terminalGiveHandAct("anchor", "telegram", "telegram-token", "mail-seats-835@novox")
|
||||
if act.Verb != "secret accept" || act.Cause != "given-at-the-terminal" {
|
||||
t.Fatalf("act %+v", act)
|
||||
}
|
||||
if !strings.Contains(act.Why, "at the controller's terminal") {
|
||||
t.Errorf("the act does not name where: %q", act.Why)
|
||||
}
|
||||
if !strings.Contains(act.Why, "asked by mail-seats-835@novox") {
|
||||
t.Errorf("the act does not name who: %q", act.Why)
|
||||
}
|
||||
if strings.Contains(act.Why, typed) {
|
||||
t.Error("the act carries the value")
|
||||
}
|
||||
// When is RecordHandAct's own job (it stamps At with time.Now when the act leaves none, and By with the
|
||||
// caller when the act leaves that empty too): this act leaves both for it to fill.
|
||||
if act.By != "" || !act.At.IsZero() {
|
||||
t.Errorf("the act sets its own who/when instead of leaving them to RecordHandAct: %+v", act)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,6 +146,11 @@ var probeRegistry = []probe{
|
||||
{ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " +
|
||||
"said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
||||
Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill},
|
||||
// A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2): the module is carried by the
|
||||
// machine's tool runner, which never hears a restricted stream, so it hears nothing it was granted.
|
||||
{ID: probeWithheldID, Asserts: "no grant of a restricted stream names a module that composes nothing for it: every " +
|
||||
"module granted one runs as an account of its own", From: "ADR 0376 §2", Kind: kindGrantComposesNothing,
|
||||
Phase: 1, run: probeWithheldHearing},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
|
||||
@@ -348,11 +348,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
if layer.From == catalogue.MeshWideLayer {
|
||||
if !meshWide[module] {
|
||||
meshWide[module] = true
|
||||
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
|
||||
}
|
||||
continue
|
||||
}
|
||||
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
|
||||
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
|
||||
}
|
||||
}
|
||||
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A change plan says its class** (novox/hq ADR 0313 §2): the issue tracker holds a fix to the budget of the
|
||||
// slowest-budgeted module the fix moves, and a fix that moves no module has no fix budget. The plan says core,
|
||||
// leaf or none, so mesh-delivery can pass it on with the fix's delivery.
|
||||
func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name, path string) inventory.Entry {
|
||||
e := fromRepo(name, repo, path)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = []string{"anchor"}
|
||||
return e
|
||||
}
|
||||
gitea := entry("gitea", "modules/gitea")
|
||||
resolver := entry("dnsmasq", "modules/dnsmasq")
|
||||
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
|
||||
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
|
||||
webApp := entry("photos", "modules/photos")
|
||||
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
|
||||
// photos stands on gitea: a change to gitea moves photos as its dependent.
|
||||
edges := []inventory.Edge{{From: "photos", To: "gitea", Kind: inventory.EdgeStandsOn}}
|
||||
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
paths []string
|
||||
want string
|
||||
}{
|
||||
{"a leaf module and its dependent", []string{"modules/gitea/index.ts"}, inventory.ClassLeaf},
|
||||
{"the delivery's owner, on the controller's own path", []string{"modules/mesh-delivery/cmd/x.go"}, inventory.ClassCore},
|
||||
{"the mesh's resolver", []string{"modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"nothing any module's build reads", []string{"README.md"}, "none"},
|
||||
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
|
||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-delivery/module.json"}, inventory.ClassCore},
|
||||
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
} {
|
||||
if got := plan(c.paths...).Class; got != c.want {
|
||||
t.Errorf("%s: class %q, want %q", c.why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
||||
}
|
||||
|
||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
@@ -48,6 +49,7 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
||||
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
@@ -81,9 +83,13 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
@@ -112,6 +118,7 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
|
||||
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
|
||||
+250
-39
@@ -28,9 +28,12 @@ import (
|
||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
|
||||
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
|
||||
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
|
||||
// are the rest sent.
|
||||
// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third
|
||||
// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the
|
||||
// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
|
||||
// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304
|
||||
// decision 3).
|
||||
// Only then are the rest sent.
|
||||
//
|
||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||
@@ -41,22 +44,30 @@ import (
|
||||
|
||||
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
|
||||
var (
|
||||
// gateSettle is how long the first machine must stay healthy, at the least.
|
||||
gateSettle = 2 * time.Minute
|
||||
// gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the
|
||||
// gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR
|
||||
// 0352). Readings gateEvery apart, the first at the apply, put three inside it.
|
||||
gateWithin = 2 * time.Minute
|
||||
// gateBound is how long after the apply the build may take to become healthy.
|
||||
gateBound = 10 * time.Minute
|
||||
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
|
||||
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
|
||||
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
|
||||
// decision 3 changed it, issue 524).
|
||||
// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
|
||||
// the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the
|
||||
// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
|
||||
// decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
gatePasses = 3
|
||||
gateLeafPasses = 1
|
||||
gateLeafSettles = false
|
||||
gateEvery = 40 * time.Second
|
||||
// gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its
|
||||
// start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either
|
||||
// counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR
|
||||
// 0351 decisions 1 and 4).
|
||||
gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin)
|
||||
)
|
||||
|
||||
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
|
||||
// own path, or one holding the mesh's resolver. Every other module is a leaf.
|
||||
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
|
||||
// delivery budget; how many readings its gate takes is gateRuleOf's.
|
||||
func coreModule(module string, m catalogue.Manifest) bool {
|
||||
if _, own := onTheControllersPath[module]; own {
|
||||
return true
|
||||
@@ -69,12 +80,72 @@ func coreModule(module string, m catalogue.Manifest) bool {
|
||||
return coreComponent(module) != ""
|
||||
}
|
||||
|
||||
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
|
||||
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
|
||||
if coreModule(module, m) {
|
||||
return gatePasses, true
|
||||
// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
|
||||
// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it;
|
||||
// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading
|
||||
// (novox/hq ADR 0352). said names the rule in the gate's text.
|
||||
type gateRule struct {
|
||||
passes int
|
||||
within time.Duration
|
||||
witnessed bool
|
||||
said string
|
||||
}
|
||||
|
||||
// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its
|
||||
// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send,
|
||||
// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never
|
||||
// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner,
|
||||
// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4);
|
||||
// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304
|
||||
// decision 3).
|
||||
func gateRuleOf(module string, m catalogue.Manifest) gateRule {
|
||||
switch {
|
||||
case coreComponent(module) == lease.ComponentController:
|
||||
return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " +
|
||||
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||
case coreComponent(module) == lease.ComponentNodeTools:
|
||||
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
|
||||
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||
case module == catalogue.DeliverySeat:
|
||||
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
|
||||
case coreModule(module, m):
|
||||
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
|
||||
"or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)}
|
||||
}
|
||||
return gateLeafPasses, gateLeafSettles
|
||||
r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
|
||||
if gateLeafSettles {
|
||||
r.within = gateWithin
|
||||
r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send",
|
||||
gateLeafPasses, gateWithin)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the
|
||||
// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A
|
||||
// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it
|
||||
// applied (applied, as the gate first read it) and, for the controller, before the new controller took the
|
||||
// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
|
||||
// A controller that starts again takes the lease again, and so moves its floor on. Pure.
|
||||
func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
|
||||
holder *lease.Holder) time.Time {
|
||||
floor := since
|
||||
if !rule.witnessed {
|
||||
return floor
|
||||
}
|
||||
for _, n := range machines {
|
||||
start := since
|
||||
if at, read := applied[n]; read && at.After(start) {
|
||||
start = at
|
||||
}
|
||||
if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) {
|
||||
start = holder.Taken
|
||||
}
|
||||
if end := start.Add(gateWitnessWindow); end.After(floor) {
|
||||
floor = end
|
||||
}
|
||||
}
|
||||
return floor
|
||||
}
|
||||
|
||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||
@@ -164,6 +235,10 @@ type gateFacts struct {
|
||||
sent map[string]inventory.SentDeclaration
|
||||
sentBuilds map[string]map[string]string
|
||||
commits map[string]string
|
||||
// provisioned answers each machine's provisioner client's newest report of a module (novox/hq ADR 0349 §15):
|
||||
// a module the gate's send handed to the provisioner is healthy only once it reported that build placed. Nil
|
||||
// reads as nothing said.
|
||||
provisioned inventory.ProvLookup
|
||||
}
|
||||
|
||||
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
|
||||
@@ -220,7 +295,7 @@ func (f gateFacts) supersededOn(module, machine string) (string, bool) {
|
||||
var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
inv := open.inventory
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: witnessed.all()}
|
||||
rolledBack: witnessed.all(), provisioned: provLookup(ctx)}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
@@ -297,6 +372,12 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
case r.Outcome != inventory.OutcomeApplied:
|
||||
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
|
||||
}
|
||||
// **And its files placed** (novox/hq ADR 0349 §15): a module the send handed to the provisioner is judged only
|
||||
// once that machine's client reported the build's templates placed, so a module whose files are not yet written
|
||||
// is never judged; a failure there is the build's on that machine.
|
||||
if h, why := judgeProvisioned(module, machine, r, f); h != healthGood {
|
||||
return h, why
|
||||
}
|
||||
// **No new condition about it**: about the machine itself, or naming the module on that machine,
|
||||
// raised since the judging began. The gate's own are not evidence about the build. A fault that was
|
||||
// there at the send and reopened since is not new; one that had cleared before the send and came back
|
||||
@@ -378,6 +459,32 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// judgeProvisioned is whether a module's files are placed on a machine, for a module the gate's send handed to the
|
||||
// provisioner there: healthy for any other.
|
||||
func judgeProvisioned(module, machine string, r inventory.Reported, f gateFacts) (health, string) {
|
||||
sent, kept := f.sent[machine]
|
||||
digest := sent.Provisioned[module]
|
||||
if !kept || digest == "" {
|
||||
return healthGood, ""
|
||||
}
|
||||
at := time.Time{}
|
||||
if r.At != nil {
|
||||
at = *r.At
|
||||
}
|
||||
one := inventory.SentDeclaration{Digest: sent.Digest, At: sent.At, Provisioned: map[string]string{module: digest}}
|
||||
got, ok := inventory.ProvisionedSend(machine, one, inventory.AppliedReport{At: at,
|
||||
Outcome: inventory.OutcomeApplied}, true, f.provisioned)
|
||||
switch {
|
||||
case !ok:
|
||||
return healthNotYet, fmt.Sprintf("the provisioner on %s has not reported %s's files placed for the new "+
|
||||
"build", machine, module)
|
||||
case got.Outcome != inventory.OutcomeApplied:
|
||||
return healthBroken, fmt.Sprintf("the provisioner on %s did not place %s's files: %s", machine, module,
|
||||
got.Outcome)
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is
|
||||
// the judging's facts without those conditions, on what each names among the modules the send moved,
|
||||
// and whole what holds the machine back as a whole.
|
||||
@@ -501,6 +608,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
|
||||
return judgeMoves(ctx, open, g, pairs, now)
|
||||
}
|
||||
|
||||
// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one
|
||||
// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352).
|
||||
func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool {
|
||||
most := g.Since.Add(gateWithin)
|
||||
return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most)
|
||||
}
|
||||
|
||||
// judged is one module on one machine, as a gate judges it.
|
||||
type judged struct{ module, node string }
|
||||
|
||||
@@ -511,7 +625,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
if g.Verdict != "" {
|
||||
return g.Verdict, nil
|
||||
}
|
||||
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
|
||||
// A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing:
|
||||
// a module healthy throughout is never held past it (novox/hq ADR 0352).
|
||||
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) {
|
||||
return "", nil
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
@@ -544,6 +660,11 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
facts.sentBuilds[j.node] = builds
|
||||
}
|
||||
}
|
||||
// **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the
|
||||
// walk says it waits there, and its gate starts at the report that says the container replaced.
|
||||
if busyWait(g, pairs, facts, now) {
|
||||
return "", nil
|
||||
}
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
@@ -568,6 +689,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
for _, j := range pairs {
|
||||
w := words[j.node]
|
||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||
if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) {
|
||||
h, said = healthNotYet, why
|
||||
}
|
||||
if h == healthGood || h == healthPerson {
|
||||
if on, named := w.on[j.module]; named {
|
||||
h, said = healthNotYet, on
|
||||
@@ -608,6 +732,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
why = said
|
||||
}
|
||||
}
|
||||
// **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read
|
||||
// each machine report its send applied, and who holds the controller lease, the earliest its reading counts.
|
||||
if g.Applied == nil {
|
||||
g.Applied = map[string]time.Time{}
|
||||
}
|
||||
for node := range byMachine {
|
||||
r, said := facts.reports[node]
|
||||
if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied &&
|
||||
facts.reportedOn(node, r) {
|
||||
g.Applied[node] = *r.At
|
||||
}
|
||||
}
|
||||
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||
for _, m := range modules {
|
||||
var on []string
|
||||
for _, j := range pairs {
|
||||
if j.module == m {
|
||||
on = append(on, j.node)
|
||||
}
|
||||
}
|
||||
rules[m] = gateRuleOf(m, shelf[m])
|
||||
floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder)
|
||||
}
|
||||
tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why,
|
||||
rules: rules, floors: floors}, now)
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading
|
||||
// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's
|
||||
// rule and the earliest its reading counts.
|
||||
type gateJudging struct {
|
||||
modules []string
|
||||
reading map[string]health
|
||||
waits map[string]string
|
||||
worst health
|
||||
why string
|
||||
rules map[string]gateRule
|
||||
floors map[string]time.Time
|
||||
}
|
||||
|
||||
// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but
|
||||
// for the gate it writes.
|
||||
func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
|
||||
modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why
|
||||
if g.Healthy == nil {
|
||||
g.Healthy = map[string]int{}
|
||||
}
|
||||
@@ -623,38 +792,53 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
delete(g.HealthyAt, m)
|
||||
continue
|
||||
}
|
||||
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
|
||||
// often while another module beside it is not yet healthy counts no faster.
|
||||
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
|
||||
g.Healthy[m]++
|
||||
g.HealthyAt[m] = now
|
||||
}
|
||||
if w := waits[m]; w != "" {
|
||||
if g.Waits == nil {
|
||||
g.Waits = map[string]string{}
|
||||
}
|
||||
g.Waits[m] = w
|
||||
}
|
||||
// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the
|
||||
// controller's and the tool runner's after the witness window, any other module's from the send.
|
||||
if now.Before(j.floors[m]) {
|
||||
continue
|
||||
}
|
||||
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
|
||||
// often while another module beside it is not yet healthy counts no faster.
|
||||
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
|
||||
g.Healthy[m]++
|
||||
g.HealthyAt[m] = now
|
||||
}
|
||||
}
|
||||
settled := now.Sub(*g.Since) >= gateSettle
|
||||
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
|
||||
// as the core.
|
||||
wants, settles := 0, false
|
||||
// What the send needs is what its most demanding module needs — the most readings, then the latest
|
||||
// floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout
|
||||
// its two minutes (novox/hq ADR 0352).
|
||||
wants, floor, within, said := 0, time.Time{}, time.Duration(0), ""
|
||||
for _, m := range modules {
|
||||
p, s := gateWants(m, shelf[m])
|
||||
wants, settles = max(wants, p), settles || s
|
||||
r, f := j.rules[m], j.floors[m]
|
||||
if r.passes > wants || (r.passes == wants && f.After(floor)) {
|
||||
wants, within, said = r.passes, r.within, r.said
|
||||
}
|
||||
if f.After(floor) {
|
||||
floor = f
|
||||
}
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
// A send judged on no machine of its modules (its first machine not running them) has no class to
|
||||
// be judged by: it is judged as the core, never passed at once without a reading.
|
||||
wants, settles = gatePasses, true
|
||||
wants, floor, within = gatePasses, *g.Since, gateWithin
|
||||
said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin)
|
||||
}
|
||||
// heldLongEnough is a rule's most: within after the send, healthy at every reading since it.
|
||||
heldLongEnough := func(within time.Duration) bool {
|
||||
return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within))
|
||||
}
|
||||
g.Wants = wants
|
||||
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
|
||||
// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
|
||||
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
|
||||
passedAlone := func(m string) bool {
|
||||
p, s := gateWants(m, shelf[m])
|
||||
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
|
||||
r := j.rules[m]
|
||||
return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within)))
|
||||
}
|
||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||
fail := func(why string) {
|
||||
@@ -671,6 +855,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
pastBound := now.Sub(*g.Since) > gateBound
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
g.Faulted = true
|
||||
g.Read(now, false, g.BrokenWhy)
|
||||
var judging []string
|
||||
for _, m := range modules {
|
||||
@@ -689,25 +874,36 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
case worst == healthWaiting:
|
||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
g.Read(now, false, why)
|
||||
case worst == healthNotYet:
|
||||
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
g.Read(now, false, why)
|
||||
if pastBound {
|
||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||
}
|
||||
case now.Before(floor):
|
||||
// Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351).
|
||||
g.HealthySeen = true
|
||||
// The bound still holds: a module that cannot be read healthy after its floor fails at it.
|
||||
g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s",
|
||||
floor.UTC().Format(time.RFC3339), said), nil
|
||||
if pastBound {
|
||||
fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last))
|
||||
}
|
||||
default:
|
||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||
g.HealthySeen = true
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
g.Read(now, true, "")
|
||||
if g.Passes >= wants && (settled || !settles) {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||
if g.Passes >= wants || heldLongEnough(within) {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
|
||||
}
|
||||
}
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each
|
||||
@@ -1039,7 +1235,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
||||
}
|
||||
return
|
||||
}
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines)
|
||||
if err != nil {
|
||||
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
|
||||
@@ -1132,7 +1328,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
|
||||
}
|
||||
inv := open.inventory
|
||||
sort.Strings(b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines)
|
||||
var back []string
|
||||
for _, r := range b.pending {
|
||||
if err != nil {
|
||||
@@ -1283,6 +1479,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
|
||||
}
|
||||
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
||||
out = append(out, backlogObservation()...)
|
||||
// And a rest that went silent (novox/hq ADR 0353 §4).
|
||||
silent, err := silentObservation(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, silent...)
|
||||
return sortedFound(dedupeObservations(out)), nil
|
||||
}
|
||||
|
||||
@@ -1372,6 +1574,15 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
case inventory.GateFailed:
|
||||
what = "FAILED"
|
||||
}
|
||||
switch {
|
||||
case g.Verdict == "" && g.Pending:
|
||||
what = "kept, not yet sent" // novox/hq ADR 0353 §8
|
||||
case g.Verdict == "" && g.Silent:
|
||||
what = "silent on its send, left" // ADR 0353 §4
|
||||
}
|
||||
if g.Verdict == "" && g.Waiting() {
|
||||
what = "waits for a busy container on " + g.Busy.Machine
|
||||
}
|
||||
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
||||
if g.Component != "" {
|
||||
line += " (core: " + g.Component + ")"
|
||||
|
||||
@@ -21,11 +21,15 @@ import (
|
||||
// wait, and a provider waiting for a login says who waits on it.
|
||||
|
||||
// withGateBounds sets the gate's bounds for one test.
|
||||
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
|
||||
func withGateBounds(t *testing.T, within, every, bound time.Duration) {
|
||||
t.Helper()
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery, gateBound = settle, every, bound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
gateWithin, gateEvery, gateBound = within, every, bound
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
}
|
||||
|
||||
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
|
||||
@@ -53,10 +57,11 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||
withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second)
|
||||
since := time.Now().Add(-time.Second)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"))
|
||||
wasGather := gatherGateFacts
|
||||
@@ -95,9 +100,10 @@ const broker_runtime = "node-tools"
|
||||
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
||||
// while a module beside it is not yet healthy counts the healthy one once.
|
||||
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 0, time.Hour, time.Hour)
|
||||
withGateBounds(t, time.Hour, time.Hour, time.Hour)
|
||||
since := time.Now().Add(-time.Minute)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
|
||||
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
|
||||
@@ -119,6 +125,7 @@ func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
|
||||
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
||||
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
@@ -143,6 +150,7 @@ func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
||||
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
||||
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
@@ -223,6 +231,7 @@ func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
||||
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
||||
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -285,6 +294,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -376,7 +386,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
|
||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
|
||||
return f, nil
|
||||
}
|
||||
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
|
||||
withGateBounds(t, time.Hour, 0, 10*time.Second)
|
||||
|
||||
release := func() inventory.Plan {
|
||||
t.Helper()
|
||||
@@ -427,6 +437,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
|
||||
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
|
||||
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
|
||||
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
|
||||
// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
|
||||
// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
|
||||
|
||||
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
|
||||
func withTheGatesTimes(t *testing.T) {
|
||||
t.Helper()
|
||||
wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
|
||||
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
|
||||
gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
|
||||
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
|
||||
t.Cleanup(func() {
|
||||
gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
|
||||
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
|
||||
})
|
||||
}
|
||||
|
||||
func TestEachModuleHasItsGateRule(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
for _, c := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
passes int
|
||||
within time.Duration
|
||||
witnessed bool
|
||||
said string
|
||||
}{
|
||||
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
||||
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
||||
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
|
||||
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"app", nil, 1, 0, false, "a leaf module's rule"},
|
||||
} {
|
||||
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
||||
if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
|
||||
t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
|
||||
c.witnessed, c.said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
||||
func TestTheCoreStaysTheCore(t *testing.T) {
|
||||
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
||||
if !coreModule(m, catalogue.Manifest{Module: m}) {
|
||||
t.Errorf("%s is no longer judged as the core for its class", m)
|
||||
}
|
||||
}
|
||||
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
|
||||
t.Error("the resolver's holder is no longer judged as the core for its class")
|
||||
}
|
||||
}
|
||||
|
||||
// gateOn judges one module on one machine, healthy at every reading.
|
||||
type gateOn struct {
|
||||
module, node string
|
||||
g *inventory.PlanGate
|
||||
holder *lease.Holder
|
||||
}
|
||||
|
||||
func aGateOn(module, node string, since time.Time) *gateOn {
|
||||
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
|
||||
Applied: map[string]time.Time{}}}
|
||||
}
|
||||
|
||||
func (o *gateOn) read(at time.Time, h health, why string) {
|
||||
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
|
||||
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
||||
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
||||
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
||||
}
|
||||
|
||||
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||
o := aGateOn("mesh-controller", "novox", sent)
|
||||
o.g.Applied["novox"] = sent.Add(15 * time.Second)
|
||||
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
|
||||
|
||||
// Ready, but inside the witness window, which opened before it took the lease: not counted.
|
||||
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
|
||||
o.read(sent.Add(after), healthGood, "")
|
||||
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
|
||||
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
|
||||
}
|
||||
}
|
||||
o.read(sent.Add(80*time.Second), healthGood, "")
|
||||
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
|
||||
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
|
||||
}
|
||||
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
|
||||
!strings.Contains(o.g.Why, "ADR 0351") {
|
||||
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller that starts again takes the lease again: its witness window starts again, and so does the
|
||||
// gate's wait.
|
||||
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
o := aGateOn("mesh-controller", "novox", sent)
|
||||
o.g.Applied["novox"] = sent.Add(10 * time.Second)
|
||||
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
|
||||
o.read(sent.Add(90*time.Second), healthGood, "")
|
||||
if o.g.Verdict != "" {
|
||||
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
|
||||
}
|
||||
o.read(sent.Add(130*time.Second), healthGood, "")
|
||||
if o.g.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("not passed after the new window: %+v", o.g)
|
||||
}
|
||||
}
|
||||
|
||||
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
|
||||
// and from the send at the least.
|
||||
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
|
||||
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
|
||||
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
|
||||
}
|
||||
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
|
||||
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
|
||||
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
|
||||
}
|
||||
}
|
||||
|
||||
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
|
||||
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
o := aGateOn("mesh-controller", "novox", sent)
|
||||
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
|
||||
if o.g.Verdict != inventory.GateFailed {
|
||||
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
|
||||
}
|
||||
o = aGateOn("mesh-controller", "novox", sent)
|
||||
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
|
||||
if o.g.Verdict != "" {
|
||||
t.Fatalf("failed before the bound: %+v", o.g)
|
||||
}
|
||||
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
|
||||
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
|
||||
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
|
||||
}
|
||||
}
|
||||
|
||||
// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
|
||||
func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
|
||||
t.Helper()
|
||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
|
||||
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
|
||||
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
|
||||
return g, func(after time.Duration, h health) {
|
||||
why := ""
|
||||
if h != healthGood {
|
||||
why = "the node-engine reports another build"
|
||||
}
|
||||
tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
|
||||
rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
|
||||
}
|
||||
}
|
||||
|
||||
// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
|
||||
// minutes (novox/hq ADR 0352).
|
||||
func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
for _, module := range []string{"mesh-host", "dnsmasq"} {
|
||||
g, judge := threeReadings(t, module, time.Now())
|
||||
judge(15*time.Second, healthGood)
|
||||
judge(55*time.Second, healthGood)
|
||||
if g.Verdict != "" || g.Passes != 2 {
|
||||
t.Fatalf("%s: after two readings %+v; want still judging", module, g)
|
||||
}
|
||||
judge(95*time.Second, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
|
||||
!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
|
||||
t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
|
||||
// two minutes comes sooner than the spacing after the last pass.
|
||||
func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
last := sent.Add(100 * time.Second)
|
||||
held := &inventory.PlanGate{Since: &sent, LastPass: &last}
|
||||
if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
||||
t.Fatal("the judging at two minutes is not due, or is due before")
|
||||
}
|
||||
held.Faulted = true
|
||||
if dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
||||
t.Fatal("a send that was not healthy at a reading is judged at two minutes")
|
||||
}
|
||||
|
||||
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||
judge(100*time.Second, healthGood)
|
||||
judge(2*time.Minute, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Passes != 2 {
|
||||
t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
|
||||
}
|
||||
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||
judge(2*time.Minute, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Passes != 1 {
|
||||
t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
|
||||
func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||
judge(10*time.Second, healthNotYet)
|
||||
judge(100*time.Second, healthGood)
|
||||
judge(2*time.Minute, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Faulted {
|
||||
t.Fatalf("%+v; want passed at two minutes", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
|
||||
// never healthy fails at the bound and is put back; one found broken fails.
|
||||
func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||
judge(20*time.Second, healthNotYet)
|
||||
judge(60*time.Second, healthGood)
|
||||
judge(100*time.Second, healthGood)
|
||||
judge(2*time.Minute+10*time.Second, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Passes != 3 {
|
||||
t.Fatalf("%+v; want passed only on its third healthy reading", g)
|
||||
}
|
||||
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||
judge(60*time.Second, healthGood)
|
||||
judge(100*time.Second, healthNotYet)
|
||||
judge(2*time.Minute, healthGood)
|
||||
if g.Verdict != "" || !g.Faulted {
|
||||
t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
|
||||
}
|
||||
judge(10*time.Minute+time.Second, healthNotYet)
|
||||
if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
|
||||
t.Fatalf("never healthy again: %+v; want failed at the bound", g)
|
||||
}
|
||||
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||
judge(60*time.Second, healthGood)
|
||||
judge(100*time.Second, healthBroken)
|
||||
if g.Verdict != inventory.GateFailed {
|
||||
t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
|
||||
}
|
||||
}
|
||||
|
||||
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
|
||||
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
o := aGateOn("node-tools", "ace", sent)
|
||||
o.g.Applied["ace"] = sent.Add(10 * time.Second)
|
||||
o.read(sent.Add(30*time.Second), healthGood, "")
|
||||
if o.g.Verdict != "" {
|
||||
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
|
||||
}
|
||||
o.read(sent.Add(70*time.Second), healthGood, "")
|
||||
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
|
||||
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
|
||||
}
|
||||
|
||||
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
|
||||
o.read(sent.Add(5*time.Second), healthGood, "")
|
||||
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
|
||||
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
|
||||
}
|
||||
}
|
||||
|
||||
// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at
|
||||
// two minutes (novox/hq ADR 0352 decision 3).
|
||||
func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
modules := []string{"mesh-controller", "mesh-host"}
|
||||
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||
for _, m := range modules {
|
||||
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
||||
}
|
||||
floors["mesh-host"] = sent
|
||||
floors["mesh-controller"] = sent.Add(90 * time.Second)
|
||||
judge := func(after time.Duration, h health) {
|
||||
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h},
|
||||
worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after))
|
||||
}
|
||||
judge(30*time.Second, healthGood)
|
||||
if g.Passes != 0 {
|
||||
t.Fatalf("a reading before the controller's floor was counted: %+v", g)
|
||||
}
|
||||
judge(70*time.Second, healthNotYet)
|
||||
judge(2*time.Minute, healthGood)
|
||||
if !g.Faulted || g.Verdict != "" {
|
||||
t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
|
||||
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
modules := []string{"mesh-controller", "mesh-host"}
|
||||
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||
for _, m := range modules {
|
||||
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
||||
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
|
||||
}
|
||||
for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
|
||||
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
|
||||
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
|
||||
}
|
||||
if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
|
||||
t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,10 @@ func aGateMesh(t *testing.T) *gateMesh {
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
// The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are.
|
||||
wasGated := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = wasGated })
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
@@ -124,10 +128,14 @@ func aGateMesh(t *testing.T) *gateMesh {
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||
gateSettle, gateEvery = 0, time.Hour
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
gateWithin, gateEvery = time.Hour, time.Hour
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
@@ -267,12 +275,40 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
|
||||
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.
|
||||
func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
gateFirstMachine = false
|
||||
ctx := t.Context()
|
||||
judged := false
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
judged = true
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) {
|
||||
t.Fatalf("sent %v, not both machines in one send", g.sent)
|
||||
}
|
||||
if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged {
|
||||
t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a rolled build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateSettle = time.Hour // a core module could not pass for an hour
|
||||
gateWithin = time.Hour // a core module could not pass on fewer readings for an hour
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
advancePlans(ctx, g.open) // one healthy judging
|
||||
p := g.plan(t)
|
||||
@@ -285,31 +321,6 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
|
||||
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
|
||||
func TestAModuleIsJudgedByItsClass(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
passes int
|
||||
settles bool
|
||||
}{
|
||||
{"app", nil, 1, false},
|
||||
{"nzbget", nil, 1, false},
|
||||
{"mesh-controller", nil, 3, true},
|
||||
{"mesh-host", nil, 3, true},
|
||||
{"node-tools", nil, 3, true},
|
||||
{"nats", nil, 3, true},
|
||||
{catalogue.DeliverySeat, nil, 3, true},
|
||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
|
||||
} {
|
||||
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
||||
if p != c.passes || s != c.settles {
|
||||
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -60,12 +61,30 @@ type sentRecord struct {
|
||||
toldGeneration int64
|
||||
sender string
|
||||
modules []string
|
||||
// plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none.
|
||||
plans []byte
|
||||
}
|
||||
|
||||
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
||||
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
||||
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
||||
r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
||||
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
||||
if len(d.plans) > 0 {
|
||||
kept := map[string]keptPlan{}
|
||||
for module, p := range d.plans {
|
||||
kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]}
|
||||
}
|
||||
// A map of plain strings and slices: it marshals.
|
||||
r.plans, _ = json.Marshal(kept)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in
|
||||
// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans).
|
||||
type keptPlan struct {
|
||||
catalogue.Plan
|
||||
RequiredBy string `json:"required-by,omitempty"`
|
||||
}
|
||||
|
||||
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
||||
|
||||
@@ -8,7 +8,8 @@ import (
|
||||
)
|
||||
|
||||
// A rotation asked through the seat carries why to the command, which records it in the hand-act
|
||||
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
|
||||
// log (novox/hq ADR 0228); and with a provision too, since the vault asks why of every rotation of a
|
||||
// credential it holds (issue 767).
|
||||
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
|
||||
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
|
||||
@@ -19,8 +20,10 @@ func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
|
||||
t.Fatalf("without why: %v %v", argv, err)
|
||||
}
|
||||
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
|
||||
t.Fatalf("why beside a provision was passed over: %v", argv)
|
||||
argv, err = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "why": "leaked",
|
||||
"cause": "leaked"})
|
||||
if err != nil || strings.Join(argv, " ") != "rotate postgres-database --consumer ace --why leaked --cause leaked" {
|
||||
t.Fatalf("why beside a provision: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A delivery group still assembles** (novox/hq ADR 0306 decision 2): a merge whose delivery group has a member
|
||||
// not yet merged waits for the merge window, whether or not a batch is assembling, so the group's members walk in
|
||||
// one batch, in their order (ADR 0276 decision 5). Which groups exist and where each member stands is
|
||||
// mesh-delivery's record (ADR 0239): a group is the pull requests of two repositories or more sharing a head
|
||||
// branch name, which is its id. The controller asks it, through its `groups` verb, when a merge from a head is
|
||||
// heard; a member whose pull request is open (proposed, checked, ready or rejected) is still to merge. When
|
||||
// mesh-delivery does not answer within groupAskWithin, or answers something unreadable, the merge waits: the safe
|
||||
// error is to wait, and its batch says why.
|
||||
|
||||
// groupAskWithin is how long mesh-delivery is given to say a merge's group, while the merge is being heard.
|
||||
var groupAskWithin = 3 * time.Second
|
||||
|
||||
// openPullStates are the states of a delivery whose pull request is open and not merged (mesh-delivery's table):
|
||||
// a member in one of them is still to merge. Closed unmerged is stopped, final, and waits for nothing.
|
||||
var openPullStates = map[string]bool{"proposed": true, "checked": true, "ready": true, "rejected": true}
|
||||
|
||||
// groupMember is one member of a delivery group as mesh-delivery's `groups` says it: `<owner/repo>@<commit> (state)`.
|
||||
type groupMember struct {
|
||||
Repository, State string
|
||||
}
|
||||
|
||||
// groupsLine is one group as mesh-delivery's `groups` says it: its id (the head branch name), its members as text and,
|
||||
// from a mesh-delivery that knows novox/hq ADR 0382, each member again as an object (`member_details`) naming its pull
|
||||
// request's number, whole head, base branch and whether it is open.
|
||||
type groupsLine struct {
|
||||
ID string `json:"id"`
|
||||
Members []string `json:"members"`
|
||||
Details []memberLine `json:"member_details,omitempty"`
|
||||
}
|
||||
|
||||
// memberLine is one member of a group, as `groups` says it beside its text.
|
||||
type memberLine struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
State string `json:"state"`
|
||||
Number int `json:"number"`
|
||||
Head string `json:"head"`
|
||||
Base string `json:"base"`
|
||||
Open bool `json:"open"`
|
||||
}
|
||||
|
||||
// groupsAsked is mesh-delivery's `groups`: every group it keeps, or an error when it cannot be asked or did not
|
||||
// answer within groupAskWithin. A seam a test replaces.
|
||||
var groupsAsked = func(ctx context.Context, held bool) ([]groupsLine, error) {
|
||||
if !held {
|
||||
return nil, fmt.Errorf("no holder of %s is on record to say it", catalogue.DeliverySeat)
|
||||
}
|
||||
// On the bus as this process is: the serving controller's own connection, lent, or — in a verb that runs as a command
|
||||
// of its own, such as a recheck's `delivery check` — one dialled for the moment (onTheBus).
|
||||
var raw json.RawMessage
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
asking, cancel := context.WithTimeout(ctx, groupAskWithin)
|
||||
defer cancel()
|
||||
var err error
|
||||
raw, err = askDeliveryOwner(asking, conn, "groups", map[string]any{})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return nil, fmt.Errorf("nothing serves %s.groups", catalogue.DeliverySeat)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var lines []groupsLine
|
||||
if err := json.Unmarshal(raw, &lines); err != nil {
|
||||
return nil, fmt.Errorf("%s.groups answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// deliveryGroupOf is what mesh-delivery says of the group a head names: its members, and whether the head names a
|
||||
// group at all; an error when it cannot be asked or did not answer. A seam a test replaces.
|
||||
var deliveryGroupOf = func(ctx context.Context, held bool, head string) ([]groupMember, bool, error) {
|
||||
lines, err := groupsAsked(ctx, held)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
for _, l := range lines {
|
||||
if l.ID != head {
|
||||
continue
|
||||
}
|
||||
var members []groupMember
|
||||
for _, s := range l.Members {
|
||||
members = append(members, memberOf(s))
|
||||
}
|
||||
return members, true, nil
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
// memberOf reads one member as `groups` says it: `novox/mesh-sdk@d41d255f28ad (held)`.
|
||||
func memberOf(s string) groupMember {
|
||||
id, state := strings.TrimSpace(s), ""
|
||||
if i := strings.LastIndex(id, " ("); i >= 0 && strings.HasSuffix(id, ")") {
|
||||
id, state = id[:i], id[i+2:len(id)-1]
|
||||
}
|
||||
repository, _, _ := strings.Cut(id, "@")
|
||||
return groupMember{Repository: repository, State: state}
|
||||
}
|
||||
|
||||
// groupWait says whether a merge heard now waits for the window for its delivery group, and why in words; nothing
|
||||
// when it names no head, its head names no group, or every other member is merged or closed. own is the merge's
|
||||
// kind (ownPath).
|
||||
//
|
||||
// A member of another repository whose merge this controller already keeps in a batch not yet cut of the same
|
||||
// kind is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins
|
||||
// that batch by their head. One kept in a batch of the other kind keeps the merge waiting, whatever mesh-delivery
|
||||
// says (review of #250): kinds never share a batch, so the merge's batch is a new one, and cut at once it would
|
||||
// walk before that member's batch is cut and break the group's order (ADR 0276 decision 5). Waiting, the two
|
||||
// batches are cut in the order their windows close, an own-path one first at one look, as cutBatchesHeld cuts
|
||||
// every pair of kinds.
|
||||
//
|
||||
// Asked before the plans are held: a slow answer holds up no walk.
|
||||
func groupWait(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held, own bool) (string, error) {
|
||||
if m.Head == "" {
|
||||
return "", nil
|
||||
}
|
||||
same, other, err := keptFromHead(ctx, inv, m.Head, own)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(other) > 0 {
|
||||
return "group " + m.Head + " has a member waiting in a batch of the other kind: " + strings.Join(other, ", "), nil
|
||||
}
|
||||
members, isGroup, err := deliveryGroupOf(ctx, held, m.Head)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("%s did not say whether group %s has a member still to merge (%v): it waits for the "+
|
||||
"window, the safe side", catalogue.DeliverySeat, m.Head, err), nil
|
||||
}
|
||||
if !isGroup {
|
||||
return "", nil
|
||||
}
|
||||
var open []string
|
||||
for _, g := range members {
|
||||
if strings.EqualFold(g.Repository, m.Owner+"/"+m.Repo) || same[strings.ToLower(g.Repository)] ||
|
||||
!openPullStates[g.State] {
|
||||
continue
|
||||
}
|
||||
open = append(open, g.Repository+" ("+g.State+")")
|
||||
}
|
||||
if len(open) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), nil
|
||||
}
|
||||
|
||||
// keptFromHead are the merges from a head that batches not yet cut hold: the repositories (lower case) of those in
|
||||
// a batch of the kind own says, and, in words, each of the other kind with its batch.
|
||||
func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string, own bool) (map[string]bool, []string, error) {
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
same := map[string]bool{}
|
||||
var other []string
|
||||
for _, b := range batches {
|
||||
merges, err := inv.MergesOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, k := range merges {
|
||||
if announcedOf(k).Head != head {
|
||||
continue
|
||||
}
|
||||
if b.OwnPath() == own {
|
||||
same[strings.ToLower(k.Repository)] = true
|
||||
} else {
|
||||
other = append(other, fmt.Sprintf("%s@%s in %s", k.Repository, short(k.Commit), b.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
return same, other, nil
|
||||
}
|
||||
@@ -105,6 +105,12 @@ var handActVerbs = []handActVerb{
|
||||
// (ADR 0257).
|
||||
{Verb: "mirrors", Decision: "which copies in the store no record names are the mesh's is a person's " +
|
||||
"word (ADR 0257)"},
|
||||
// A move of data a release runs once after its delivery (design 57): a person's step of the release, never a
|
||||
// repair.
|
||||
{Verb: "move-placement", Decision: "a release's move of data runs once after its delivery, on a person's " +
|
||||
"word (design 57, issue 736)"},
|
||||
{Verb: "move-config", Decision: "a release's move of data runs once after its delivery, on a person's " +
|
||||
"word (design 57, issue 664)"},
|
||||
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
|
||||
"person starts (ADR 0236)"},
|
||||
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
|
||||
@@ -118,6 +124,9 @@ var handActVerbs = []handActVerb{
|
||||
{Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"},
|
||||
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||
DecidedFor: []string{causeLeakedInLogs}},
|
||||
// A provider's credential the vault holds, rotated there on a person's word (issue 767): the same judgement.
|
||||
{Verb: "rotate", Decision: "a value a person judged disclosed is replaced on their word",
|
||||
DecidedFor: []string{causeLeakedInLogs}},
|
||||
}
|
||||
|
||||
// personsDecision is whether an act in the log is a person's decision by design, by the verb that
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
want string
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||
"push anchor --wait 0s --why stuck --cause sent-not-reported"},
|
||||
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||
"plans close plan-1 --why the report will not come"},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
|
||||
@@ -167,3 +167,54 @@ func TestASetuidSearchAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||
t.Fatalf("a start: %v, printed %q", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The directory judgement's line asks only a known node, one name and nothing else, fails on a refusal, and says it
|
||||
// changed nothing (novox/hq issue 358).
|
||||
func TestADirectoryJudgementAsksOnlyAKnownNodeAndSaysItIsReadOnly(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||
asked := 0
|
||||
ask := func(answer link.HandOverAnswer) func(node, by string) (link.HandOverAnswer, error) {
|
||||
return func(node, by string) (link.HandOverAnswer, error) {
|
||||
asked++
|
||||
if node != "novox" || by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("asked %q %q", node, by)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
}
|
||||
known := func(string) error { return nil }
|
||||
var out bytes.Buffer
|
||||
for _, args := range [][]string{nil, {"novox", "extra"}, {"-x"}} {
|
||||
if err := judgeDirectoriesAsked(args, known, ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("%v was asked: %v", args, err)
|
||||
}
|
||||
}
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, func(string) error { return errors.New("no node called novox") },
|
||||
ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("an unknown node: %v", err)
|
||||
}
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Refused: "no; nothing was judged"}),
|
||||
&out); err == nil || !strings.Contains(err.Error(), "novox refused") || out.Len() != 0 {
|
||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||
}
|
||||
said := "2 directories declared: 1 placed, 1 would be refused\n/var/lib/a: placed\n/var/lib/b: would be refused: …"
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Said: said}), &out); err != nil ||
|
||||
out.String() != "novox (read-only: nothing on novox was changed)\n"+said+"\n" {
|
||||
t.Fatalf("a judgement: %v, printed %q", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The judgement is a seat verb an agent may call — it reads, and changes nothing on the node — and runs exactly
|
||||
// `node judge-directories <node>` (novox/hq issue 358).
|
||||
func TestTheDirectoryJudgementIsAVerbThatOnlyReads(t *testing.T) {
|
||||
argv, err := argvFor("judge-directories", map[string]any{"node": "novox"})
|
||||
if err != nil || strings.Join(argv, " ") != "node judge-directories novox" {
|
||||
t.Fatalf("the verb runs %v, %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("judge-directories", map[string]any{}); err == nil {
|
||||
t.Fatal("the verb ran with no node")
|
||||
}
|
||||
if !readingVerbs["judge-directories"] {
|
||||
t.Fatal("the judgement is not counted a reading verb")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -642,7 +642,8 @@ func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if u.RollOut {
|
||||
// A recorded module whose move goes out on a machine is still the walk's to send (novox/hq ADR 0377).
|
||||
if u.RollOut || len(wentOut(s.Recorded)) > 0 {
|
||||
return "", "", nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2).
|
||||
//
|
||||
// A restricted stream is heard only by a module that runs as an account of its own (`runs-as`, ADR 0259): the
|
||||
// tool runner's bus account is the one every agent session on its machine reaches the bus as, so the controller
|
||||
// composes no consumer, grant or bind word for a module it carries (broker.ComposedHears). Where a grant still
|
||||
// names such a module, the module hears nothing it was granted, and that is said — in `doctor` and in the
|
||||
// controller's log on every raise and send — naming the module, the stream and the remedy. Never silent.
|
||||
|
||||
const (
|
||||
// probeWithheldID is the self-check's probe of it, and kindGrantComposesNothing the condition it raises.
|
||||
probeWithheldID = "D-heard"
|
||||
kindGrantComposesNothing = "grant-composes-nothing"
|
||||
)
|
||||
|
||||
// probeWithheldHearing says every granted hearing of a restricted stream composed for nothing.
|
||||
func probeWithheldHearing(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
records, err := d.open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return withheldObservations(broker.WithheldOf(records)), nil
|
||||
}
|
||||
|
||||
// withheldObservations is each withheld hearing as a condition of its module on its machine.
|
||||
func withheldObservations(withheld []broker.Withheld) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, w := range withheld {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeModule,
|
||||
ID: w.Module + "." + strings.ToLower(w.Stream) + "." + w.Node,
|
||||
Kind: kindGrantComposesNothing, Machine: w.Node, Severity: conditions.Warning,
|
||||
Summary: w.Said(), Said: fmt.Sprintf("%s on %s granted %s; composed nothing", w.Module, w.Node, w.Stream)})
|
||||
}
|
||||
return sortedFound(out)
|
||||
}
|
||||
|
||||
// sayWithheld puts every withheld hearing in the controller's log, once per raise or send that composes it.
|
||||
func sayWithheld(records broker.Records, indent string) {
|
||||
for _, w := range broker.WithheldOf(records) {
|
||||
fmt.Printf("%sA GRANT COMPOSES NOTHING: %s\n", indent, w.Said())
|
||||
}
|
||||
}
|
||||
|
||||
// withheldWords are the plain words of a withheld hearing: the module, the machine and what to do.
|
||||
func withheldWords(o conditions.Observation) words {
|
||||
module, m := "a module", machineOr(o, "a machine")
|
||||
if o.Scope == conditions.ScopeModule && idPart(o, 0) != "" {
|
||||
module = idPart(o, 0)
|
||||
}
|
||||
return words{Headline: clip(conditions.Capital(fmt.Sprintf("%s on %s hears nothing it was granted", module, m)), 60),
|
||||
Needs: fmt.Sprintf("give %s an account of its own in its manifest; the details say which stream.", module),
|
||||
Explanation: fmt.Sprintf("You granted %s part of a stream that only modules running under an account of "+
|
||||
"their own may hear. It runs inside the shared tool runner on %s, which every assistant session there "+
|
||||
"also uses, so the mesh gives it none of that stream.", module, m),
|
||||
Resolved: fmt.Sprintf("%s on %s hears what it was granted, or is no longer granted it", module, m)}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A grant naming a module the tool runner carries composes nothing, and doctor says so in plain words, naming the
|
||||
// module, the stream and the remedy (novox/hq ADR 0376 §2).
|
||||
func TestAGrantThatComposesNothingIsSaid(t *testing.T) {
|
||||
records := broker.Records{Nodes: []string{"ace"}, Assigned: map[string][]broker.Declared{"ace": {
|
||||
{Module: broker.RuntimeModule},
|
||||
{Module: "reader", Hears: []broker.Heard{{Stream: "MAIL", Emitter: "mesh-mail", Restricted: true,
|
||||
Filters: []string{"mesh.stream.mesh-mail.event.mail.mesh.>"}}}},
|
||||
{Module: "mailer", RunsAs: "mailer", Hears: []broker.Heard{{Stream: "MAIL", Emitter: "mesh-mail", Restricted: true,
|
||||
Filters: []string{"mesh.stream.mesh-mail.event.mail.mesh.>"}}}},
|
||||
}}}
|
||||
found := withheldObservations(broker.WithheldOf(records))
|
||||
if len(found) != 1 {
|
||||
t.Fatalf("said %+v", found)
|
||||
}
|
||||
o := found[0]
|
||||
if o.Kind != kindGrantComposesNothing || o.Machine != "ace" || !strings.Contains(o.Summary, "reader on ace") ||
|
||||
!strings.Contains(o.Summary, "MAIL") || !strings.Contains(o.Summary, "runs-as") {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
w := plainWordings[kindGrantComposesNothing](o)
|
||||
if why, ok := conditions.PlainWords(w, "ace"); !ok {
|
||||
t.Fatalf("not plain: %s (%+v)", why, w)
|
||||
}
|
||||
registered := false
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeWithheldID && p.Kind == kindGrantComposesNothing && p.run != nil {
|
||||
registered = true
|
||||
}
|
||||
}
|
||||
if !registered {
|
||||
t.Fatal("doctor does not run the probe of a grant that composes nothing")
|
||||
}
|
||||
}
|
||||
@@ -146,12 +146,28 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
|
||||
same[m] = current[m].Commit
|
||||
}
|
||||
}
|
||||
why := heldBack(node, modules, same, known, current, plans)
|
||||
// A move under a module's own `record` that recreates nothing and moves no data goes as a module that rolls
|
||||
// out goes (novox/hq ADR 0377): a walk's to send, not a person's.
|
||||
here, copied := current, false
|
||||
for m, was := range same {
|
||||
if now, held := current[m]; held && now.OwnRecord() && !f.identical(m, was, now.Commit) && f.rolls(m, was) {
|
||||
if !copied {
|
||||
copied = true
|
||||
here = make(map[string]inventory.CurrentBuild, len(current))
|
||||
for k, v := range current {
|
||||
here[k] = v
|
||||
}
|
||||
}
|
||||
now.RollOut = true
|
||||
here[m] = now
|
||||
}
|
||||
}
|
||||
why := heldBack(node, modules, same, known, here, plans)
|
||||
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
|
||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||
if planStillToSend(plans, mv.Module, node) == "" {
|
||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
}
|
||||
}
|
||||
if len(why) > 0 {
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
|
||||
// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
|
||||
// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back
|
||||
// behind the build its machine was last sent: the send keeps it there, and carries the rest.
|
||||
|
||||
// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed
|
||||
// their gate.
|
||||
func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) {
|
||||
return func(module, commit string) (askedBuild, bool, error) {
|
||||
at, known := table[module+"@"+commit]
|
||||
b := askedBuild{asked: at}
|
||||
for _, f := range failed {
|
||||
if f == module+"@"+commit {
|
||||
b.failedGate = true
|
||||
}
|
||||
}
|
||||
return b, known, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
|
||||
old, now := t0, t0.Add(time.Minute)
|
||||
table := map[string]time.Time{
|
||||
"mesh-controller@c1": old, "mesh-controller@c2": now,
|
||||
"mesh-host@h1": old, "mesh-host@h2": now,
|
||||
"app@a1": old, "app@a2": now,
|
||||
}
|
||||
sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"}
|
||||
record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"}
|
||||
found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController ||
|
||||
found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine {
|
||||
t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found)
|
||||
}
|
||||
if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") ||
|
||||
!strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") {
|
||||
t.Fatalf("it says the machine, the build kept and the two ways out: %s", said)
|
||||
}
|
||||
|
||||
// The record forward, the same build, or a build the records do not hold: nothing kept.
|
||||
for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} {
|
||||
found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]},
|
||||
map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table))
|
||||
if err != nil || len(found) != 0 {
|
||||
t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A sent build that failed its gate is never ahead: the record put back before it is sent.
|
||||
if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 ||
|
||||
found[0].module != "mesh-host" {
|
||||
t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found)
|
||||
}
|
||||
|
||||
// A gate's own put-back carries its own modules back, and nothing else.
|
||||
putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
|
||||
if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" {
|
||||
t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
|
||||
}
|
||||
if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}},
|
||||
askedIn(table)); len(found) != 2 {
|
||||
t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
|
||||
// controller is the build before it, c1, which passed its gate when it rolled out.
|
||||
func core201(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
record := func(commit string, asked time.Time) inventory.Build {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
|
||||
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
|
||||
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
before := record("c1aaaaaaaaaa", old)
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record("c2bbbbbbbbbb", now)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasMoves := machineMoves
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
t.Cleanup(func() { machineMoves = wasMoves })
|
||||
return open
|
||||
}
|
||||
|
||||
// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
|
||||
// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
|
||||
// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
|
||||
// as a person's push does, clears the condition.
|
||||
func TestReplay201(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
|
||||
judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
|
||||
if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
|
||||
}
|
||||
keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
|
||||
if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
|
||||
t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
|
||||
if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
|
||||
t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
|
||||
}
|
||||
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
|
||||
if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
|
||||
t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
|
||||
}
|
||||
|
||||
putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
|
||||
if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
|
||||
}
|
||||
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
|
||||
if _, found, _ := k.Get(ctx, key); !found {
|
||||
t.Fatal("a send that kept the newer controller cleared the condition")
|
||||
}
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
|
||||
if _, found, _ := k.Get(ctx, key); found {
|
||||
t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
|
||||
}
|
||||
}
|
||||
|
||||
// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
|
||||
// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
|
||||
// failed its gate is never kept.
|
||||
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
|
||||
Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
|
||||
Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healer := withScope(ctx, sendScope{})
|
||||
if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("the healer's send is made: %v %v", kept, err)
|
||||
}
|
||||
if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
|
||||
}
|
||||
if open, _ := k.Open(ctx); len(open) != 0 {
|
||||
t.Fatalf("a completed put-back raised a condition: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept
|
||||
// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own
|
||||
// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing.
|
||||
func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
withConditionsInMemory(t)
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
for _, b := range []struct {
|
||||
commit string
|
||||
at time.Time
|
||||
}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
|
||||
Asked: b.at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send as it is made: the controller kept at c2, the app moved to the record.
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
|
||||
if err != nil || len(sent) != 1 {
|
||||
t.Fatalf("the gated send is made: %v %v", sent, err)
|
||||
}
|
||||
if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
|
||||
t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
|
||||
}
|
||||
builds, _, err := inv.SentBuilds(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &inventory.PlanGate{Carried: moves}
|
||||
f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
|
||||
for _, m := range []string{"app", "mesh-controller"} {
|
||||
if why, superseded := f.supersededOn(m, "anchor"); superseded {
|
||||
t.Fatalf("the gate is superseded by its own send: %s", why)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||
// to another build supersedes the judging: no verdict, nothing put back.
|
||||
func TestReplay352(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||
@@ -67,6 +68,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
||||
|
||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -82,6 +84,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
|
||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -140,6 +143,7 @@ func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
||||
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
||||
// without its send reads the report against the send made last, as before. Pure.
|
||||
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
||||
for _, c := range []struct {
|
||||
r inventory.Reported
|
||||
@@ -185,6 +189,7 @@ func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
||||
// judging is superseded, whatever the machine reports. Pure.
|
||||
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
at := time.Now()
|
||||
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
@@ -212,6 +217,7 @@ func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
// A move of another build of a module to a machine where a release or a plan is judging that module
|
||||
// waits for that judging; the same build to that machine is already there. Pure.
|
||||
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
at := time.Now()
|
||||
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
||||
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
||||
@@ -249,6 +255,7 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
// The controller is never put back to a build that reaches less of the store's schema than the store
|
||||
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
||||
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -345,6 +352,7 @@ func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T)
|
||||
|
||||
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
||||
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
||||
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
||||
t.Fatalf("the holder's build is %q", h.Build)
|
||||
|
||||
@@ -123,11 +123,12 @@ func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) {
|
||||
// that builds the other repository's module alone.
|
||||
func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
@@ -181,12 +182,13 @@ func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
|
||||
// ended. A merge the walk does not answer is refused, and nothing ends.
|
||||
func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
first := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second))
|
||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head)
|
||||
hear(t, open, first, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
hear(t, open, later, t0.Add(11*time.Second))
|
||||
@@ -238,11 +240,12 @@ func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
|
||||
// (review of #232): its window closes, and its walk is at the newer commit, answering both.
|
||||
func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one1, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
@@ -290,11 +293,12 @@ func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
|
||||
// that walked its merges again is refused a retry, saying why.
|
||||
func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one1, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
@@ -335,11 +339,12 @@ func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
|
||||
// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped.
|
||||
func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second))
|
||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
@@ -382,6 +387,7 @@ func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
|
||||
// else since it was read refuses all of it, and no merge moves, no plan is written.
|
||||
func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
endsItsLine(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second))
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 611: mesh-controller #293 changed one test file. Its delivery plan, made while another walk built
|
||||
// build-agent and route-proxy (issue 606), read them whole and named them; at the cut their recorded build sources
|
||||
// held none of its files, so its walk moved nothing and its record named no module. mesh-delivery could not tell
|
||||
// "matched what runs" from "never built" and stopped the delivery, asking the operator about a change already on
|
||||
// every machine. The walk now says, per module, that its build matched what runs (delivery.unchanged, ADR 0267 §5).
|
||||
|
||||
// The shape of #293: a test file of the controller's repository. The controller, built from the root, and the two
|
||||
// modules packaging the repository each recorded a build source holding none of it: each is said unchanged, with
|
||||
// why; a module of another repository is not named.
|
||||
func TestAMergeSaysTheModulesWhoseBuildSourceItLeftUnchanged(t *testing.T) {
|
||||
const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
entries := []inventory.Entry{
|
||||
fromRepo("mesh-controller", controllerRepo, ""),
|
||||
fromRepo("build-agent", catalogueRepo, "modules/build-agent"),
|
||||
fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"),
|
||||
fromRepo("gitea", catalogueRepo, "modules/gitea"),
|
||||
}
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"mesh-controller": {{Own: true, Paths: []string{"go.mod", "module.json", "cmd/mesh-controller/", "internal/builder/"}}},
|
||||
"build-agent": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "cmd/mesh-builder/", "internal/builder/"}},
|
||||
{Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
|
||||
},
|
||||
"route-proxy": {
|
||||
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "examples/route-proxy/"}},
|
||||
{Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
|
||||
},
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "8f1aec0",
|
||||
Paths: []string{"internal/builder/replay363_test.go"}}
|
||||
if moved := reachOfMerge(m, entries, read, sharedRepositoryEdges).Moved(); len(moved) != 0 {
|
||||
t.Fatalf("the fixture moves %v: a test file is in no build source", moved)
|
||||
}
|
||||
got := unchangedOfMerge(m, entries, read)
|
||||
var names []string
|
||||
for n, why := range got {
|
||||
names = append(names, n)
|
||||
if !strings.Contains(why, "holds none of") || !strings.Contains(why, "internal/builder/replay363_test.go") ||
|
||||
!strings.Contains(why, "the build the mesh holds is its build") {
|
||||
t.Errorf("%s is said unchanged as %q", n, why)
|
||||
}
|
||||
}
|
||||
if want := []string{"build-agent", "mesh-controller", "route-proxy"}; !reflect.DeepEqual(sortedOf(names), want) {
|
||||
t.Fatalf("said unchanged: %v; want %v", sortedOf(names), want)
|
||||
}
|
||||
|
||||
// A module whose build source holds a changed file is moved, never said unchanged.
|
||||
touched := m
|
||||
touched.Paths = []string{"internal/builder/builder.go"}
|
||||
if got := unchangedOfMerge(touched, entries, read); got["mesh-controller"] != "" || got["build-agent"] != "" {
|
||||
t.Errorf("a module the merge moves is said unchanged: %v", got)
|
||||
}
|
||||
if got := unchangedOfMerge(touched, entries, read); got["route-proxy"] == "" {
|
||||
t.Errorf("the route proxy, whose build source holds none of it, is not said unchanged: %v", got)
|
||||
}
|
||||
// With no build source recorded, the module is read whole and moved: nothing says it unchanged.
|
||||
unsaid := map[string][]inventory.ReadRepository{"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}}}
|
||||
if got := unchangedOfMerge(m, entries, unsaid); len(got) != 0 {
|
||||
t.Errorf("modules with no recorded build source are said unchanged: %v", got)
|
||||
}
|
||||
// Files not all said: every module is moved, none said unchanged.
|
||||
truncated := m
|
||||
truncated.PathsTruncated = true
|
||||
if got := unchangedOfMerge(truncated, entries, read); len(got) != 0 {
|
||||
t.Errorf("a merge whose files were not all said says %v unchanged", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module the walk builds anyway — another merge's, or a dependent — is not said unchanged: the modules map keeps
|
||||
// meaning what the walk builds, and the two never name one module.
|
||||
func TestAWalkSaysUnchangedOnlyWhatItDoesNotBuild(t *testing.T) {
|
||||
unchanged := map[string]string{"build-agent": "why a", "route-proxy": "why b"}
|
||||
got := unchangedLeft(unchanged, map[string]*inventory.PlanModule{"build-agent": {}})
|
||||
if !reflect.DeepEqual(got, map[string]string{"route-proxy": "why b"}) {
|
||||
t.Fatalf("left %v", got)
|
||||
}
|
||||
if got := unchangedLeft(map[string]string{"x": "y"}, map[string]*inventory.PlanModule{"x": {}}); got != nil {
|
||||
t.Fatalf("left %v; want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Against the store, the path a real merge takes: a merge whose module recorded a build source that holds none
|
||||
// of its files is cut into a walk done with nothing to walk, whose record names the module unchanged and why.
|
||||
func TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main",
|
||||
Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour),
|
||||
Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readme := repoMerge("one", "one1aaaaaaaa", t0)
|
||||
readme.Paths = []string{"README.md"}
|
||||
hear(t, open, readme, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("walks: %+v", ws)
|
||||
}
|
||||
w := ws[0]
|
||||
if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 {
|
||||
t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules)
|
||||
}
|
||||
if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "holds none of the 1 changed file(s)") {
|
||||
t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery)
|
||||
}
|
||||
if !strings.Contains(w.Note, "unchanged") || !strings.Contains(w.Note, "one") {
|
||||
t.Errorf("the walk's note does not name it: %q", w.Note)
|
||||
}
|
||||
|
||||
// A merge that changes the build source is walked, and says nothing unchanged.
|
||||
code := repoMerge("one", "one2bbbbbbbb", t0.Add(3*time.Minute))
|
||||
hear(t, open, code, t0.Add(3*time.Minute+time.Second))
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
if len(ws) != 2 {
|
||||
t.Fatalf("walks: %+v", ws)
|
||||
}
|
||||
if _, built := ws[0].Modules["one"]; !built || ws[0].Delivery == nil || len(ws[0].Delivery.Unchanged) != 0 {
|
||||
t.Fatalf("the walk of a change to main.go reads modules %v, delivery %+v", ws[0].Modules, ws[0].Delivery)
|
||||
}
|
||||
}
|
||||
|
||||
func sortedOf(names []string) []string {
|
||||
out := append([]string{}, names...)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A tier whose module is no move — made from the source every machine running it runs, so nothing is
|
||||
// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
|
||||
// send that last carried that build before the walk got there, and nothing will make them report again:
|
||||
// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
|
||||
func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
// app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates
|
||||
// nothing.
|
||||
for _, m := range []catalogue.Manifest{
|
||||
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
|
||||
{Module: "web", Version: "w1"},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
|
||||
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
|
||||
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
|
||||
} {
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Each machine answered that send, applied, before the walk's tier was built.
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
|
||||
Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
|
||||
At: time.Now().Add(-10 * time.Second)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
built := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Tiers = [][]string{{"app"}, {"web"}}
|
||||
p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
if len(g.sent) != 0 {
|
||||
t.Fatalf("a no-move build was sent: %v", g.sent)
|
||||
}
|
||||
p = g.plan(t)
|
||||
if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
|
||||
t.Fatalf("the build was not marked no move: %+v", s)
|
||||
}
|
||||
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
|
||||
}
|
||||
|
||||
// A machine that has not answered its last send is not waited on either: a built-by edge gates no tier
|
||||
// (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop.
|
||||
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
p = g.plan(t)
|
||||
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||
t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note)
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,32 @@ const issuedAsk = 20 * time.Second
|
||||
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
|
||||
type issuedProvider struct {
|
||||
Node, Module, Tool string
|
||||
// Holders are the provision's holders from this assignment's machine, named to the vault, whose tool may answer
|
||||
// a value for each (issue 749).
|
||||
Holders []issuedHolder
|
||||
}
|
||||
|
||||
// issuedHolder is one holder of a provision from a provider: a consuming module on one machine (issue 749).
|
||||
type issuedHolder struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
|
||||
func (h issuedHolder) String() string { return h.Module + " on " + h.Node }
|
||||
|
||||
// issuedHoldersFrom are the provision's holders from provider, each consuming module on each machine once.
|
||||
func issuedHoldersFrom(holders []inventory.Holder, provider string) []issuedHolder {
|
||||
seen := map[issuedHolder]bool{}
|
||||
var out []issuedHolder
|
||||
for _, h := range holders {
|
||||
one := issuedHolder{Node: h.Consumer, Module: h.ConsumerModule}
|
||||
if h.Provider != provider || seen[one] {
|
||||
continue
|
||||
}
|
||||
seen[one] = true
|
||||
out = append(out, one)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
|
||||
@@ -53,6 +79,10 @@ type issuedTaker struct {
|
||||
keep func(provision, provider, value string) (inventory.IssuedKept, error)
|
||||
record func(act link.HandAct) error
|
||||
send func(machines []string) error
|
||||
// ofTheVault says whether a provider module's secrets are filled the new way; for one that is, the value goes
|
||||
// into the vault (vault, vault_secrets.go) and nothing is kept, sealed or sent here. Nil is none.
|
||||
ofTheVault vaultSwitch
|
||||
vault vaultAsker
|
||||
}
|
||||
|
||||
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
|
||||
@@ -90,6 +120,15 @@ func (t issuedTaker) take(provision string) (string, error) {
|
||||
|
||||
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
|
||||
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
|
||||
if t.ofTheVault != nil {
|
||||
ours, err := t.ofTheVault(p.Module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ours {
|
||||
return t.takeIntoVault(provision, p)
|
||||
}
|
||||
}
|
||||
public, private, err := secrets.Keypair()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no key could be made to take the value: %w", err)
|
||||
@@ -174,6 +213,16 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
out, err := issuedProvidersFrom(shelf, holders, func(node string) ([]string, error) {
|
||||
return inv.Assigned(ctx, node)
|
||||
}, provision)
|
||||
return out, true, err
|
||||
}
|
||||
|
||||
// issuedProvidersFrom are the assignments to ask, from the provision's holders: on every machine providing it, the
|
||||
// assigned modules whose offer says their software issues it, each with that machine's holders (issue 749).
|
||||
func issuedProvidersFrom(shelf map[string]catalogue.Manifest, holders []inventory.Holder,
|
||||
assignedOn func(node string) ([]string, error), provision string) ([]issuedProvider, error) {
|
||||
nodes := map[string]bool{}
|
||||
for _, h := range holders {
|
||||
nodes[h.Provider] = true
|
||||
@@ -185,13 +234,17 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision
|
||||
sort.Strings(names)
|
||||
var out []issuedProvider
|
||||
for _, node := range names {
|
||||
assigned, err := inv.Assigned(ctx, node)
|
||||
assigned, err := assignedOn(node)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, issuedOn(shelf, node, assigned, provision)...)
|
||||
on := issuedOn(shelf, node, assigned, provision)
|
||||
for i := range on {
|
||||
on[i].Holders = issuedHoldersFrom(holders, node)
|
||||
}
|
||||
out = append(out, on...)
|
||||
}
|
||||
return out, true, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// offersIssued is whether any module offers the provision with a credential its software issues.
|
||||
@@ -262,7 +315,9 @@ func secretIssued(ctx context.Context, args []string) error {
|
||||
return err
|
||||
})
|
||||
},
|
||||
send: func(machines []string) error { return sendTo(ctx, open, machines) },
|
||||
send: func(machines []string) error { return sendTo(ctx, open, machines) },
|
||||
ofTheVault: filledTheNewWay,
|
||||
vault: theVault(ctx),
|
||||
}
|
||||
words, err := t.take(provision)
|
||||
if words != "" {
|
||||
|
||||
@@ -193,7 +193,7 @@ func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
|
||||
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
|
||||
}
|
||||
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
|
||||
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
|
||||
if len(got) != 1 || got[0].Node != "ace" || got[0].Module != "plex" || got[0].Tool != "plex_issued_credential" {
|
||||
t.Fatalf("the providers asked: %+v", got)
|
||||
}
|
||||
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
|
||||
@@ -216,3 +216,84 @@ func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
|
||||
}
|
||||
t.Fatal("secret issued is not in the hand-act table")
|
||||
}
|
||||
|
||||
// **A need's credential is the vault's for its holder alone** (issue 757) exactly when its providing module is marked
|
||||
// moved, so `secret issued` goes to the vault naming each holder, and its software issues the provision's credential.
|
||||
func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssuesIt(t *testing.T) {
|
||||
forge := func(moved bool, issued bool) catalogue.Manifest {
|
||||
offer := catalogue.Offer{Name: "gitea-read"}
|
||||
if issued {
|
||||
offer.Credential = &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"}
|
||||
}
|
||||
return catalogue.Manifest{Module: "gitea", Moved: moved, Provides: []catalogue.Offer{offer}}
|
||||
}
|
||||
shared := func(moved bool) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "gitea", Moved: moved, OwnSecrets: catalogue.OwnSecrets{"key": {Path: "/k"}},
|
||||
Provides: []catalogue.Offer{{Name: "gitea-read", Credential: &catalogue.OfferCredential{Own: "key"}}}}
|
||||
}
|
||||
given := func() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "gitea", Moved: true, Provides: []catalogue.Offer{{Name: "gitea-read",
|
||||
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}}
|
||||
}
|
||||
need := catalogue.Needed{Name: "gitea-read", From: "novox", Module: "gitea", For: "claude-code"}
|
||||
for why, c := range map[string]struct {
|
||||
shelf map[string]catalogue.Manifest
|
||||
need catalogue.Needed
|
||||
want bool
|
||||
}{
|
||||
"moved and issued by its software": {map[string]catalogue.Manifest{"gitea": forge(true, true)}, need, true},
|
||||
"not moved": {map[string]catalogue.Manifest{"gitea": forge(false, true)}, need, false},
|
||||
"a login for each consumer": {map[string]catalogue.Manifest{"gitea": forge(true, false)}, need, true},
|
||||
"a login for each, not moved": {map[string]catalogue.Manifest{"gitea": forge(false, false)}, need, false},
|
||||
"a credential every holder shares": {map[string]catalogue.Manifest{"gitea": shared(true)}, need, false},
|
||||
"a credential a person gives": {map[string]catalogue.Manifest{"gitea": given()}, need, false},
|
||||
"a provider not on the shelf": {map[string]catalogue.Manifest{}, need, false},
|
||||
"answered by a record": {map[string]catalogue.Manifest{"gitea": forge(true, true)},
|
||||
catalogue.Needed{Name: "gitea-read", ByRecord: true, Module: "gitea", For: "claude-code"}, false},
|
||||
} {
|
||||
if got := issuedToHolder(c.shelf, c.need); got != c.want {
|
||||
t.Errorf("%s: issued to its holder is %v, want %v", why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The vault is a moved provider for its consumers' credentials** (issue 793): its pair rows are the vault's since
|
||||
// V3, so a need of `secret` is issued to its holder and answered by a moved provider, whether or not the module is
|
||||
// marked moved; another module is not.
|
||||
func TestTheModuleHoldingTheVaultSeatCountsAsMovedForItsConsumers(t *testing.T) {
|
||||
vault := catalogue.Manifest{Module: "mesh-vault", Claims: []catalogue.Claim{{Name: "mesh-vault"}},
|
||||
Provides: []catalogue.Offer{{Name: "secret"}}}
|
||||
other := catalogue.Manifest{Module: "mesh-vault", Provides: []catalogue.Offer{{Name: "secret"}}}
|
||||
need := catalogue.Needed{Name: "secret", Local: "admin", From: "novox", Module: "mesh-vault", For: "gitea"}
|
||||
if !issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": vault}, need) ||
|
||||
!providerMoved(map[string]catalogue.Manifest{"mesh-vault": vault}, need) {
|
||||
t.Error("the vault's consumer is not answered by a moved provider")
|
||||
}
|
||||
if issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": other}, need) ||
|
||||
providerMoved(map[string]catalogue.Manifest{"mesh-vault": other}, need) {
|
||||
t.Error("a module not holding the vault seat counts as moved")
|
||||
}
|
||||
}
|
||||
|
||||
// **A provision answered by the helper makes its provider a moved one for that provision alone** (issue 792): the
|
||||
// provides entry says `"answered-by": "helper"`, the module need not be marked moved, and no credential is the vault's
|
||||
// for it. Another provision of the module, or a module without the entry, does not count.
|
||||
func TestAnOfferAnsweredByTheHelperCountsAsMovedForThatProvisionOnly(t *testing.T) {
|
||||
proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{
|
||||
{Name: "route", AnsweredBy: catalogue.AnsweredByHelper}, {Name: "other"}}}
|
||||
plain := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route"}}}
|
||||
shelf := func(m catalogue.Manifest) map[string]catalogue.Manifest {
|
||||
return map[string]catalogue.Manifest{m.Module: m}
|
||||
}
|
||||
route := catalogue.Needed{Name: "route", From: "novox", Module: "route-proxy", For: "gitea"}
|
||||
other := catalogue.Needed{Name: "other", From: "novox", Module: "route-proxy", For: "gitea"}
|
||||
if !providerMoved(shelf(proxy), route) {
|
||||
t.Error("a provision answered by the helper is not answered by a moved provider")
|
||||
}
|
||||
if providerMoved(shelf(proxy), other) || providerMoved(shelf(plain), route) {
|
||||
t.Error("a provision not answered by the helper counts as moved")
|
||||
}
|
||||
if issuedToHolder(shelf(proxy), route) {
|
||||
t.Error("the flag made a credential the vault's")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787).
|
||||
//
|
||||
// **Covered** are the bus accounts `module issue` mints for a core module (catalogue.CoreCredentialModules) or for a
|
||||
// module marked moved; no other account is. Until the vault makes bus accounts (ADR 0113 clause 6, issue 774) the
|
||||
// controller mints such an account and writes its hash to the bus's user list, as before, and then hands the whole
|
||||
// value at once to the vault's `accept` with origin `issued`: sealed here to the key the vault made for that call, so
|
||||
// the vault alone opens it (ADR 0349 clause 8). The vault keeps it as the module's own secret on that machine and
|
||||
// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. An account
|
||||
// not covered is minted and kept as it was and handed to nobody.
|
||||
|
||||
// vaultAskerForIssue is the vault as `module issue` asks it. A test replaces it.
|
||||
var vaultAskerForIssue = func(ctx context.Context) vaultAsker { return theVault(ctx) }
|
||||
|
||||
// accountIsCovered says whether the bus accounts minted for m are the vault's to hold (ADR 0374 clause 1).
|
||||
func accountIsCovered(m catalogue.Manifest) bool {
|
||||
return catalogue.CarriesVaultSealed(m.Module) || m.Moved
|
||||
}
|
||||
|
||||
// vaultInstalled says whether the vault is assigned to any machine. Until it is, as in genesis (ADR 0113 §15 to §18,
|
||||
// mesh-host's bootstrap, which issues the bus's and the provisioner's accounts before the vault step), there is no
|
||||
// vault to hand to: the value is kept in the controller's store as before, and the vault takes it afterwards by the
|
||||
// move (move-in takes a value of the controller's store with origin made, accepted or issued, as it is). Only a
|
||||
// vault that is installed and does not answer fails the issue.
|
||||
func vaultInstalled(ctx context.Context, inv *inventory.Inventory) (bool, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == "mesh-vault" && len(e.On) > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, when the module's
|
||||
// accounts are covered, and does nothing otherwise. Loud when the vault does not take it: the bus's user list
|
||||
// already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in.
|
||||
func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error {
|
||||
if !accountIsCovered(m) {
|
||||
return nil
|
||||
}
|
||||
if installed, err := vaultInstalled(ctx, inv); err != nil {
|
||||
return fmt.Errorf("whether the vault is installed cannot be read, so %s of %s was not handed to it: %w", name,
|
||||
m.Module, err)
|
||||
} else if !installed {
|
||||
return nil
|
||||
}
|
||||
target := ownSecretArgs(node, m.Module, name)
|
||||
target["origin"] = "issued"
|
||||
if _, err := acceptIntoVault(vaultAskerForIssue(ctx), target, held, "module issue"); err != nil {
|
||||
return fmt.Errorf("the credential is minted and kept here, and the bus's user list holds its hash, but the "+
|
||||
"vault did not take %s of %s on %s: %w (`module issue` again hands a new one)", name, m.Module, node, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below.
|
||||
func init() {
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||
}
|
||||
|
||||
// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed
|
||||
// to the key the vault made for the call, with origin issued. A module that is neither core nor moved hands nothing.
|
||||
func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
theVaultIsInstalled(t, open)
|
||||
inv := open.inventory
|
||||
known := broker.Broker{Fingerprint: "ab"}
|
||||
|
||||
core := aProvisioner()
|
||||
core.Module = "nats"
|
||||
moved := aBot("moved-bot", true)
|
||||
moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"}
|
||||
plain := aBot("kept-bot", false)
|
||||
plain.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/kept-bot/broker"}
|
||||
|
||||
vault := newStandInVault(t)
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||
t.Cleanup(func() {
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||
})
|
||||
|
||||
for _, m := range []catalogue.Manifest{core, moved, plain} {
|
||||
register(t, open, m)
|
||||
if _, err := assign(ctx, open, "laptop", m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
issue := func(m catalogue.Manifest, password string) {
|
||||
t.Helper()
|
||||
if err := issueWith(ctx, inv, m, "laptop", "", known, "10.0.0.1:4222", "module."+m.Module+".laptop", password); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
issue(core, "first-core-password")
|
||||
issue(moved, "first-moved-password")
|
||||
issue(plain, "kept-password")
|
||||
if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said := vault.said()
|
||||
for _, want := range []string{`"module":"nats"`, `"module":"moved-bot"`, `"name":"broker"`,
|
||||
`"name":"` + broker.ProvisionerSecret + `"`, `"node":"laptop"`} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Errorf("the vault was told %s, without %s", said, want)
|
||||
}
|
||||
}
|
||||
if n := strings.Count(said, `"origin":"issued"`); n != 3 {
|
||||
t.Errorf("%d accounts were handed with origin issued, want 3 (the core's two and the moved module's): %s", n, said)
|
||||
}
|
||||
if strings.Contains(said, "kept-bot") {
|
||||
t.Errorf("an account that is neither core nor moved was handed to the vault: %s", said)
|
||||
}
|
||||
if strings.Contains(said, "first-core-password") || strings.Contains(said, "first-moved-password") {
|
||||
t.Errorf("a value crossed the bus in the clear: %s", said)
|
||||
}
|
||||
|
||||
// A re-issue hands the new value, which the vault opens: the whole {url, fingerprint, node, user, password}.
|
||||
issue(moved, "second-moved-password")
|
||||
if got := vault.taken["accept"]; !strings.Contains(got, "second-moved-password") ||
|
||||
!strings.Contains(got, `"user":"module.moved-bot.laptop"`) || !strings.Contains(got, `"node":"laptop"`) ||
|
||||
!strings.Contains(got, `"url":"nats://10.0.0.1:4222"`) || !strings.Contains(got, `"fingerprint":"ab"`) {
|
||||
t.Errorf("the vault was handed %q after a re-issue, not the new account whole", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A vault that does not take the value fails the issue loudly: the bus's user list holds the new hash already.
|
||||
func TestModuleIssueFailsWhenTheVaultDoesNotTakeACoveredAccount(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
moved := aBot("moved-bot", true)
|
||||
moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"}
|
||||
theVaultIsInstalled(t, open)
|
||||
register(t, open, moved)
|
||||
if _, err := assign(ctx, open, "laptop", moved.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vault := newStandInVault(t)
|
||||
vault.refuse = errors.New("the vault is not there")
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||
t.Cleanup(func() {
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||
})
|
||||
err := issueWith(ctx, open.inventory, moved, "laptop", "", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222",
|
||||
"module.moved-bot.laptop", "pw")
|
||||
if err == nil || !strings.Contains(err.Error(), "vault did not take") {
|
||||
t.Fatalf("a vault that took nothing was reported as %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func theVaultIsInstalled(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
vault := catalogue.Manifest{Module: "mesh-vault", Version: "1"}
|
||||
register(t, open, vault)
|
||||
if _, err := assign(t.Context(), open, "laptop", vault.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// In genesis the bus's and the provisioner's accounts are issued before the vault is installed (ADR 0113 §15 to §18):
|
||||
// there is no vault to hand to, so nothing is asked and nothing fails, and the value stays in the controller's store
|
||||
// as before, for the move to carry into the vault. Once the vault is installed, one that does not answer fails.
|
||||
func TestModuleIssueBeforeTheVaultIsInstalledKeepsTheValueAndHandsNothing(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
known := broker.Broker{Fingerprint: "ab"}
|
||||
core := aProvisioner()
|
||||
core.Module = "nats"
|
||||
register(t, open, core)
|
||||
if _, err := assign(ctx, open, "laptop", core.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vault := newStandInVault(t)
|
||||
vault.refuse = errors.New("there is no vault yet")
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||
t.Cleanup(func() {
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||
})
|
||||
|
||||
if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw"); err != nil {
|
||||
t.Fatalf("genesis's issue of the bus's account failed with no vault installed: %v", err)
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil {
|
||||
t.Fatalf("genesis's issue of the client's account failed with no vault installed: %v", err)
|
||||
}
|
||||
if said := vault.said(); said != "" {
|
||||
t.Errorf("the vault was asked before it was installed: %s", said)
|
||||
}
|
||||
if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "nats", "broker"); err != nil || !held {
|
||||
t.Errorf("the value is not kept in the controller's store (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
theVaultIsInstalled(t, open)
|
||||
if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw2"); err == nil {
|
||||
t.Error("a vault that is installed and does not answer did not fail the issue")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send,
|
||||
// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading.
|
||||
func TestAWalkEndsOnItsMachinesReports(t *testing.T) {
|
||||
now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
at := now.Add(-10 * time.Second)
|
||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
||||
"app": {Rest: map[string]inventory.SentDeclaration{
|
||||
"ace": {Digest: "d-ace", At: sent},
|
||||
"novox": {Digest: "d-novox", At: sent},
|
||||
"shanks": {Digest: "d-shanks", At: sent},
|
||||
"laptop": {Digest: "d-laptop", At: long},
|
||||
}},
|
||||
"proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}},
|
||||
}}
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
|
||||
{Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"},
|
||||
{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"},
|
||||
}
|
||||
waiting, failed, silent := walkReports(p, reports, now, nil)
|
||||
if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) ||
|
||||
!slices.Equal(silent, []string{"laptop"}) {
|
||||
t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
reports[2].Declared = "d-shanks"
|
||||
if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 {
|
||||
t.Fatalf("the walk still waits once every machine reported: %v", waiting)
|
||||
}
|
||||
}
|
||||
|
||||
// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before
|
||||
// novox/hq ADR 0360 started.
|
||||
func withTheGate(t *testing.T) {
|
||||
t.Helper()
|
||||
was := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = was })
|
||||
}
|
||||
|
||||
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
|
||||
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
|
||||
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
said := events(t)
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, n := range []string{"desk", "laptop", "anchor"} {
|
||||
if !slices.Contains(*said, "send "+n) {
|
||||
t.Fatalf("%s was not sent: %v", n, *said)
|
||||
}
|
||||
}
|
||||
if slices.Contains(*said, "judge") {
|
||||
t.Fatalf("a machine was judged by its health: %v", *said)
|
||||
}
|
||||
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
|
||||
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
|
||||
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
b.broken["desk"] = true // desk fails what it is sent
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
|
||||
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
|
||||
t.Fatalf("desk was put back: %v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||
// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7).
|
||||
func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
p, err := releaseBacklog(ctx, b.open, "")
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("no release opened: %v", err)
|
||||
}
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
|
||||
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send is made, and the controller is replaced before it keeps it.
|
||||
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
|
||||
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||
}
|
||||
g := b.release(t).Release.Gates["anchor"]
|
||||
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7).
|
||||
func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
reporting := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||
b.sent = append(b.sent, names)
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||
}
|
||||
return reporting(ctx, open, names)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = reporting })
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
g := p.Release.Gates["laptop"]
|
||||
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||
}
|
||||
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
g.Since = &since
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
p = b.release(t)
|
||||
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
|
||||
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||
t.Fatal("a silent machine's build was marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||
t.Fatalf("laptop was put back: %v", sent)
|
||||
}
|
||||
obs, err := silentObservation(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the silence is not said: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once
|
||||
// the engine and the machine's provisioner client both said so.
|
||||
func TestAWalkEndWaitsForTheProvisioner(t *testing.T) {
|
||||
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||
at := now.Add(-10 * time.Second)
|
||||
plan := func(sentAt time.Time) inventory.Plan {
|
||||
return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{
|
||||
"ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
|
||||
"g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
|
||||
}}}}
|
||||
}
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
|
||||
{Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"},
|
||||
}
|
||||
says := map[string]inventory.ProvReport{}
|
||||
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
|
||||
applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at}
|
||||
|
||||
// (a) the engine applied, the client has not said: the end waits on both.
|
||||
waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
|
||||
if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 {
|
||||
t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason.
|
||||
says["ace/app"] = applied
|
||||
says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at}
|
||||
waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
|
||||
if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") ||
|
||||
!strings.Contains(failed[0], "no value") {
|
||||
t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// (c) no client report past ApplySilentAfter: named silent, never applied.
|
||||
delete(says, "g14/app")
|
||||
waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov)
|
||||
if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) {
|
||||
t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// both said it: nothing waits.
|
||||
says["g14/app"] = applied
|
||||
if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
|
||||
t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
// (d) a send handing nothing to the provisioner is the engine's report alone, as before.
|
||||
bare := plan(now.Add(-time.Minute))
|
||||
bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}}
|
||||
if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) {
|
||||
return inventory.ProvReport{}, false
|
||||
}); len(waiting)+len(failed)+len(silent) != 0 {
|
||||
t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
}
|
||||
@@ -55,8 +55,14 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Each renews five times and looks again ten times as often as a serving controller does, so the
|
||||
// handover and the loss below are seen in a second, not in a renewal of five (novox/hq issue 591): what
|
||||
// is checked is the order of events, which does not change with the interval. Each renewal still has a
|
||||
// serving controller's five seconds to answer (acting.go), so a slow bus does not lose A the lease.
|
||||
const renewEvery, poll = lease.RenewEvery / 5, lease.Poll / 10
|
||||
|
||||
// Controller A takes the lease.
|
||||
a := &actor{}
|
||||
a := &actor{renewEvery: renewEvery, poll: poll}
|
||||
aCtx, stopA := context.WithCancel(ctx)
|
||||
defer stopA()
|
||||
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
|
||||
@@ -69,7 +75,7 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
|
||||
b := &actor{}
|
||||
b := &actor{renewEvery: renewEvery, poll: poll}
|
||||
bCtx, stopB := context.WithCancel(ctx)
|
||||
defer stopB()
|
||||
tookB := make(chan (<-chan struct{}), 1)
|
||||
@@ -82,10 +88,11 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
tookB <- lost
|
||||
}()
|
||||
// B looks fifteen times, across one of A's renewals at least.
|
||||
select {
|
||||
case <-tookB:
|
||||
t.Fatal("B took the lease while A held it")
|
||||
case <-time.After(3 * time.Second):
|
||||
case <-time.After(15 * poll):
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B, waiting, may act: %v", err)
|
||||
@@ -125,7 +132,7 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
select {
|
||||
case <-lostB:
|
||||
case <-time.After(2 * lease.RenewEvery):
|
||||
case <-time.After(2 * renewEvery):
|
||||
t.Fatal("B was not told it lost the lease")
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// localNetworksOf is a machine's attached networks judged (novox/hq ADR 0340): which count as its local
|
||||
// network, from what it last said of itself and the links it said face outside it. A machine that has
|
||||
// said none judges to nothing, and a port reached from `local-network` is then admitted from the private
|
||||
// network only, and said so.
|
||||
func localNetworksOf(ctx context.Context, inv *inventory.Inventory, node string) ([]catalogue.LocalNetwork, error) {
|
||||
account, had, err := inv.MachineOf(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outward, err := inv.OutwardLinksOf(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return judgedNetworks(account, had, outward, overlay.Interface), nil
|
||||
}
|
||||
|
||||
// judgedNetworks is the judging alone, from what was read.
|
||||
func judgedNetworks(account inventory.MachineAccount, had bool, outward []string, tunnel string) []catalogue.LocalNetwork {
|
||||
if !had || account.Machine == nil {
|
||||
return nil
|
||||
}
|
||||
attached := make([]catalogue.AttachedNetwork, 0, len(account.Machine.Networks))
|
||||
for _, n := range account.Machine.Networks {
|
||||
attached = append(attached, catalogue.AttachedNetwork{Link: n.Link, Network: n.Network})
|
||||
}
|
||||
return catalogue.JudgeLocalNetworks(attached, outward, tunnel)
|
||||
}
|
||||
|
||||
// localNetworkLines is what `node` shows of a machine's attached networks (ADR 0340 §8): each one, with
|
||||
// whether it counts as the local network and why, or why none is shown.
|
||||
func localNetworkLines(account inventory.MachineAccount, had bool, judged []catalogue.LocalNetwork) []string {
|
||||
out := []string{" its local network (what a port reached from local-network is admitted from):"}
|
||||
switch {
|
||||
case !had || account.Machine == nil:
|
||||
return append(out, " not said — this machine has not reported what it is attached to")
|
||||
case len(account.Machine.Networks) == 0:
|
||||
why := account.Machine.Unknown["networks"]
|
||||
if why == "" {
|
||||
why = "its node-engine is older than the one that says it"
|
||||
}
|
||||
return append(out, " not said — "+why)
|
||||
}
|
||||
counted := 0
|
||||
for _, j := range judged {
|
||||
verdict := "not counted"
|
||||
if j.Counts {
|
||||
verdict = "counts"
|
||||
counted++
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-22s on %-10s %s: %s", j.Network, j.Link, verdict, j.Why))
|
||||
}
|
||||
if counted == 0 {
|
||||
out = append(out, " none counts: a port reached from local-network is admitted from the private network only")
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **node shows what the local network resolved to** (novox/hq ADR 0340 §8): every attached network,
|
||||
// whether it counts, and why the others do not — the public subnet named as never counted.
|
||||
func TestNodeShowsWhichAttachedNetworksCountAndWhy(t *testing.T) {
|
||||
account := inventory.MachineAccount{Architecture: "amd64", Kernel: "linux", Machine: &inventory.MadeOf{
|
||||
Networks: []inventory.AttachedNetwork{
|
||||
{Link: "eth0", Network: "192.168.1.10/24"},
|
||||
{Link: "eth1", Network: "203.0.113.7/24"},
|
||||
{Link: "wg-mesh", Network: "10.10.0.4/24"},
|
||||
}}}
|
||||
judged := judgedNetworks(account, true, []string{"eth0", "eth1"}, "wg-mesh")
|
||||
text := strings.Join(localNetworkLines(account, true, judged), "\n")
|
||||
for _, want := range []string{
|
||||
"192.168.1.0/24", "counts: in a private range (RFC 1918, 192.168.0.0/16)",
|
||||
"203.0.113.0/24", "not counted: public, never counted",
|
||||
"10.10.0.0/24", "the private network's own link",
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("node does not say %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that said none, or whose node-engine is older, is said as that — never as an empty list
|
||||
// that reads like a machine on no network.
|
||||
func TestNodeSaysWhenNoNetworkWasSaid(t *testing.T) {
|
||||
older := inventory.MachineAccount{Machine: &inventory.MadeOf{}}
|
||||
if text := strings.Join(localNetworkLines(older, true, nil), "\n"); !strings.Contains(text, "older than the one that says it") {
|
||||
t.Errorf("an older node-engine is not said:\n%s", text)
|
||||
}
|
||||
public := inventory.MachineAccount{Machine: &inventory.MadeOf{
|
||||
Networks: []inventory.AttachedNetwork{{Link: "eth0", Network: "198.51.100.9/24"}}}}
|
||||
judged := judgedNetworks(public, true, []string{"eth0"}, "wg-mesh")
|
||||
if text := strings.Join(localNetworkLines(public, true, judged), "\n"); !strings.Contains(text, "none counts") {
|
||||
t.Errorf("a machine on a public network only is not said to have no local network:\n%s", text)
|
||||
}
|
||||
if text := strings.Join(localNetworkLines(inventory.MachineAccount{}, false, nil), "\n"); !strings.Contains(text, "not said") {
|
||||
t.Errorf("a machine that never reported is not said:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// The preview of a flip says what a port from the local network keeps: the local networks it stays
|
||||
// open to, or that the machine reported none (ADR 0340 §6).
|
||||
func TestTheFlipPreviewSaysWhatALocalNetworkPortKeeps(t *testing.T) {
|
||||
rules := []catalogue.Rule{{Port: 1883, Protocol: "tcp", From: catalogue.FromLocalNetwork, Because: []string{"mosquitto"}}}
|
||||
at := inventory.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 1883}
|
||||
with := derivedFilter{rules: rules, mesh: []string{"10.10.0.1"}, outwardLinks: []string{"eth0"},
|
||||
local: []string{"192.168.1.0/24"}}
|
||||
if fate := with.fate(at); !strings.Contains(fate, "192.168.1.0/24") {
|
||||
t.Errorf("the preview says %q, without the local network it stays open to", fate)
|
||||
}
|
||||
with.local = nil
|
||||
if fate := with.fate(at); !strings.Contains(fate, "reported none") {
|
||||
t.Errorf("the preview says %q, without saying no local network was reported", fate)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The tests of novox/hq issue 525's window skip: a merge waits for the merge window only when a batch assembling
|
||||
// holds a merge related to it, or its delivery group may have a member still to merge (ADR 0304 decision 1, ADR
|
||||
// 0306 decisions 2 and 3). Walks stay one at a time.
|
||||
|
||||
// A merge from no head that shares nothing, with no walk open, is cut into its walk when it is heard: no window.
|
||||
func TestALoneMergeSharingNothingIsCutAtOnce(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, lone(repoMerge("one", "c1", t0)), t0)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 || !ws[0].Open() || ws[0].Commit != "c1" {
|
||||
t.Fatalf("a lone merge sharing nothing: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
|
||||
}
|
||||
if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(t0) {
|
||||
t.Fatalf("its window closed at %+v; want when it was heard, %s", ws[0].Times, t0)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge related to the batch assembling joins it and waits for the window, which its coming restarts: by its
|
||||
// repository, by a module in common (a dependent of what the batch moves), by an edge of the module graph between
|
||||
// their modules (one that widens nothing), or by its delivery group's head.
|
||||
func TestARelatedMergeJoinsTheBatchAndWaitsForTheWindow(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
for _, c := range []string{"same repository", "a module in common", "an edge of the graph", "the same group head"} {
|
||||
t.Run(c, func(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c1")
|
||||
asksWithPaths(t)
|
||||
inv := open.inventory
|
||||
first := repoMerge("one", "c1", t0)
|
||||
second := lone(repoMerge("two", "c2", t0.Add(10*time.Second)))
|
||||
why := map[string]string{"same repository": "both change novox/one", "a module in common": "both move two",
|
||||
"an edge of the graph": "two depends on builder (built-by)",
|
||||
"the same group head": "both are members of group " + first.Head}[c]
|
||||
switch c {
|
||||
case "same repository":
|
||||
second = lone(repoMerge("one", "c2", t0.Add(10*time.Second)))
|
||||
case "a module in common":
|
||||
// two is built on one: a merge of one moves two too.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "two", Version: "1",
|
||||
Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "one"}}}},
|
||||
inventory.Source{Repository: "novox/two", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "an edge of the graph":
|
||||
// two is built by the build machine: an edge that orders and widens nothing.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-build-machine", Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/builder", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first = repoMerge("builder", "c1", t0)
|
||||
case "the same group head":
|
||||
second = fromHead(repoMerge("two", "c2", t0.Add(10*time.Second)), first.Head)
|
||||
}
|
||||
hear(t, open, first, t0)
|
||||
said := captured(t, func() error {
|
||||
return (following{open: open}).hearMerge(ctx, second, t0.Add(10*time.Second))
|
||||
})
|
||||
if !strings.Contains(said, "("+why+")") {
|
||||
t.Fatalf("the merge was not said related by %q: %s", why, said)
|
||||
}
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the related merge: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if bs[0].Delivery.Batch.CutAtOnce {
|
||||
t.Fatal("a batch a related merge joined is cut at once")
|
||||
}
|
||||
cutAt(t, open, t0.Add(99*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window the related merge restarted closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A merge from a head whose delivery group has a member still to merge waits for the window, even with no batch
|
||||
// assembling, and says why; its sibling joins the same batch by their head (ADR 0306 decision 2). A merge from a
|
||||
// head that names no group shares nothing and is cut at once, as one from no head is.
|
||||
func TestAMergeOfAGroupWaitsEvenAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
groupsWaiting(t, "feat/x")
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce || windowClosed(bs[0], t0) {
|
||||
t.Fatalf("a group's merge alone: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if words := batchWords(bs[0], t0); !strings.HasPrefix(words, "assembling: 90 s left") ||
|
||||
!strings.Contains(words, "group feat/x has a member still to merge: novox/elsewhere (ready)") {
|
||||
t.Fatalf("it reads %q", words)
|
||||
}
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/x"), t0.Add(5*time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the group's second member did not join the first: %+v", bs)
|
||||
}
|
||||
cutAt(t, open, t0.Add(94*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before its window closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(95*time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" {
|
||||
t.Fatalf("at its window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// A merge from a head that names no group, sharing nothing, with no walk open, is cut at once.
|
||||
func TestAHeadedMergeInNoGroupIsCutAtOnce(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "fix/alone"), t0)
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 || !ws[0].Open() {
|
||||
t.Fatalf("a headed merge in no group: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// When mesh-delivery cannot say a head's group, the merge waits for the window, the safe side, and its batch says
|
||||
// why.
|
||||
func TestAGroupNobodyCanSayWaits(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return nil, false, errors.New("nothing serves mesh-delivery.groups")
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce {
|
||||
t.Fatalf("a merge whose group nobody could say: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if words := batchWords(bs[0], t0); !strings.Contains(words, "mesh-delivery did not say whether group feat/x has a "+
|
||||
"member still to merge (nothing serves mesh-delivery.groups): it waits for the window, the safe side") {
|
||||
t.Fatalf("it reads %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
// A member counts as still to merge only while its pull request is open, and not once this controller keeps its
|
||||
// merge: a group whose other members are merged or closed makes nobody wait.
|
||||
func TestAGroupsMergedOrClosedMembersMakeNobodyWait(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
members := []groupMember{{Repository: "novox/one", State: "ready"}, {Repository: "novox/two", State: "published"},
|
||||
{Repository: "novox/three", State: "stopped"}}
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return members, true, nil }
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
m := fromHead(repoMerge("one", "c1", t0), "feat/x")
|
||||
if why, err := groupWait(ctx, open.inventory, m, true, false); err != nil || why != "" {
|
||||
t.Fatalf("the group's other members are merged or closed, yet: %q %v", why, err)
|
||||
}
|
||||
members = append(members, groupMember{Repository: "novox/four", State: "checked"})
|
||||
if why, _ := groupWait(ctx, open.inventory, m, true, false); !strings.Contains(why, "novox/four (checked)") {
|
||||
t.Fatalf("an open member: %q", why)
|
||||
}
|
||||
// novox/four's merge is kept in a batch already: merged, whatever mesh-delivery says yet.
|
||||
hear(t, open, fromHead(repoMerge("two", "c4", t0), "feat/x"), t0)
|
||||
members[3].Repository = "novox/two"
|
||||
if why, _ := groupWait(ctx, open.inventory, m, true, false); why != "" {
|
||||
t.Fatalf("a member this controller keeps the merge of still waits: %q", why)
|
||||
}
|
||||
if got := memberOf("novox/mesh-sdk@d41d255f28ad (held)"); got != (groupMember{Repository: "novox/mesh-sdk", State: "held"}) {
|
||||
t.Fatalf("a member reads %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge related to two batches assembling folds the newer into the older, which answers every merge of both
|
||||
// and names nothing twice; the folded one names the batch that took it over.
|
||||
func TestAMergeRelatedToTwoBatchesFoldsThem(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/a", "feat/b")
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/a"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/b"), t0.Add(5*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 2 {
|
||||
t.Fatalf("two merges sharing nothing: %d batch(es)", len(bs))
|
||||
}
|
||||
older, newer := bs[1], bs[0] // newest first
|
||||
// Of one's repository and from two's head: related to both.
|
||||
hear(t, open, fromHead(repoMerge("one", "c3", t0.Add(10*time.Second)), "feat/b"), t0.Add(10*time.Second))
|
||||
_, bs = walks(t, open)
|
||||
if len(bs) != 1 || bs[0].ID != older.ID || len(bs[0].Delivery.Merges) != 3 {
|
||||
t.Fatalf("after a merge related to both: batches %+v; want %s answering three merges", bs, older.ID)
|
||||
}
|
||||
folded, err := open.inventory.PlanByID(ctx, newer.ID)
|
||||
if err != nil || folded.State != inventory.PlanSuperseded || folded.Delivery == nil ||
|
||||
folded.Delivery.TakenOverBy != older.ID {
|
||||
t.Fatalf("the newer batch reads %s %+v (%v)", folded.State, folded.Delivery, err)
|
||||
}
|
||||
if left, _ := open.inventory.MergesOf(ctx, newer.ID); len(left) != 0 {
|
||||
t.Fatalf("the folded batch still answers %+v", left)
|
||||
}
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
var cut []inventory.Plan
|
||||
for _, w := range ws {
|
||||
if w.State != inventory.PlanSuperseded {
|
||||
cut = append(cut, w)
|
||||
}
|
||||
}
|
||||
if len(cut) != 1 || len(bs) != 0 || cut[0].ID != older.ID || cut[0].CommitOf("novox/one") != "c3" ||
|
||||
cut[0].CommitOf("novox/two") != "c2" {
|
||||
t.Fatalf("the folded batch's walk: %+v, %d batch(es)", cut, len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
// With a walk open, a batch cut at once still waits for it, said queued behind it (walks stay one at a time, ADR
|
||||
// 0276 decision 4); it is cut once that walk ended, with no window to wait for.
|
||||
func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c1")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("no walk open: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID ||
|
||||
!bs[0].Delivery.Batch.CutAtOnce {
|
||||
t.Fatalf("a lone merge during a walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if !strings.HasPrefix(batchWords(bs[0], heard), "queued behind "+first.ID) {
|
||||
t.Fatalf("it reads %q", batchWords(bs[0], heard))
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||
t.Fatalf("the queued batch was not cut once the walk ended: %+v %+v", ws, bs)
|
||||
}
|
||||
if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(heard) {
|
||||
t.Fatalf("its window closed at %+v; want when its merge was heard, %s", ws[0].Times, heard)
|
||||
}
|
||||
}
|
||||
|
||||
// A stopped walk's other merges are walked again in a batch they are related to, and a batch of their own when
|
||||
// they share nothing with the one assembling (ADR 0299, ADR 0304 decision 1): never in a batch of another
|
||||
// change, whose window the stop would close.
|
||||
func TestAStopWalksTheOthersAgainInABatchOfTheirOwnWhenTheyShareNothing(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/cat1aaaaaaaa")
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
|
||||
hear(t, open, cat, t0.Add(time.Second))
|
||||
hear(t, open, one, t0.Add(6*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
walk := ws[0]
|
||||
hear(t, open, repoMerge("two", "two1cccccccc", t0.Add(3*time.Minute)), t0.Add(3*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 {
|
||||
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
|
||||
}
|
||||
other := bs[0]
|
||||
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
|
||||
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said.WalkedAgainBy == "" || said.WalkedAgainBy == other.ID {
|
||||
t.Fatalf("walked again by %q; want a batch of its own, not %s", said.WalkedAgainBy, other.ID)
|
||||
}
|
||||
b, _ := open.inventory.PlanByID(ctx, other.ID)
|
||||
if b.Delivery.Batch == nil || b.Delivery.Batch.Closed || len(b.Delivery.Merges) != 1 {
|
||||
t.Fatalf("the other change's batch reads %+v; want it untouched by the stop", b.Delivery)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue batch's cut folds in every walk waiting for its delivery's word (cutBatch's carry), so what that
|
||||
// walk moves is the batch's too: a lone merge related to the waiting walk joins the batch assembling and waits for
|
||||
// its window, rather than being cut at once as sharing nothing with what the batch's own merges move.
|
||||
func TestALoneMergeRelatedToAWalkTheCutFoldsJoinsTheBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c3")
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
hear(t, open, repoMerge("three", "c3", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
hear(t, open, lone(repoMerge("one", "c2", t0.Add(2*time.Minute+5*time.Second))), t0.Add(2*time.Minute+5*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 || bs[0].Delivery.Batch.CutAtOnce {
|
||||
t.Fatalf("a lone merge related to the waiting walk: batches %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery group with a member of each kind (review of #250): a merge on the controller's own path and a
|
||||
// catalogue merge never share a batch, so the later member cannot join the earlier one's. Kept in a batch of the
|
||||
// other kind, the earlier member keeps the later one waiting, whatever mesh-delivery says yet — never cut at once
|
||||
// to walk before it. The two batches are then cut in the order their windows close (cutBatchesHeld: the oldest
|
||||
// closed batch first, an own-path one first at one look, and the other queued behind a walk that started): the
|
||||
// earlier member's window closes first, so it walks first, in both orders.
|
||||
func TestAGroupAcrossBothKindsKeepsItsOrder(t *testing.T) {
|
||||
for _, ownFirst := range []bool{true, false} {
|
||||
name := "catalogue member first"
|
||||
if ownFirst {
|
||||
name = "controller member first"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
was := deliveryGroupOf
|
||||
// mesh-delivery still says both members ready: it marks one merged only once its walk opened.
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return []groupMember{{Repository: "novox/mesh-controller", State: "ready"},
|
||||
{Repository: "novox/mesh-catalog", State: "ready"}}, true, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds")
|
||||
cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds")
|
||||
first, second := cat, own
|
||||
if ownFirst {
|
||||
first, second = own, cat
|
||||
}
|
||||
second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano)
|
||||
hear(t, open, first, t0)
|
||||
hear(t, open, second, t0.Add(10*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 0 || len(bs) != 2 {
|
||||
t.Fatalf("both members heard: %d walk(s), %d batch(es); want two batches, nothing cut", len(ws), len(bs))
|
||||
}
|
||||
later := bs[0] // newest first
|
||||
if later.Delivery.Batch.CutAtOnce || !strings.Contains(later.Delivery.Batch.Waits,
|
||||
"has a member waiting in a batch of the other kind: novox/"+first.Repo+"@") {
|
||||
t.Fatalf("the later member's batch reads %+v; want it waiting for the earlier member", later.Delivery.Batch)
|
||||
}
|
||||
// The earlier member's window closes first: its batch alone is cut.
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() || ws[0].CommitOf("novox/"+first.Repo) != first.Commit || len(bs) != 1 ||
|
||||
bs[0].ID != later.ID {
|
||||
t.Fatalf("at the earlier window: walks %+v, batches %+v; want the earlier member alone cut", ws, bs)
|
||||
}
|
||||
earlier := ws[0]
|
||||
// The later member's window closes while the earlier walk runs: it queues behind it.
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != earlier.ID {
|
||||
t.Fatalf("the later member's batch does not queue behind the earlier walk: %+v", bs)
|
||||
}
|
||||
earlier.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &earlier); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(101*time.Second))
|
||||
got, err := open.inventory.PlanByID(ctx, later.ID)
|
||||
if err != nil || got.Batch() || got.CommitOf("novox/"+second.Repo) != second.Commit {
|
||||
t.Fatalf("the later member was not walked after the earlier: %s %v", got.State, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// machineLines is what a machine said it is made of (novox/hq issue 160), as `node` shows it: every fact in
|
||||
// one block with its age, the architecture and kernel the profile always carried among them, and each fact
|
||||
// the machine could not determine said as unknown with its why — never as a zero, and never left out.
|
||||
func machineLines(a inventory.MachineAccount, had bool, now time.Time) []string {
|
||||
if !had {
|
||||
return []string{" what it is made of: never said — this machine has not reported a profile"}
|
||||
}
|
||||
m := a.Machine
|
||||
if m == nil {
|
||||
return []string{
|
||||
" what it is made of: not said — its node-engine is older than the one that says it",
|
||||
" " + factLine("architecture", orUnsaid(a.Architecture)),
|
||||
" " + factLine("kernel", orUnsaid(a.Kernel)),
|
||||
}
|
||||
}
|
||||
age := roughly(now.Sub(m.ObservedAt)) + " ago"
|
||||
if m.ObservedAt.IsZero() {
|
||||
// Said with no time: its age is not known, and is never read as two thousand years.
|
||||
age = "at a time it did not say (age unknown)"
|
||||
}
|
||||
out := []string{fmt.Sprintf(" what it is made of, as it said %s:", age)}
|
||||
add := func(name, value string) { out = append(out, " "+factLine(name, value)) }
|
||||
unknown := map[string]string{}
|
||||
for k, v := range m.Unknown {
|
||||
unknown[k] = v
|
||||
}
|
||||
// said is the value, or the fact's own reason it is unknown; a fact neither said nor unknown is a
|
||||
// node-engine that broke its own rule, and is said so rather than shown blank.
|
||||
said := func(fact, value string) string {
|
||||
if value != "" {
|
||||
delete(unknown, fact)
|
||||
return value
|
||||
}
|
||||
if why, ok := unknown[fact]; ok {
|
||||
delete(unknown, fact)
|
||||
return "unknown: " + why
|
||||
}
|
||||
return "unknown: the machine said neither it nor why"
|
||||
}
|
||||
|
||||
add("architecture", orUnsaid(a.Architecture))
|
||||
release := said("kernel release", m.KernelRelease)
|
||||
if m.KernelRelease == "" {
|
||||
release = "(release " + release + ")"
|
||||
}
|
||||
add("kernel", orUnsaid(a.Kernel)+" "+release)
|
||||
dist := said("distribution", m.Distribution)
|
||||
if m.Distribution != "" {
|
||||
dist += " " + said("version", m.Version)
|
||||
}
|
||||
add("distribution", dist)
|
||||
add("processor", said("processor", m.Processor))
|
||||
add("cores", said("cores", countWords(m.Cores)))
|
||||
add("threads", said("threads", countWords(m.Threads)))
|
||||
add("memory", said("memory", bytesWords(m.MemoryBytes)))
|
||||
add("virtual", said("virtual or physical", m.Virtual))
|
||||
up := ""
|
||||
if m.BootedAt != nil {
|
||||
up = fmt.Sprintf("%s, since %s", roughly(now.Sub(*m.BootedAt)), m.BootedAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
add("up", said("uptime", up))
|
||||
add("timezone", said("timezone", m.Timezone))
|
||||
if len(m.Filesystems) == 0 {
|
||||
add("filesystems", said("filesystems", ""))
|
||||
} else {
|
||||
for i, f := range m.Filesystems {
|
||||
name := ""
|
||||
if i == 0 {
|
||||
name = "filesystems"
|
||||
}
|
||||
add(name, fmt.Sprintf("%s %s, %s free (%s %s)", f.Mount, bytesWords(f.SizeBytes),
|
||||
bytesWords(f.FreeBytes), f.Type, f.Device))
|
||||
}
|
||||
if why, ok := unknown["filesystems"]; ok {
|
||||
add("", "and some unknown: "+why)
|
||||
delete(unknown, "filesystems")
|
||||
}
|
||||
}
|
||||
add("public address", said("public address", ""))
|
||||
// Any other fact a newer node-engine named unknown, so nothing it said is dropped.
|
||||
rest := make([]string, 0, len(unknown))
|
||||
for k := range unknown {
|
||||
rest = append(rest, k)
|
||||
}
|
||||
sort.Strings(rest)
|
||||
for _, k := range rest {
|
||||
add(k, "unknown: "+unknown[k])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func factLine(name, value string) string { return fmt.Sprintf("%-15s %s", name, value) }
|
||||
|
||||
func orUnsaid(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "not said"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// countWords is a number a machine said, or nothing when it said none (its unknown says why).
|
||||
func countWords(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(n)
|
||||
}
|
||||
|
||||
// bytesWords is an amount of bytes a person reads, in binary units; nothing when none was said.
|
||||
func bytesWords(n int64) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB"}
|
||||
v, u := float64(n), 0
|
||||
for v >= 1024 && u < len(units)-1 {
|
||||
v /= 1024
|
||||
u++
|
||||
}
|
||||
if u == 0 {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", v, units[u])
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 160: `node` shows every fact a machine said it is made of, in one block with its
|
||||
// age, the architecture and kernel among them; and a fact it could not determine as unknown with why.
|
||||
func TestNodeShowsWhatAMachineIsMadeOfWithItsAge(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
|
||||
booted := now.Add(-75 * time.Hour)
|
||||
a := inventory.MachineAccount{Architecture: "amd64", Kernel: "linux", Machine: &inventory.MadeOf{
|
||||
ObservedAt: now.Add(-3 * time.Hour), KernelRelease: "7.2.8-arch1-2", Distribution: "Arch Linux",
|
||||
Version: "rolling", Processor: "AMD Ryzen 9 8945HS", Cores: 8, Threads: 16, MemoryBytes: 32891494400,
|
||||
Virtual: "physical", BootedAt: &booted, Timezone: "Europe/Brussels",
|
||||
Filesystems: []inventory.MountedFilesystem{{Mount: "/", Device: "/dev/nvme0n1p3", Type: "ext4",
|
||||
SizeBytes: 987978522624, FreeBytes: 31830110208}},
|
||||
Unknown: map[string]string{"public address": "how it learns it is not decided yet"},
|
||||
}}
|
||||
got := strings.Join(machineLines(a, true, now), "\n")
|
||||
t.Log("\n" + got)
|
||||
for _, want := range []string{
|
||||
"what it is made of, as it said 3h ago:",
|
||||
"architecture amd64",
|
||||
"kernel linux 7.2.8-arch1-2",
|
||||
"distribution Arch Linux rolling",
|
||||
"processor AMD Ryzen 9 8945HS",
|
||||
"cores 8",
|
||||
"threads 16",
|
||||
"memory 30.6 GiB",
|
||||
"virtual physical",
|
||||
"up 3d, since",
|
||||
"timezone Europe/Brussels",
|
||||
"filesystems / 920.1 GiB, 29.6 GiB free (ext4 /dev/nvme0n1p3)",
|
||||
"public address unknown: how it learns it is not decided yet",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFactAMachineCouldNotReadIsShownUnknownNeverZero(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
|
||||
a := inventory.MachineAccount{Architecture: "arm64", Kernel: "linux", Machine: &inventory.MadeOf{
|
||||
ObservedAt: now, Unknown: map[string]string{"memory": "MemTotal not found in /proc/meminfo",
|
||||
"cores": "/proc/cpuinfo says no core id", "kernel release": "permission denied"}}}
|
||||
got := strings.Join(machineLines(a, true, now), "\n")
|
||||
t.Log("\n" + got)
|
||||
for _, want := range []string{
|
||||
"memory unknown: MemTotal not found in /proc/meminfo",
|
||||
"cores unknown: /proc/cpuinfo says no core id",
|
||||
"kernel linux (release unknown: permission denied)",
|
||||
// Neither said nor named unknown: the node-engine broke its rule, and that is shown.
|
||||
"timezone unknown: the machine said neither it nor why",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, " 0 ") || strings.Contains(got, "0 B") {
|
||||
t.Error("a fact the machine could not read was shown as a zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderNodeEngineStillShowsArchitectureAndKernel(t *testing.T) {
|
||||
got := strings.Join(machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux"}, true,
|
||||
time.Now()), "\n")
|
||||
if !strings.Contains(got, "older than the one that says it") || !strings.Contains(got, "architecture amd64") ||
|
||||
!strings.Contains(got, "kernel linux") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
if got := strings.Join(machineLines(inventory.MachineAccount{}, false, time.Now()), "\n"); !strings.Contains(got, "never said") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAccountWithoutItsTimeSaysAgeUnknown(t *testing.T) {
|
||||
got := machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux",
|
||||
Machine: &inventory.MadeOf{KernelRelease: "7.2"}}, true, time.Now())[0]
|
||||
if !strings.Contains(got, "age unknown") || strings.Contains(got, "d ago") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,15 @@ func run() error {
|
||||
// 45 §6), so the next controller takes it at once rather than after its age.
|
||||
defer theLease.release()
|
||||
|
||||
// Every walk a verb keeps is said on the bus, whichever verb kept it (novox/hq ADR 0308 decision 1): the save
|
||||
// that starts or ends a walk keeps its mark, and no later save says it again. The serving controller says its
|
||||
// own saves.
|
||||
if args[0] != "serve" {
|
||||
said, done := sayingWhatAVerbKept(ctx)
|
||||
inventory.PlanSaved = said
|
||||
defer done()
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
return buildCommand(ctx, args[1:])
|
||||
@@ -95,6 +104,10 @@ func run() error {
|
||||
return replayCommand(ctx, args[1:])
|
||||
case "kill":
|
||||
return killCommand(ctx, args[1:])
|
||||
case "forget-checks":
|
||||
return forgetChecksCommand(ctx, args[1:])
|
||||
case "try":
|
||||
return tryCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
@@ -108,6 +121,10 @@ func run() error {
|
||||
return imagesCommand(ctx, args[1:])
|
||||
case "mirrors":
|
||||
return mirrorsCommand(ctx, args[1:])
|
||||
case "move-placement":
|
||||
return movePlacementCommand(ctx, args[1:])
|
||||
case "move-config":
|
||||
return moveConfigCommand(ctx, args[1:])
|
||||
case "plans":
|
||||
return plansCommand(ctx, args[1:])
|
||||
case "delivery":
|
||||
@@ -262,6 +279,10 @@ func usage() {
|
||||
artifacts [--json] [--repository <r>] [--collected] every artifact the mesh made: kept and why, eligible, collected (ADR 0251)
|
||||
collect [--json] [--most <n>] what the store's sweep would let go of, now; nothing is done
|
||||
collect --confirm --why <text> [--most <n>] run the sweep now: hold every kept archive, let go of the eligible
|
||||
move-placement [--json] which settings layers still hold placement keys (endpoints, expose, reach, ports); nothing is moved
|
||||
move-placement --confirm --why <text> move them beside the assignment, once, after the delivery (issue 736)
|
||||
move-config [--json] every config value the settings layers hold, and where in mesh-config it goes; nothing is set
|
||||
move-config --confirm --why <text> set them in mesh-config, read them back, and read config from there (issue 664)
|
||||
images <node> [--json] every container image the machine's declaration names, now and as last sent
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
@@ -318,7 +339,7 @@ func usage() {
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||
licence refresh <name> mint a new access token and seal it to every holder
|
||||
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||
rotate <provision> [--consumer <n>] [--module <m>] [--why <text>] a new credential for every holder, both ends at once (--why where the vault holds it)
|
||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||
pin <node> <provision> <from-node> <module>
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
@@ -370,6 +391,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||
// recorded or registered.
|
||||
if result.Check != nil || result.Checked != nil {
|
||||
if isTry(result) {
|
||||
// A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the
|
||||
// build's log. Never said as `checked`, so the forge's holder sets no status.
|
||||
fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID,
|
||||
result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
checked(ctx, result)
|
||||
return nil
|
||||
}
|
||||
@@ -420,8 +448,14 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
}
|
||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
// What the walk's build did, kept on the walk whose ask it answers (novox/hq ADR 0350 §4); a build seat from
|
||||
// before it says nothing, and that is what is kept.
|
||||
made := result.MadeWords()
|
||||
if result.Reused != "" || result.BuiltWhy != "" {
|
||||
fmt.Printf("%s: %s %s\n", result.ID, manifest.Module, made)
|
||||
}
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
planBuiltSaying(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID, made)
|
||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
|
||||
@@ -550,8 +550,17 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
|
||||
out[k] = m
|
||||
}
|
||||
var failures []string
|
||||
// The list of predefined keys at the tree's root (novox/hq ADR 0348 §7): what the build of this tree judges each
|
||||
// manifest against and registration publishes, so what the gate judges each against too. A list that is
|
||||
// wrong is refused by name, and every manifest is then judged against none.
|
||||
predefined, err := catalogue.ReadPredefined(tree)
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",
|
||||
catalogue.PredefinedFile, version, oneLine(err.Error())))
|
||||
predefined = nil
|
||||
}
|
||||
found := map[string]string{} // module → its directory in the tree
|
||||
err := filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
|
||||
err = filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -575,7 +584,19 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
m, err := catalogue.ParseManifestWith(raw, predefined)
|
||||
// The manifest as written, still naming its templates: what the stand-in build resolves, as a real build
|
||||
// does, so a moved module is handed its templates (issue 876).
|
||||
written := m
|
||||
var texts map[string]string
|
||||
if err == nil {
|
||||
// Each template's text from the tree beside the manifest, as the build reads it back from the archive it
|
||||
// packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module.
|
||||
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
|
||||
}
|
||||
if err == nil && written.Templated() {
|
||||
texts, err = catalogue.ReadTemplates(written, filepath.Dir(path))
|
||||
}
|
||||
if err != nil {
|
||||
if inScope(dir) {
|
||||
failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",
|
||||
@@ -600,12 +621,12 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
|
||||
"of one module name (issue 239)", orRoot(dir), m.Module, s.Repository))
|
||||
return nil
|
||||
}
|
||||
resolved, err := standInBuild(m)
|
||||
resolved, err := standInBuild(written, texts)
|
||||
if err == nil {
|
||||
// As registration reads a built manifest: strictly, again (takeIn).
|
||||
var raw []byte
|
||||
if raw, err = json.Marshal(resolved); err == nil {
|
||||
resolved, err = catalogue.ParseManifest(raw)
|
||||
resolved, err = catalogue.ParseManifestWith(raw, predefined)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
@@ -723,15 +744,24 @@ func builtAlike(m catalogue.Manifest) string {
|
||||
// standInBuild resolves a manifest as a build would, with artifacts nobody built: a digest made from the
|
||||
// module's and the artifact's names, in the store's own vocabulary. Composition and validation read the
|
||||
// shape of a reference, never its bytes.
|
||||
func standInBuild(m catalogue.Manifest) (catalogue.Manifest, error) {
|
||||
func standInBuild(m catalogue.Manifest, texts map[string]string) (catalogue.Manifest, error) {
|
||||
if m.Build == nil {
|
||||
return m.Resolve(nil)
|
||||
}
|
||||
// The templates archive carries each template's text by its path inside the archive, as a build reads it back.
|
||||
archive, inside := catalogue.TemplatesArtifact(m), m.TemplatesDir()+"/"
|
||||
read := map[string]string{}
|
||||
for template, text := range texts {
|
||||
read[strings.TrimPrefix(template, inside)] = text
|
||||
}
|
||||
var built []catalogue.Built
|
||||
for _, a := range m.Build.Artifacts {
|
||||
sum := sha256.Sum256([]byte("gate\x00" + m.Module + "\x00" + a.Name))
|
||||
digest := fmt.Sprintf("sha256:%x", sum)
|
||||
b := catalogue.Built{Name: a.Name, Kind: a.Kind, Digest: digest}
|
||||
if a.Name == archive && archive != "" {
|
||||
b.Texts = read
|
||||
}
|
||||
switch a.Kind {
|
||||
case catalogue.ArtifactImage, catalogue.ArtifactUpstream:
|
||||
b.Reference = catalogue.ArtifactStoreScheme + m.Module + "/" + a.Name + "@" + digest
|
||||
@@ -1083,6 +1113,19 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
|
||||
continue
|
||||
}
|
||||
}
|
||||
// And the provisioner client's account, the same way (novox/hq ADR 0349 clause 11, design 57 P3).
|
||||
if a.Provider == "" && a.Name == broker.ProvisionerSecret {
|
||||
if pm, held := shelf[a.Module]; !held || !runsTheProvisionerClient(pm) {
|
||||
continue
|
||||
}
|
||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Kind: inventory.BusProvisioner, Node: m.Name,
|
||||
Module: a.Module, Username: broker.Principal{Kind: broker.KindProvisioner, Node: m.Name,
|
||||
Module: a.Module}.Username()}); err != nil {
|
||||
notes = append(notes, fmt.Sprintf("%s's provisioner client account for %s is not kept: %s",
|
||||
m.Described(), a.Module, oneLine(err.Error())))
|
||||
continue
|
||||
}
|
||||
}
|
||||
var err error
|
||||
if a.Provider == "" {
|
||||
err = inv.AcceptSecretForModule(ctx, m.Name, a.Module, a.Name, standIn)
|
||||
|
||||
@@ -33,7 +33,21 @@ import (
|
||||
func aMesh(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
|
||||
licences.ForTest(t) // planning reaches this one too, by name and never by connection
|
||||
return aMeshOnTheStore(t)
|
||||
}
|
||||
|
||||
// aMeshBefore is aMesh on a store migrated only below migration `number`, with the function that runs the
|
||||
// rest: for a test of what a migration that moves data does to what the mesh composes (novox/hq issue 651).
|
||||
func aMeshBefore(t *testing.T, number int) (*stores, func()) {
|
||||
t.Helper()
|
||||
_, migrate := inventory.ForTestBefore(t, number)
|
||||
return aMeshOnTheStore(t), migrate
|
||||
}
|
||||
|
||||
// aMeshOnTheStore is the mesh aMesh makes, on the store the environment points at.
|
||||
func aMeshOnTheStore(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
licences.ForTest(t) // planning reaches this one too, by name and never by connection
|
||||
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
|
||||
@@ -7,7 +7,10 @@ import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -123,8 +126,82 @@ func answerMeshCLI(inv *inventory.Inventory) link.CLIHandler {
|
||||
// cliJournal says one line in the controller's journal (its standard output); a variable so a test can read it.
|
||||
var cliJournal = func(line string) { fmt.Println(line) }
|
||||
|
||||
// runForMeshCLI runs a judged line and answers what it said. Every line is said in the journal first: its call,
|
||||
// who asked on which node, its command word only, and how it runs (review of ADR 0272).
|
||||
// onlyReads says a line only reads, judged by the generic `command` verb's reading forms (novox/hq ADR 0266) — the
|
||||
// same allow list that decides what an ordinary call may run, never a second list kept here — judged on the line's
|
||||
// every word, so `plans go`, `conditions silence` and `settings set` are not reads while `plans`, `conditions` and
|
||||
// `settings show` are. A line those forms do not name is not judged a read, and is said (novox/hq issue 762).
|
||||
func onlyReads(asked link.CLIAsked, v cliVerdict) bool {
|
||||
if v.refused != "" || len(asked.Line) == 0 || len(asked.Stdin) > 0 || cliServers[asked.Line[0]] {
|
||||
return false
|
||||
}
|
||||
return commandReads(asked.Line) == nil && terminalOnly(asked.Line) == nil
|
||||
}
|
||||
|
||||
// readCalls counts the read calls said in no line, per caller, until the next hourly line says them.
|
||||
type readCalls struct {
|
||||
mu sync.Mutex
|
||||
counts map[string]int
|
||||
}
|
||||
|
||||
// cliReads are this controller's read calls not yet said.
|
||||
var cliReads = &readCalls{}
|
||||
|
||||
func (r *readCalls) add(caller string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.counts == nil {
|
||||
r.counts = map[string]int{}
|
||||
}
|
||||
r.counts[caller]++
|
||||
}
|
||||
|
||||
// said is the line that says the read calls counted since the last, and starts counting again; empty when there
|
||||
// were none, so an idle hour says nothing.
|
||||
func (r *readCalls) said(over time.Duration) string {
|
||||
r.mu.Lock()
|
||||
counts := r.counts
|
||||
r.counts = nil
|
||||
r.mu.Unlock()
|
||||
if len(counts) == 0 {
|
||||
return ""
|
||||
}
|
||||
callers := make([]string, 0, len(counts))
|
||||
for c := range counts {
|
||||
callers = append(callers, c)
|
||||
}
|
||||
sort.Strings(callers)
|
||||
parts := make([]string, len(callers))
|
||||
for i, c := range callers {
|
||||
parts[i] = fmt.Sprintf("%s %d", c, counts[c])
|
||||
}
|
||||
return fmt.Sprintf("mesh-cli read calls in the last %s, said in no line of their own: %s", over,
|
||||
strings.Join(parts, ", "))
|
||||
}
|
||||
|
||||
// readCallsEvery is how often the count of read calls is said.
|
||||
const readCallsEvery = time.Hour
|
||||
|
||||
// sayReadCalls says, every readCallsEvery until ctx ends, how many read calls each caller made — when any did — so
|
||||
// a runaway poller stays visible while its calls write no line each (novox/hq issue 762).
|
||||
func sayReadCalls(ctx context.Context) {
|
||||
tick := time.NewTicker(readCallsEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if line := cliReads.said(readCallsEvery); line != "" {
|
||||
cliJournal(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runForMeshCLI runs a judged line and answers what it said. Every line that changes something, is refused or fails
|
||||
// is said in the journal: its call, who asked on which node, its command word only, and how it runs (review of ADR
|
||||
// 0272). A line that only reads and succeeds is counted instead, and the count said hourly (novox/hq issue 762): a
|
||||
// page polling `plans` every few seconds would otherwise write thousands of lines a day.
|
||||
func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer {
|
||||
how := "as an ordinary call"
|
||||
switch {
|
||||
@@ -137,7 +214,23 @@ func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliV
|
||||
if call == "" {
|
||||
call = "(no call)"
|
||||
}
|
||||
cliJournal(fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how))
|
||||
line := fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how)
|
||||
if !onlyReads(asked, v) {
|
||||
cliJournal(line)
|
||||
return runJudgedLine(ctx, node, asked, v)
|
||||
}
|
||||
answer := runJudgedLine(ctx, node, asked, v)
|
||||
if answer.Exit != 0 || answer.Refused != "" {
|
||||
// A read that was refused or failed is said, as a change is.
|
||||
cliJournal(fmt.Sprintf("%s, and it failed (exit %d)", line, answer.Exit))
|
||||
} else {
|
||||
cliReads.add(fmt.Sprintf("%s on %s", asked.Account, node))
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// runJudgedLine runs a line runForMeshCLI has said or counted.
|
||||
func runJudgedLine(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer {
|
||||
if v.refused != "" {
|
||||
return link.CLIRefusal(v.refused)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// echoFails makes the echoing test binary end with a failure, as a read that failed would.
|
||||
const echoFails = "MESH_TEST_ECHO_FAILS"
|
||||
|
||||
// journalled runs lines through runForMeshCLI and answers what the journal said, with a fresh count of read calls.
|
||||
func journalled(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
t.Setenv(echoEnvironment, "1")
|
||||
said := &[]string{}
|
||||
was, wasReads := cliJournal, cliReads
|
||||
cliJournal = func(line string) { *said = append(*said, line) }
|
||||
cliReads = &readCalls{}
|
||||
t.Cleanup(func() { cliJournal, cliReads = was, wasReads })
|
||||
return said
|
||||
}
|
||||
|
||||
// **A call that only reads writes no journal line; one that changes something does** (novox/hq issue 762).
|
||||
func TestAReadCallWritesNoJournalLine(t *testing.T) {
|
||||
said := journalled(t)
|
||||
ctx := link.WithCallID(context.Background(), "call-1")
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
for _, line := range [][]string{{"plans"}, {"status"}, {"conditions"}, {"node", "list"}, {"builds"}, {"queue"},
|
||||
{"settings", "show", "zsh"}, {"plans", "--json"}} {
|
||||
if a := runForMeshCLI(ctx, "g14", cliAsked("jochen", 1000, line...), ordinary); a.Exit != 0 || a.Refused != "" {
|
||||
t.Fatalf("%q did not run: %+v", line, a)
|
||||
}
|
||||
}
|
||||
runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "plans"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if len(*said) != 0 {
|
||||
t.Fatalf("read calls wrote journal lines: %q", *said)
|
||||
}
|
||||
runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "assign", "g14", "zsh"), cliVerdict{terminal: true, why: "the terminal"})
|
||||
if len(*said) != 1 || !strings.Contains((*said)[0], `asked "assign"`) {
|
||||
t.Fatalf("a call that changes something was not said: %q", *said)
|
||||
}
|
||||
}
|
||||
|
||||
// **A refused or failed read keeps its line** (novox/hq issue 762).
|
||||
func TestAFailedOrRefusedReadIsSaid(t *testing.T) {
|
||||
said := journalled(t)
|
||||
runForMeshCLI(context.Background(), "g14", cliAsked("root", 0, "plans"), cliVerdict{refused: "root is not answered"})
|
||||
t.Setenv(echoFails, "1")
|
||||
a := runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), cliVerdict{why: "not the terminal"})
|
||||
if a.Exit == 0 {
|
||||
t.Fatalf("the read did not fail: %+v", a)
|
||||
}
|
||||
if len(*said) != 2 || !strings.Contains((*said)[0], "refused") || !strings.Contains((*said)[1], "failed (exit 3)") {
|
||||
t.Fatalf("the journal said %q", *said)
|
||||
}
|
||||
if line := cliReads.said(readCallsEvery); line != "" {
|
||||
t.Fatalf("a failed read was counted as a read: %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verb is classed by what the call does, not its name** (novox/hq issue 762): the same command word reads with
|
||||
// some arguments and writes with others, and a line no reading form names is said.
|
||||
func TestAVerbIsClassedByItsArguments(t *testing.T) {
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
for _, c := range []struct {
|
||||
line []string
|
||||
reads bool
|
||||
}{
|
||||
{[]string{"settings", "show", "zsh"}, true},
|
||||
{[]string{"settings", "preferences"}, true},
|
||||
{[]string{"settings", "set", "zsh", `{"a":1}`}, false},
|
||||
{[]string{"settings", "clear", "zsh"}, false},
|
||||
{[]string{"plans"}, true},
|
||||
{[]string{"plans", "p-1"}, true},
|
||||
{[]string{"plans", "go", "p-1"}, false},
|
||||
{[]string{"plans", "--json", "stop", "p-1"}, false},
|
||||
{[]string{"plans", "close", "p-1"}, false},
|
||||
{[]string{"plans", "retry", "p-1"}, false},
|
||||
{[]string{"conditions"}, true},
|
||||
{[]string{"conditions", "show", "x"}, true},
|
||||
{[]string{"conditions", "silence", "x"}, false},
|
||||
{[]string{"doctor", "run"}, false},
|
||||
{[]string{"node", "account", "g14", "x"}, false},
|
||||
{[]string{"assign", "g14", "zsh"}, false},
|
||||
{[]string{"a-command-added-later"}, false},
|
||||
{[]string{"serve"}, false},
|
||||
} {
|
||||
if got := onlyReads(cliAsked("jochen", 1000, c.line...), ordinary); got != c.reads {
|
||||
t.Errorf("%q judged a read: %v, want %v", c.line, got, c.reads)
|
||||
}
|
||||
}
|
||||
if onlyReads(cliAsked("jochen", 1000, "plans"), cliVerdict{refused: "no"}) {
|
||||
t.Error("a refused line judged a read")
|
||||
}
|
||||
stdin := cliAsked("jochen", 1000, "status")
|
||||
stdin.Stdin = []byte("x")
|
||||
if onlyReads(stdin, cliVerdict{terminal: true}) {
|
||||
t.Error("a line carrying standard input judged a read")
|
||||
}
|
||||
}
|
||||
|
||||
// **Once an hour the read calls are counted per caller, only if there were any** (novox/hq issue 762).
|
||||
func TestTheReadCallsAreCountedPerCaller(t *testing.T) {
|
||||
said := journalled(t)
|
||||
ordinary := cliVerdict{why: "not the terminal"}
|
||||
for range 3 {
|
||||
runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), ordinary)
|
||||
}
|
||||
runForMeshCLI(context.Background(), "ace", cliAsked("jochen", 1000, "status"), ordinary)
|
||||
if len(*said) != 0 {
|
||||
t.Fatalf("read calls wrote lines: %q", *said)
|
||||
}
|
||||
line := cliReads.said(readCallsEvery)
|
||||
if !strings.Contains(line, "jochen on ace 1, jochen on g14 3") || !strings.Contains(line, "1h0m0s") {
|
||||
t.Fatalf("the hourly line said %q", line)
|
||||
}
|
||||
if again := cliReads.said(readCallsEvery); again != "" {
|
||||
t.Fatalf("an hour with no read calls said %q", again)
|
||||
}
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved,
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
f := following{open: open}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -75,7 +75,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||
for _, w := range r.Waits {
|
||||
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
|
||||
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, Provision: w.Provision, What: w.What})
|
||||
}
|
||||
resources = append(resources, kept)
|
||||
}
|
||||
@@ -240,6 +240,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
continue
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
declaredSeverity(&o, hold, m)
|
||||
if hold != nil {
|
||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
||||
@@ -347,6 +348,7 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
|
||||
} else {
|
||||
_, rest := besideAWait(p.Module, rs)
|
||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
||||
declaredSeverity(&o, hold, p.Module)
|
||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
||||
}
|
||||
@@ -481,6 +483,19 @@ func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
}
|
||||
|
||||
// declaredSeverity raises a module's unhealthy condition urgent when its manifest declares `unhealthy` urgent
|
||||
// (novox/hq ADR 0355 §1). Only that condition changes (ADR 0355 §4): a wait for a new login, a module that needs
|
||||
// the operator and a directory used as found keep their severities, and without the field, or without a manifest
|
||||
// to read it from, it stays the warning it was.
|
||||
func declaredSeverity(o *conditions.Observation, hold *holding, module string) {
|
||||
if o.Kind != kindModuleUnhealthy || hold == nil {
|
||||
return
|
||||
}
|
||||
if m, ok := hold.manifestOf(module); ok && m.UnhealthyIsUrgent() {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -127,6 +128,10 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2).
|
||||
if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil {
|
||||
return err
|
||||
}
|
||||
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -237,7 +242,15 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
says = append(says, "provides "+describeOffers(m.Provides))
|
||||
}
|
||||
if len(m.Requires) > 0 {
|
||||
says = append(says, "requires "+strings.Join(m.Requires, ", "))
|
||||
// An optional requirement says so (novox/hq ADR 0330): it is not refused when absent.
|
||||
required := make([]string, 0, len(m.Requires))
|
||||
for _, r := range m.Requires {
|
||||
if m.IsOptional(r) {
|
||||
r += " (optional)"
|
||||
}
|
||||
required = append(required, r)
|
||||
}
|
||||
says = append(says, "requires "+strings.Join(required, ", "))
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
says = append(says, "claims "+c.At()+"/"+c.Name)
|
||||
@@ -433,6 +446,10 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
if len(positionals) == 2 {
|
||||
values = positionals[1]
|
||||
}
|
||||
// Once its config values are mesh-config's, they are proposed there (issue 664).
|
||||
if forwarded, err := forwardPropose(ctx, inv, positionals[0], *node, values, *clear); forwarded || err != nil {
|
||||
return err
|
||||
}
|
||||
return proposeCommand(ctx, positionals[0], *node, values, *clear, *replace)
|
||||
case "proposals":
|
||||
if len(positionals) > 1 || *node != "" || *clear || *replace || *history {
|
||||
@@ -453,6 +470,15 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
where = *node
|
||||
}
|
||||
|
||||
// Once the move of config has run, a module in the config form is read and set through mesh-config (issue 664).
|
||||
if (args[0] == "set" && len(positionals) == 2) || ((args[0] == "show" && !*history) || args[0] == "clear") &&
|
||||
len(positionals) == 1 {
|
||||
if forwarded, err := forwardSettings(ctx, inv, args[0], positionals[0], *node, positionals, *replace); forwarded ||
|
||||
err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "set":
|
||||
if len(positionals) != 2 {
|
||||
@@ -529,6 +555,10 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
if p.SetBy != "" {
|
||||
by = "; " + p.SetBy
|
||||
}
|
||||
if p.Why != "" {
|
||||
// A layer whose placement moved out of it (migration 0097): still the layer, as it stands.
|
||||
by += "; " + p.Why
|
||||
}
|
||||
fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where,
|
||||
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown)
|
||||
}
|
||||
@@ -932,7 +962,55 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||
if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil {
|
||||
return err
|
||||
}
|
||||
if !runsTheProvisionerClient(m) {
|
||||
return nil
|
||||
}
|
||||
return issueProvisionerClient(ctx, inv, m, node, known, reachable)
|
||||
}
|
||||
|
||||
// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an
|
||||
// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret
|
||||
// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor).
|
||||
func runsTheProvisionerClient(m catalogue.Manifest) bool {
|
||||
_, reads := m.OwnSecrets[broker.ProvisionerSecret]
|
||||
return reads && m.ClaimsSeat(broker.NodeProvisionerSeat)
|
||||
}
|
||||
|
||||
// issueProvisionerClient gives a machine's provisioner client its account: minted under the client's own name,
|
||||
// never the module's or the tool runner's, and sealed to the machine as the module's own secret
|
||||
// broker.ProvisionerSecret, which the client's service reads (novox/hq ADR 0349 clause 11, design 57 P3). No
|
||||
// claims and no consumer: the client holds no seat on this account and is delivered nothing but its answers.
|
||||
func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string,
|
||||
known broker.Broker, reachable string) error {
|
||||
user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: inventory.BusProvisioner, Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}{URL: "nats://" + reachable, Fingerprint: known.Fingerprint, Node: node, User: user, Password: password})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, node, m.Module, broker.ProvisionerSecret, string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := handIssuedToVault(ctx, inv, m, node, broker.ProvisionerSecret, string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("bus user %s minted for %s's provisioner client, scoped to %s's keys, its reports and the vault\n",
|
||||
user, m.Module, node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
|
||||
@@ -977,6 +1055,9 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := handIssuedToVault(ctx, inv, m, node, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// And how it hears what it consumes. Derived from its declaration, and only when it declared
|
||||
// something: a module that consumes nothing needs no consumer, and creating one would be a
|
||||
@@ -1051,6 +1132,11 @@ func namesNoInstallation(m catalogue.Manifest) error {
|
||||
if problems := catalogue.TrustProblems(m); len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
// And a module that declares a setting names the keys of its mergeable files (novox/hq ADR 0326
|
||||
// §3, issue 574): judged where a definition is written, never where a stored build is read again.
|
||||
if problems := catalogue.WrittenSettingsProblems(m); len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
named := catalogue.InstallationProblems(m)
|
||||
if len(named) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -684,7 +684,7 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||
serves, _, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
"github.com/novox/mesh-controller/internal/token"
|
||||
)
|
||||
|
||||
@@ -127,9 +128,14 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
// and whoever may call a verb includes agents.
|
||||
return nodeSetuidSearch(ctx, open, args[1:])
|
||||
|
||||
case "judge-directories":
|
||||
// Which of a node's declared directories the walk that follows no link another account could make would
|
||||
// refuse (novox/hq issue 358). Read-only on the node and here, so a verb may run it too.
|
||||
return nodeJudgeDirectories(ctx, open, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over "+
|
||||
"and setuid-search", args[0])
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over, "+
|
||||
"setuid-search and judge-directories", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -870,6 +876,27 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// What it is made of, in one block with its age (novox/hq issue 160): collected and shown in the same
|
||||
// breath, so no fact the machine says is kept and read by nothing.
|
||||
if account, had, err := inv.MachineOf(ctx, name); err != nil {
|
||||
fmt.Printf("\n what it is made of could NOT be read: %v\n", err)
|
||||
} else {
|
||||
fmt.Println()
|
||||
for _, line := range machineLines(account, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// And which of the networks it is attached to count as its local network, and why the others
|
||||
// do not (novox/hq ADR 0340 §8).
|
||||
outward, err := inv.OutwardLinksOf(ctx, name)
|
||||
if err != nil {
|
||||
fmt.Printf(" its local network could NOT be judged: %v\n", err)
|
||||
} else {
|
||||
for _, line := range localNetworkLines(account, had, judgedNetworks(account, had, outward, overlay.Interface)) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -913,6 +940,13 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// And which optional requirements of its modules nothing assigned answers (novox/hq ADR 0330 §7).
|
||||
if lines := unboundLines(plan); len(lines) > 0 {
|
||||
fmt.Println()
|
||||
for _, line := range lines {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -937,3 +971,57 @@ func printJSONTo(w io.Writer, v any) error {
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
const judgeDirectoriesUsage = "node judge-directories <node> — read-only: which of the directories <node> is " +
|
||||
"declared the node-engine's walk that follows no link another account could make would refuse, one line each " +
|
||||
"(novox/hq issue 358). Nothing is made, chmodded or chowned"
|
||||
|
||||
// nodeJudgeDirectories asks the node's engine to judge its declared directories, signed with the mesh's key as a
|
||||
// hand-over is. Read-only: the engine changes nothing and keeps nothing.
|
||||
func nodeJudgeDirectories(ctx context.Context, open *stores, args []string) error {
|
||||
known := func(node string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, node)
|
||||
return err
|
||||
}
|
||||
ask := func(node, by string) (link.HandOverAnswer, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||
node, err)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||
}
|
||||
defer js.Close()
|
||||
return link.AskJudgeDirectories(ctx, js.Conn(), ident, node, by, link.HandOverWithin)
|
||||
}
|
||||
return judgeDirectoriesAsked(args, known, ask, os.Stdout)
|
||||
}
|
||||
|
||||
// judgeDirectoriesAsked is the line with its two acts given: whether the mesh knows the node, and the ask. The
|
||||
// engine's refusal is this command's failure.
|
||||
func judgeDirectoriesAsked(args []string, known func(node string) error,
|
||||
ask func(node, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
||||
return errors.New(judgeDirectoriesUsage)
|
||||
}
|
||||
node := args[0]
|
||||
if err := known(node); err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
answer, err := ask(node, handOverBy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||
}
|
||||
fmt.Fprintf(out, "%s (read-only: nothing on %s was changed)\n", node, node)
|
||||
fmt.Fprintln(out, answer.Said)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq issue 237: every line of an act's answer comes from one resolution of the node taken after
|
||||
// its write, so the answer agrees with `plan` for the node right after.
|
||||
|
||||
func aProviderAndItsConsumer(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "rows", Scope: catalogue.ScopeNode}}})
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"rows"}})
|
||||
}
|
||||
|
||||
// twoRivalsAndANote is a node that does not resolve — two modules claim one node seat there — and a
|
||||
// module beside them that changes nothing about it.
|
||||
func twoRivalsAndANote(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
||||
}
|
||||
|
||||
func TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
twoRivalsAndANote(t, open)
|
||||
// Straight into the store, as a node that came to this before the act.
|
||||
for _, m := range []string{"rival-one", "rival-two", "notes"} {
|
||||
if _, err := open.inventory.Assign(ctx, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
said := answered(t, func() error { return assignCommand(ctx, "unassign", []string{"laptop", "notes"}) })
|
||||
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) {
|
||||
t.Fatalf("laptop resolves with two claimants of one seat: %v", err)
|
||||
}
|
||||
if n := strings.Count(said, "laptop does not resolve now"); n != 1 {
|
||||
t.Errorf("the answer says laptop does not resolve %d times, want once:\n%s", n, said)
|
||||
}
|
||||
if strings.Contains(said, "is left out of the rest of the mesh") {
|
||||
t.Errorf("the answer carries a line from a read that is not its resolution:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
aProviderAndItsConsumer(t, open)
|
||||
if _, err := assign(ctx, open, "laptop", "db", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := unassign(ctx, open, "laptop", "app")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "does not resolve") || strings.Contains(said, "could not be read") {
|
||||
t.Errorf("a node that resolves is said not to:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
twoRivalsAndANote(t, open)
|
||||
if _, err := assign(ctx, open, "laptop", "rival-one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Kept and not resolvable: a second claimant of the seat, which assignment keeps.
|
||||
said, err := assign(ctx, open, "laptop", "rival-two")
|
||||
if err == nil {
|
||||
t.Fatalf("two claimants of one node seat resolved:\n%s", said)
|
||||
}
|
||||
_, _, planErr := planFor(ctx, open, "laptop")
|
||||
if planErr == nil || planErr.Error() != err.Error() {
|
||||
t.Errorf("the act's refusal is not the plan's right after:\nact: %v\nplan: %v", err, planErr)
|
||||
}
|
||||
if strings.Contains(said, "now has") {
|
||||
t.Errorf("the answer says a seat is now held on a node that does not resolve:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The other readers of the whole mesh still say a machine they leave out, on standard error (novox/hq
|
||||
// issue 188); an act sends it elsewhere.
|
||||
func TestAMachineLeftOutIsSaidWhereTheContextSays(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
sayLeftOut(leftOutSaidBy(context.Background(), &buf), "laptop", errors.New("why"))
|
||||
if buf.String() != "laptop is left out of the rest of the mesh: it does not resolve: why\n" {
|
||||
t.Errorf("said %q", buf.String())
|
||||
}
|
||||
got := answered(t, func() error {
|
||||
sayLeftOut(context.Background(), "laptop", errors.New("why"))
|
||||
return nil
|
||||
})
|
||||
if !strings.Contains(got, "laptop is left out") {
|
||||
t.Errorf("without an act's writer the line did not reach standard error: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -44,6 +45,28 @@ func needsOperatorKey(module, node string) string {
|
||||
type operatorWaitFacts struct {
|
||||
manifests map[string]catalogue.Manifest
|
||||
given map[string]map[string]time.Time
|
||||
// outside is, per provision some module offers with a credential issued outside the mesh, that provider module
|
||||
// and the machines it is assigned to (novox/hq ADR 0337); pairsGiven, per "<module>@<machine>", the provisions
|
||||
// it holds a value a person gave for, sealed to the keys both ends have now. Absent is not known: no wait for a
|
||||
// provision of it is excused.
|
||||
outside map[string]outsideProvider
|
||||
pairsGiven map[string]map[string]bool
|
||||
}
|
||||
|
||||
// outsideProvider is the module offering a provision whose credential only a person gives, and where it runs.
|
||||
type outsideProvider struct {
|
||||
Module string
|
||||
Machines []string
|
||||
}
|
||||
|
||||
// acceptLine is the one line that gives an outside provision's credential for every consumer of its provider: on
|
||||
// the provider's machine when it runs on one, named as <machine> when on several or none.
|
||||
func (p outsideProvider) acceptLine(provision string) string {
|
||||
machine := "<its machine>"
|
||||
if len(p.Machines) == 1 {
|
||||
machine = p.Machines[0]
|
||||
}
|
||||
return inventory.AcceptLine(machine, p.Module, provision)
|
||||
}
|
||||
|
||||
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
||||
@@ -52,9 +75,18 @@ func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) er
|
||||
if !known {
|
||||
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
||||
}
|
||||
named := 0
|
||||
for _, n := range []string{w.Secret, w.Setting, w.Provision} {
|
||||
if n != "" {
|
||||
named++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case w.Secret != "" && w.Setting != "", w.Secret == "" && w.Setting == "":
|
||||
return fmt.Errorf("says it waits, naming %s, where a wait names one secret or one setting", waitNames(w))
|
||||
case named != 1:
|
||||
return fmt.Errorf("says it waits, naming %s, where a wait names one secret, one setting or one provision",
|
||||
waitNames(w))
|
||||
case w.Provision != "":
|
||||
return checkProvisionWait(module, machine, w, m, f)
|
||||
case w.Setting != "":
|
||||
if _, declared := m.Settings[w.Setting]; !declared {
|
||||
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
||||
@@ -80,20 +112,46 @@ func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) er
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkProvisionWait excuses a wait for a provision's credential only when the store bears it out (novox/hq ADR 0337
|
||||
// §6): the module requires the provision, a provider offers it with a credential issued outside the mesh, and the
|
||||
// module on this machine holds no value a person gave for it under the keys both ends have now.
|
||||
func checkProvisionWait(module, machine string, w inventory.Wait, m catalogue.Manifest, f operatorWaitFacts) error {
|
||||
if !slices.Contains(m.Requires, w.Provision) {
|
||||
return fmt.Errorf("says it waits for the provision %s, which %s does not require", w.Provision, module)
|
||||
}
|
||||
if _, outside := f.outside[w.Provision]; !outside {
|
||||
return fmt.Errorf("says it waits for the provision %s, whose credential the mesh gives itself: only a "+
|
||||
"credential issued outside the mesh waits for the operator", w.Provision)
|
||||
}
|
||||
given, readable := f.pairsGiven[module+"@"+machine]
|
||||
if !readable {
|
||||
return fmt.Errorf("says it waits for the provision %s, and what was given on %s could not be read", w.Provision, machine)
|
||||
}
|
||||
if given[w.Provision] {
|
||||
return fmt.Errorf("says it waits for the provision %s, whose credential was given", w.Provision)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
||||
const waitRefusedPrefix = "says it waits"
|
||||
|
||||
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
||||
func waitNames(w inventory.Wait) string {
|
||||
switch {
|
||||
case w.Secret != "" && w.Setting != "":
|
||||
return "the secret " + w.Secret + " and the setting " + w.Setting
|
||||
case w.Secret != "":
|
||||
return "the secret " + w.Secret
|
||||
case w.Setting != "":
|
||||
return "the setting " + w.Setting
|
||||
var names []string
|
||||
if w.Secret != "" {
|
||||
names = append(names, "the secret "+w.Secret)
|
||||
}
|
||||
return "nothing"
|
||||
if w.Setting != "" {
|
||||
names = append(names, "the setting "+w.Setting)
|
||||
}
|
||||
if w.Provision != "" {
|
||||
names = append(names, "the provision "+w.Provision)
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return "nothing"
|
||||
}
|
||||
return strings.Join(names, " and ")
|
||||
}
|
||||
|
||||
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
||||
@@ -107,11 +165,17 @@ func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitF
|
||||
if len(r.Waits) == 0 {
|
||||
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
||||
}
|
||||
waits := make([]inventory.Wait, 0, len(r.Waits))
|
||||
for _, w := range r.Waits {
|
||||
if why == nil {
|
||||
why = checkWait(r.Module, machine, w, f)
|
||||
}
|
||||
if p, outside := f.outside[w.Provision]; outside && w.Provision != "" {
|
||||
w.Act = p.acceptLine(w.Provision)
|
||||
}
|
||||
waits = append(waits, w)
|
||||
}
|
||||
r.Waits = waits
|
||||
if why != nil {
|
||||
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
||||
}
|
||||
@@ -150,6 +214,10 @@ func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
||||
if w.Secret != "" {
|
||||
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
||||
}
|
||||
if w.Provision != "" && w.Act != "" {
|
||||
// One line for the provider, which seals it to every consumer of it (ADR 0337 §3, §7).
|
||||
part += fmt.Sprintf(", given for every consumer with `%s`", w.Act)
|
||||
}
|
||||
parts = append(parts, part+")")
|
||||
}
|
||||
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
||||
@@ -172,7 +240,7 @@ func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []
|
||||
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
||||
var acts []string
|
||||
seen := map[string]bool{}
|
||||
secret := false
|
||||
secret, provision := false, false
|
||||
for _, w := range waits {
|
||||
var act string
|
||||
switch {
|
||||
@@ -180,6 +248,9 @@ func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
||||
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
||||
case w.Setting != "":
|
||||
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
||||
case w.Provision != "":
|
||||
// ADR 0337 §7: "give <what> of <provider> on <machine>"; the line is in the summary.
|
||||
act, provision = "give "+w.What, true
|
||||
}
|
||||
if act != "" && !seen[act] {
|
||||
seen[act] = true
|
||||
@@ -197,6 +268,10 @@ func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
||||
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
||||
"is sealed to the machine."
|
||||
}
|
||||
if provision {
|
||||
explanation += " A key only you can give is given once for the app that issued it, and reaches every " +
|
||||
"module that uses it."
|
||||
}
|
||||
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
||||
return words{
|
||||
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
||||
@@ -217,6 +292,12 @@ func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string
|
||||
if f.given == nil {
|
||||
f.given = map[string]map[string]time.Time{}
|
||||
}
|
||||
if f.outside == nil {
|
||||
f.outside = map[string]outsideProvider{}
|
||||
}
|
||||
if f.pairsGiven == nil {
|
||||
f.pairsGiven = map[string]map[string]bool{}
|
||||
}
|
||||
if inv == nil {
|
||||
return
|
||||
}
|
||||
@@ -238,15 +319,59 @@ func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, read := f.given[module+"@"+machine]; read {
|
||||
if _, read := f.given[module+"@"+machine]; !read {
|
||||
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
||||
f.given[module+"@"+machine] = given
|
||||
}
|
||||
}
|
||||
if _, read := f.pairsGiven[module+"@"+machine]; !read {
|
||||
if given, err := inv.GivenPairsOf(ctx, machine, module); err == nil {
|
||||
f.pairsGiven[module+"@"+machine] = given
|
||||
}
|
||||
}
|
||||
// The provisions it requires whose credential only a person gives, and where their provider runs (ADR 0337).
|
||||
m, known := f.manifests[module]
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
||||
f.given[module+"@"+machine] = given
|
||||
for _, provision := range m.Requires {
|
||||
if _, read := f.outside[provision]; read {
|
||||
continue
|
||||
}
|
||||
if shelf == nil {
|
||||
var err error
|
||||
if shelf, err = inv.Catalogue(ctx); err != nil {
|
||||
shelf = map[string]catalogue.Manifest{}
|
||||
}
|
||||
}
|
||||
provider, outside := catalogue.OffersIssuedOutside(shelf, provision)
|
||||
if !outside {
|
||||
continue
|
||||
}
|
||||
f.outside[provision] = outsideProvider{Module: provider, Machines: machinesRunning(ctx, inv, provider)}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// machinesRunning is every machine a module is assigned to, by name; a read that fails is left out.
|
||||
func machinesRunning(ctx context.Context, inv *inventory.Inventory, module string) []string {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
// Only the summary's accept line reads it, which then names the provider's machine "<its machine>".
|
||||
// empty-on-error: no wait is excused or refused by it
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err == nil && slices.Contains(assigned, module) {
|
||||
out = append(out, n.Name)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// waitingModules is every module with a waiting resource in a statement.
|
||||
func waitingModules(rs []inventory.ResourceHealth) []string {
|
||||
seen := map[string]bool{}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user