Compare commits
183
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9c38aed789 | ||
|
|
a3f1d0460d | ||
|
|
0af12b369b | ||
|
|
e338b1f0f4 | ||
|
|
e42818a0ab | ||
|
|
bdb8193456 | ||
|
|
ca25e4ab45 | ||
|
|
7a64075702 | ||
|
|
9abe0b59fa | ||
|
|
183d7a99ff | ||
|
|
fad3d7ee71 | ||
|
|
3c50e80972 | ||
|
|
6e4bca7fce | ||
|
|
f4998a7266 | ||
|
|
53d9c4de19 | ||
|
|
9372733861 | ||
|
|
d640ef15ec | ||
|
|
28e1b29e0c | ||
|
|
f0815c4f18 | ||
|
|
72bbdeed35 | ||
|
|
dfd81d8daf | ||
|
|
8c54139905 | ||
|
|
6ed5de186d | ||
|
|
8fab4d8b22 | ||
|
|
b879128efb | ||
|
|
9641b88725 | ||
|
|
36704abc08 | ||
|
|
7fccba4658 | ||
|
|
98e948e30e | ||
|
|
f18bac4533 | ||
|
|
e05481e992 | ||
|
|
dd58fbcb2c | ||
|
|
1ed5afe679 | ||
|
|
38523c6509 | ||
|
|
a78f244621 | ||
|
|
4ab4529440 | ||
|
|
370673195c | ||
|
|
3ad12f8ec7 | ||
|
|
f6e78c837f | ||
|
|
35e330c7f0 | ||
|
|
aa7e53112b | ||
|
|
af6bcea41e | ||
|
|
49fc8b6d7d | ||
|
|
f20d221a60 | ||
|
|
bd8ea8ad4f | ||
|
|
1350c428ed | ||
|
|
76c48fa074 | ||
|
|
df6aab6f44 | ||
|
|
7f8e5a821c | ||
|
|
3db472ce56 | ||
|
|
3ae2d6e17c | ||
|
|
29ef458d89 | ||
|
|
9d0798a1c0 | ||
|
|
0e85230769 | ||
|
|
ad0aabc2a9 | ||
|
|
fe3faaef83 | ||
|
|
34b105ea88 | ||
|
|
8f90d3ecd6 | ||
|
|
b4a62e9e6e | ||
|
|
03894e5e51 | ||
|
|
565de8c46a | ||
|
|
c19d4516ef | ||
|
|
a8d63653fb | ||
|
|
aaadb5695c | ||
|
|
73a31eb205 | ||
|
|
f56458da32 | ||
|
|
66650e315c | ||
|
|
c40e15268e | ||
|
|
0f8c47842a | ||
|
|
aee0a2718f | ||
|
|
4f95be7e08 | ||
|
|
788fb681ec | ||
|
|
96d7784bce | ||
|
|
dc456d5660 | ||
|
|
44d9d40582 | ||
|
|
74bed5a800 | ||
|
|
ecbb317dc3 | ||
|
|
dd2d8b91be | ||
|
|
1b0fd8b442 | ||
|
|
c06f75d083 | ||
|
|
5f2da0af69 | ||
|
|
16ea033124 | ||
|
|
9088db9a04 | ||
|
|
d0f5ae26de | ||
|
|
e8b66a2eb3 | ||
|
|
f2da7d0540 | ||
|
|
5b03b2e26b | ||
|
|
a5b874a2a0 | ||
|
|
491cbe2548 | ||
|
|
b2bf14ab01 | ||
|
|
9db1459cb0 | ||
|
|
3cfcf5e3e4 | ||
|
|
f99e499288 | ||
|
|
3c9a4d56c0 | ||
|
|
e57224c25a | ||
|
|
bb2073b6bb | ||
|
|
9131ef0dbc | ||
|
|
8831c74f70 | ||
|
|
fecfb40ed4 | ||
|
|
a1969403c6 | ||
|
|
893aa85349 | ||
|
|
2ba9eef58d | ||
|
|
3b51d3f86e | ||
|
|
5ee08cb5e1 | ||
|
|
443ba6b265 | ||
|
|
a96bd2ff38 | ||
|
|
64fbef7713 | ||
|
|
a77aa8520e | ||
|
|
a9de578cb9 | ||
|
|
d951aa954e | ||
|
|
ab9a7771f4 | ||
|
|
1c50608246 | ||
|
|
760351b4ae | ||
|
|
899389c710 | ||
|
|
a203f605d4 | ||
|
|
0741880d31 | ||
|
|
fa734d3098 | ||
|
|
f13d42abb4 | ||
|
|
fdf26a1fe2 | ||
|
|
343e271ec7 | ||
|
|
05c872485a | ||
|
|
12dd2bb3f0 | ||
|
|
5831eec770 | ||
|
|
c9a7102bc5 | ||
|
|
67279199e9 | ||
|
|
9d2a9e62d1 | ||
|
|
b345d10bec | ||
|
|
13842a7234 | ||
|
|
4238318601 | ||
|
|
758c8e84e5 | ||
|
|
9222575ae9 | ||
|
|
c5b8fca8bd | ||
|
|
ff543d3def | ||
|
|
ef211faefb | ||
|
|
3c247e80af | ||
|
|
bb0d5e17db | ||
|
|
24ad859629 | ||
|
|
5ccc87f4d0 | ||
|
|
68c1220e87 | ||
|
|
5b5cd6160a | ||
|
|
d7d70cd539 | ||
|
|
df70ca43a7 | ||
|
|
f60c742d93 | ||
|
|
2db5dcdcb4 | ||
|
|
d9cdb3615d | ||
|
|
dec3b83e66 | ||
|
|
2e36a63096 | ||
|
|
eb8286fc68 | ||
|
|
5540fd4f85 | ||
|
|
e419d0182f | ||
|
|
5ca4f14479 | ||
|
|
c43883910d | ||
|
|
548681cc4e | ||
|
|
9c4bf1f08b | ||
|
|
6300a5b31c | ||
|
|
505694c424 | ||
|
|
72186815e3 | ||
|
|
526eed99fa | ||
|
|
b506852388 | ||
|
|
791786d5b8 | ||
|
|
0b312218f9 | ||
|
|
c834e4f821 | ||
|
|
149f89e8f9 | ||
|
|
b2af3d654f | ||
|
|
7676e67c5e | ||
|
|
0173d9c70e | ||
|
|
89abce5df9 | ||
|
|
185d0294eb | ||
|
|
4d4d51eb01 | ||
|
|
b7d996878b | ||
|
|
219fdcd7f0 | ||
|
|
55ba7f2cb8 | ||
|
|
102f8dc267 | ||
|
|
296e11d499 | ||
|
|
eed4976483 | ||
|
|
bbadbb2af0 | ||
|
|
1414ed7f7b | ||
|
|
f67342f0d8 | ||
|
|
fae7710bed | ||
|
|
289bbf68e4 | ||
|
|
f2a4363a1a | ||
|
|
fd4d26b42c | ||
|
|
acaf8830ec |
+3
-1
@@ -2,9 +2,11 @@
|
|||||||
|
|
||||||
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
||||||
# built artifacts belong in build/, which is already ignored.
|
# built artifacts belong in build/, which is already ignored.
|
||||||
/mesh-builder
|
|
||||||
/mesh-controller
|
/mesh-controller
|
||||||
/postgres-provisioner
|
/postgres-provisioner
|
||||||
/redis-provisioner
|
/redis-provisioner
|
||||||
# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
|
# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
|
||||||
/cmd/mesh-controller/mesh-controller
|
/cmd/mesh-controller/mesh-controller
|
||||||
|
|
||||||
|
# No repository commits its dependencies (issue 1305).
|
||||||
|
vendor/
|
||||||
|
|||||||
+3
-5
@@ -21,17 +21,15 @@ ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7
|
|||||||
FROM ${GO_BASE} AS build
|
FROM ${GO_BASE} AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
|
# **No dependency is committed** (issue 1305): go.sum names each one's version and checksum, and the build
|
||||||
# the image builds from this repository alone — the host's validator among them, whose module no
|
# resolves them with -mod=mod. Dependencies first, so a change to the source does not re-copy them.
|
||||||
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
|
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
COPY vendor/ vendor/
|
|
||||||
|
|
||||||
# `COPY . .` brings `.git` along, so Go is told not to stamp the commit: the version is said by -X, and a
|
# `COPY . .` brings `.git` along, so Go is told not to stamp the commit: the version is said by -X, and a
|
||||||
# rebuild of an unchanged program is the same bytes (novox/hq ADR 0267 rule 6, issue 363).
|
# rebuild of an unchanged program is the same bytes (novox/hq ADR 0267 rule 6, issue 363).
|
||||||
COPY . .
|
COPY . .
|
||||||
ARG VERSION=development
|
ARG VERSION=development
|
||||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath -buildvcs=false \
|
RUN CGO_ENABLED=0 GOFLAGS=-mod=mod go build -trimpath -buildvcs=false \
|
||||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||||
-o /mesh-controller ./cmd/mesh-controller
|
-o /mesh-controller ./cmd/mesh-controller
|
||||||
|
|
||||||
|
|||||||
@@ -46,17 +46,6 @@ image:
|
|||||||
@echo
|
@echo
|
||||||
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
# The builder ships as an image too, because it is a module the mesh assigns rather than a program
|
|
||||||
# somebody starts on a machine by hand.
|
|
||||||
BUILDER_IMAGE ?= mesh-builder:$(VERSION)
|
|
||||||
BUILDER_DEV_TAG ?= mesh-builder:development
|
|
||||||
|
|
||||||
builder-image:
|
|
||||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
|
||||||
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
|
||||||
@echo
|
|
||||||
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
|
||||||
|
|
||||||
# The provisioner ships as an image too, because it is the thing that makes a sealed credential
|
# The provisioner ships as an image too, because it is the thing that makes a sealed credential
|
||||||
# true on a machine -- and the mesh cannot, having discarded the plaintext.
|
# true on a machine -- and the mesh cannot, having discarded the plaintext.
|
||||||
PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
|
PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
|
||||||
@@ -90,26 +79,6 @@ redis-provisioner-image:
|
|||||||
@echo
|
@echo
|
||||||
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
# The whole check. Raises a database, runs the repository's own check against it -- merge-check.sh,
|
|
||||||
# the very script `mesh/repo-check` runs -- and takes the database down again, including when the
|
|
||||||
# check fails, which is why the teardown is not conditional and the script's exit status is the
|
|
||||||
# target's.
|
|
||||||
#
|
|
||||||
# **It calls the script rather than restating it** (novox/hq issue 431): this target used to list its
|
|
||||||
# own steps, its formatting step walked vendor/ where the script's does not, and it failed on
|
|
||||||
# third-party code no change could fix -- so a developer's check and the gate disagreed, and the steps
|
|
||||||
# after formatting never ran through it. The script is the one list of steps; this target only gives it
|
|
||||||
# a database (MESH_TEST_POSTGRES, exported above).
|
|
||||||
#
|
|
||||||
# **Two levels** (issue 895): the unit level, `go test ./...` untagged, timed and held to a bound; and the integration
|
|
||||||
# level, `-tags integration`, the tests that start a bus, a store or another process. Each is said with its time.
|
|
||||||
#
|
|
||||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
|
|
||||||
# as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
|
|
||||||
# order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
|
|
||||||
check: postgres
|
|
||||||
@sh merge-check.sh ; status=$$? ; docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
|
|
||||||
|
|
||||||
# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
|
# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
|
||||||
# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
|
# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
|
||||||
# store's tests would skip rather than fail anyway.
|
# store's tests would skip rather than fail anyway.
|
||||||
@@ -119,8 +88,7 @@ test:
|
|||||||
vet:
|
vet:
|
||||||
go vet ./...
|
go vet ./...
|
||||||
|
|
||||||
# Quick steps for a developer, not part of `check`. The directories gofmt reads must be the ones
|
# A quick step for a developer: the checker runs the build module's own formatting and vet (ADR 0501 §5, D7).
|
||||||
# merge-check.sh lists, never `.`, which walks vendor/ (novox/hq issue 431).
|
|
||||||
fmt:
|
fmt:
|
||||||
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
@unformatted=$$(gofmt -l cmd internal examples) ; \
|
||||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||||
|
|||||||
@@ -1,6 +0,0 @@
|
|||||||
# Release 57: config and provisioning
|
|
||||||
|
|
||||||
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
|
|
||||||
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
|
|
||||||
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
|
|
||||||
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# The unit tests of every Go module of its own below this repository's root, run from the root, for merge-check.sh
|
|
||||||
# and for the test that tries it on a tree (merge_check_nested_test.go). Its own file so that the test needs no
|
|
||||||
# particular merge-check.sh: the check that judges a change to merge-check.sh is main's.
|
|
||||||
set -eu
|
|
||||||
cgo="${cgo:-0}"
|
|
||||||
race="${race:-}"
|
|
||||||
|
|
||||||
# nested_modules vets and tests every Go module of its own below the root, `modules/<name>/go.mod` (novox/hq ADR 0420
|
|
||||||
# §5, issue 1025): `./...` stops at a nested go.mod, so a module that lives whole in this repository would have its
|
|
||||||
# unit tests run by nobody. None is a pass. Each runs in its own directory, outside the vendor directory this
|
|
||||||
# repository's own packages are built from; a nested module vendors what the forge holds (its own `vendor/`, so the
|
|
||||||
# check runs offline), as this repository's root module does (issue 1019). A failing module fails the check; the others are still run.
|
|
||||||
nested_modules() {
|
|
||||||
nested=0
|
|
||||||
for gomod in modules/*/go.mod; do
|
|
||||||
[ -e "$gomod" ] || continue
|
|
||||||
dir=$(dirname "$gomod")
|
|
||||||
echo "nested module $dir:"
|
|
||||||
if ! (cd "$dir" && GOFLAGS= CGO_ENABLED=0 go vet ./... &&
|
|
||||||
GOFLAGS= CGO_ENABLED="${cgo:-0}" go test ${race:-} -count=1 -timeout 30m ./...); then
|
|
||||||
echo "nested module $dir FAILED"
|
|
||||||
nested=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
return $nested
|
|
||||||
}
|
|
||||||
|
|
||||||
nested_modules
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
ARG ALPINE_BASE=alpine:3
|
|
||||||
ARG GO_BASE=golang:1.25-alpine
|
|
||||||
# The builder, as a module ships one.
|
|
||||||
#
|
|
||||||
# Not FROM scratch, unlike the control plane: this one runs git and a container client, so it
|
|
||||||
# needs a filesystem with them on it. That is the honest cost of a machine whose job is to build —
|
|
||||||
# and it is why building is a MODULE on a machine that has a runtime rather than something the
|
|
||||||
# control plane does (novox/hq ADR 0005).
|
|
||||||
FROM ${GO_BASE} AS build
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY . .
|
|
||||||
RUN CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags '-s -w' -o /mesh-builder ./cmd/mesh-builder
|
|
||||||
|
|
||||||
FROM ${ALPINE_BASE}
|
|
||||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
|
||||||
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
|
||||||
# rather than running one inside itself.
|
|
||||||
RUN apk add --no-cache git docker-cli
|
|
||||||
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
|
||||||
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
|
||||||
@@ -1,235 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
|
||||||
// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
|
|
||||||
// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
|
|
||||||
// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
|
|
||||||
// it runs, instead of the build queue handing it out again from the start (issue 594).
|
|
||||||
//
|
|
||||||
// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
|
|
||||||
// without them, and a restarted one removes them before it takes work again.
|
|
||||||
//
|
|
||||||
// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
|
|
||||||
// when the build ends.
|
|
||||||
// - drained: written by the ask, before it reads whether a build is in hand.
|
|
||||||
//
|
|
||||||
// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
|
|
||||||
// or the agent sees the drain and hands the request back; never neither.
|
|
||||||
|
|
||||||
func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
|
|
||||||
func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
|
|
||||||
|
|
||||||
// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
|
|
||||||
// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
|
|
||||||
// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
|
|
||||||
// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
|
|
||||||
// came, which is a fault to see.
|
|
||||||
const (
|
|
||||||
drainLapse = 2 * time.Minute
|
|
||||||
drainTooLong = 15*time.Minute + 5*time.Minute
|
|
||||||
)
|
|
||||||
|
|
||||||
// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
|
|
||||||
func drainedAt() (began, asked time.Time, ok bool) {
|
|
||||||
info, err := os.Stat(drainMark())
|
|
||||||
if err != nil {
|
|
||||||
return time.Time{}, time.Time{}, false
|
|
||||||
}
|
|
||||||
asked = info.ModTime()
|
|
||||||
began = asked
|
|
||||||
if raw, err := os.ReadFile(drainMark()); err == nil {
|
|
||||||
if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
|
|
||||||
began = t
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return began, asked, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
|
|
||||||
func drained() bool { return drainedSince(time.Now()) }
|
|
||||||
|
|
||||||
func drainedSince(now time.Time) bool {
|
|
||||||
_, asked, ok := drainedAt()
|
|
||||||
return ok && now.Sub(asked) < drainLapse
|
|
||||||
}
|
|
||||||
|
|
||||||
// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
|
|
||||||
// longer than its node-engine's bound and the slack.
|
|
||||||
func drainedTooLong(now time.Time) error {
|
|
||||||
began, asked, ok := drainedAt()
|
|
||||||
if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
|
|
||||||
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
|
|
||||||
}
|
|
||||||
|
|
||||||
// inHand is true while this process has claimed a build and not yet released it. The mark in the temporary directory
|
|
||||||
// is only believed while it is: a mark with no build of this process behind it is stale (a crash, a fault in an
|
|
||||||
// earlier path) and is removed, because left alone it hands back every ask for ever (issue 1768).
|
|
||||||
var inHand atomic.Bool
|
|
||||||
|
|
||||||
// markRetry is how long an agent that cannot mark a build in hand takes no work, and markFault when it last failed to.
|
|
||||||
// Taking nothing is the point: an ask fetched and handed back at once, many times a second, spends all its deliveries.
|
|
||||||
const markRetry = 30 * time.Second
|
|
||||||
|
|
||||||
var (
|
|
||||||
markMu sync.Mutex
|
|
||||||
markFault time.Time
|
|
||||||
markSaid string
|
|
||||||
)
|
|
||||||
|
|
||||||
// markFaulty says the agent could not mark a build in hand and has not yet been able to since: it takes no work. Once
|
|
||||||
// markRetry has passed it tries the mark itself, with no ask in hand, so that an ask is fetched only when marking works.
|
|
||||||
func markFaulty(now time.Time) bool {
|
|
||||||
markMu.Lock()
|
|
||||||
failed := markFault
|
|
||||||
markMu.Unlock()
|
|
||||||
if failed.IsZero() {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if now.Sub(failed) < markRetry {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
|
||||||
if err != nil && errors.Is(err, os.ErrExist) && !inHand.Load() {
|
|
||||||
_ = os.Remove(inHandMark())
|
|
||||||
f, err = os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
markFailed(err)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
_ = f.Close()
|
|
||||||
_ = os.Remove(inHandMark())
|
|
||||||
markMu.Lock()
|
|
||||||
markFault, markSaid = time.Time{}, ""
|
|
||||||
markMu.Unlock()
|
|
||||||
fmt.Fprintf(os.Stderr, "the mark of a build in hand can be made again (%s): taking builds\n", inHandMark())
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// markFailed notes that a build could not be marked in hand, and says it when it is a new reason.
|
|
||||||
func markFailed(err error) {
|
|
||||||
markMu.Lock()
|
|
||||||
defer markMu.Unlock()
|
|
||||||
markFault = time.Now()
|
|
||||||
if said := err.Error(); said != markSaid {
|
|
||||||
markSaid = said
|
|
||||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so this agent takes no build for %s and tries again: %v\n",
|
|
||||||
inHandMark(), markRetry, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
|
||||||
// back to the queue.
|
|
||||||
func claim() bool {
|
|
||||||
if !inHand.Load() {
|
|
||||||
// Nothing of this process is in hand, so a mark already there is stale.
|
|
||||||
if err := os.Remove(inHandMark()); err == nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "removed the mark of a build in hand (%s): no build of this agent is in hand\n", inHandMark())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
|
||||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
|
||||||
if err != nil {
|
|
||||||
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
|
||||||
markFailed(err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
|
||||||
if closeErr := f.Close(); err == nil {
|
|
||||||
err = closeErr
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
_ = os.Remove(inHandMark())
|
|
||||||
markFailed(err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if drained() {
|
|
||||||
_ = os.Remove(inHandMark())
|
|
||||||
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
inHand.Store(true)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// holding says what is in hand, for the ask to print: written once the request is read.
|
|
||||||
func holding(what string) {
|
|
||||||
_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
|
|
||||||
}
|
|
||||||
|
|
||||||
// released ends the build in hand.
|
|
||||||
func released() {
|
|
||||||
inHand.Store(false)
|
|
||||||
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// undrained clears both marks at the start of a holder: a container started again is asked afresh.
|
|
||||||
func undrained() error {
|
|
||||||
for _, mark := range []string{drainMark(), inHandMark()} {
|
|
||||||
if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fmt.Errorf("cannot remove %s: %w", mark, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
|
|
||||||
func busy() (string, error) {
|
|
||||||
// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
|
|
||||||
// ask: its time is the last ask.
|
|
||||||
began := time.Now().UTC()
|
|
||||||
if was, _, ok := drainedAt(); ok && drained() {
|
|
||||||
began = was.UTC()
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
|
||||||
// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
|
|
||||||
fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
|
|
||||||
}
|
|
||||||
held, err := os.ReadFile(inHandMark())
|
|
||||||
if err != nil {
|
|
||||||
return "", errors.New("idle: no build in hand, and this agent takes no new one")
|
|
||||||
}
|
|
||||||
since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
|
|
||||||
if what == "" {
|
|
||||||
what = "a build"
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
roomMu sync.Mutex
|
|
||||||
roomSaid bool
|
|
||||||
)
|
|
||||||
|
|
||||||
// noRoom says this node is under the floor of free space (novox/hq issue 1834): it takes no work, so another build node
|
|
||||||
// takes the ask instead of this one taking it and ending it in an error. Said once when it starts and again when it ends.
|
|
||||||
func noRoom(workspace string) bool {
|
|
||||||
err := builder.RoomToWork("/", workspace)
|
|
||||||
roomMu.Lock()
|
|
||||||
defer roomMu.Unlock()
|
|
||||||
if err != nil && !roomSaid {
|
|
||||||
fmt.Fprintf(os.Stderr, "taking no builds: %v\n", err)
|
|
||||||
} else if err == nil && roomSaid {
|
|
||||||
fmt.Fprintf(os.Stderr, "there is room again: taking builds\n")
|
|
||||||
}
|
|
||||||
roomSaid = err != nil
|
|
||||||
return err != nil
|
|
||||||
}
|
|
||||||
@@ -1,239 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
|
|
||||||
// agent takes no new build request (novox/hq ADR 0335 §5).
|
|
||||||
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
|
|
||||||
t.Setenv("TMPDIR", t.TempDir())
|
|
||||||
if err := undrained(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if drained() {
|
|
||||||
t.Fatal("an agent never asked is drained")
|
|
||||||
}
|
|
||||||
// A build is claimed and runs.
|
|
||||||
if !claim() {
|
|
||||||
t.Fatal("an agent never asked refused a build")
|
|
||||||
}
|
|
||||||
holding("novox/mesh-controller (check-280)")
|
|
||||||
said, err := busy()
|
|
||||||
if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
|
|
||||||
t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
|
|
||||||
}
|
|
||||||
// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
|
|
||||||
if !drained() {
|
|
||||||
t.Fatal("the ask did not drain the agent")
|
|
||||||
}
|
|
||||||
if claim() {
|
|
||||||
t.Fatal("a drained agent took a new build")
|
|
||||||
}
|
|
||||||
// The build in hand is still in hand after the refused claim, and busy goes on saying so.
|
|
||||||
if _, err := busy(); err != nil {
|
|
||||||
t.Fatalf("a refused claim ended the build in hand: %v", err)
|
|
||||||
}
|
|
||||||
// It ends: idle, and still drained.
|
|
||||||
released()
|
|
||||||
if _, err := busy(); err == nil {
|
|
||||||
t.Fatal("an agent with nothing in hand answered busy")
|
|
||||||
}
|
|
||||||
if claim() {
|
|
||||||
t.Fatal("a drained agent took a build after its last one ended")
|
|
||||||
}
|
|
||||||
// Started again, it is asked afresh.
|
|
||||||
if err := undrained(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if drained() || !claim() {
|
|
||||||
t.Fatal("a holder started again is still drained")
|
|
||||||
}
|
|
||||||
released()
|
|
||||||
}
|
|
||||||
|
|
||||||
// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
|
|
||||||
// neither, which would replace the container under a build just taken.
|
|
||||||
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
|
|
||||||
t.Setenv("TMPDIR", t.TempDir())
|
|
||||||
for i := 0; i < 200; i++ {
|
|
||||||
if err := undrained(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var claimed bool
|
|
||||||
var askErr error
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
wg.Add(2)
|
|
||||||
go func() { defer wg.Done(); claimed = claim() }()
|
|
||||||
go func() { defer wg.Done(); _, askErr = busy() }()
|
|
||||||
wg.Wait()
|
|
||||||
if claimed && askErr != nil {
|
|
||||||
t.Fatalf("round %d: the agent took a build and the ask said idle", i)
|
|
||||||
}
|
|
||||||
released()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
|
|
||||||
// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
|
|
||||||
func TestADrainNoLongerAskedLapses(t *testing.T) {
|
|
||||||
t.Setenv("TMPDIR", t.TempDir())
|
|
||||||
if err := undrained(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := busy(); err == nil {
|
|
||||||
t.Fatal("an idle agent answered busy")
|
|
||||||
}
|
|
||||||
if !drained() || claim() {
|
|
||||||
t.Fatal("an agent just asked takes work")
|
|
||||||
}
|
|
||||||
// The last ask was three minutes ago.
|
|
||||||
old := time.Now().Add(-3 * time.Minute)
|
|
||||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if drained() {
|
|
||||||
t.Fatal("a drain no longer asked for three minutes still holds")
|
|
||||||
}
|
|
||||||
if !claim() {
|
|
||||||
t.Fatal("an agent whose drain lapsed takes no work")
|
|
||||||
}
|
|
||||||
released()
|
|
||||||
}
|
|
||||||
|
|
||||||
// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
|
|
||||||
// replacement it was drained for never came. One asked again after a lapse starts a new drain.
|
|
||||||
func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
|
|
||||||
t.Setenv("TMPDIR", t.TempDir())
|
|
||||||
if err := undrained(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
taking("mesh-build-machine")
|
|
||||||
if _, err := busy(); err == nil {
|
|
||||||
t.Fatal("an idle agent answered busy")
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
if err := drainedTooLong(now); err != nil {
|
|
||||||
t.Fatalf("an agent drained a moment ago is not ready: %v", err)
|
|
||||||
}
|
|
||||||
// The drain began 25 minutes ago, and the node-engine asked a moment ago.
|
|
||||||
began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
|
|
||||||
if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
err := drainedTooLong(time.Now())
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "has not come") {
|
|
||||||
t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
|
|
||||||
}
|
|
||||||
// Each ask keeps when the drain began.
|
|
||||||
if _, err := busy(); err == nil {
|
|
||||||
t.Fatal("an idle agent answered busy")
|
|
||||||
}
|
|
||||||
if err := drainedTooLong(time.Now()); err == nil {
|
|
||||||
t.Fatal("an ask started the drain again")
|
|
||||||
}
|
|
||||||
// The asks stopped: the drain lapsed, and so did the fault.
|
|
||||||
old := time.Now().Add(-3 * time.Minute)
|
|
||||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := drainedTooLong(time.Now()); err != nil {
|
|
||||||
t.Fatalf("a lapsed drain still reads not ready: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
|
|
||||||
// node-engine's whole bound.
|
|
||||||
func TestAnUnknownCommandFails(t *testing.T) {
|
|
||||||
was := os.Args
|
|
||||||
t.Cleanup(func() { os.Args = was })
|
|
||||||
os.Args = []string{"mesh-builder", "some-later-word"}
|
|
||||||
if err := run(); err == nil {
|
|
||||||
t.Fatal("an unknown command exited 0")
|
|
||||||
}
|
|
||||||
os.Args = []string{"mesh-builder", "help"}
|
|
||||||
if err := run(); err != nil {
|
|
||||||
t.Fatalf("help failed: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1768/AC2: a mark naming no build of this agent (left by a crash or a fault) is stale: the next ask removes it and is
|
|
||||||
// run, never fetched and handed back in a loop.
|
|
||||||
func TestAStaleInHandMarkIsRemovedAndTheAskRuns(t *testing.T) {
|
|
||||||
t.Setenv("TMPDIR", t.TempDir())
|
|
||||||
inHand.Store(false)
|
|
||||||
if err := undrained(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(inHandMark(), []byte("2026-10-21T18:45:06Z\n"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !claim() {
|
|
||||||
t.Fatal("a stale in-hand mark made the agent hand back an ask")
|
|
||||||
}
|
|
||||||
// A live one — this agent's own build in hand — is never taken over.
|
|
||||||
if claim() {
|
|
||||||
t.Fatal("a second build was taken while one is in hand")
|
|
||||||
}
|
|
||||||
if _, err := busy(); err != nil {
|
|
||||||
t.Fatalf("the build in hand was lost: %v", err)
|
|
||||||
}
|
|
||||||
released()
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1768/AC3: an agent that cannot mark a build in hand for another reason takes no work, instead of fetching asks and
|
|
||||||
// handing them back; it takes work again once marking works.
|
|
||||||
func TestAnAgentThatCannotMarkABuildTakesNoWorkUntilItCan(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
t.Setenv("TMPDIR", dir)
|
|
||||||
inHand.Store(false)
|
|
||||||
markFault, markSaid = time.Time{}, ""
|
|
||||||
t.Cleanup(func() { markFault, markSaid = time.Time{}, "" })
|
|
||||||
if err := undrained(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// The mark's place is a directory that cannot be removed or made over: a directory named as the mark.
|
|
||||||
if err := os.MkdirAll(inHandMark()+"/x", 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if claim() {
|
|
||||||
t.Fatal("an agent that cannot mark a build took it")
|
|
||||||
}
|
|
||||||
if !markFaulty(time.Now()) {
|
|
||||||
t.Fatal("an agent that cannot mark a build still takes work")
|
|
||||||
}
|
|
||||||
// Still failing after the retry time: still takes nothing, and no ask was needed to find out.
|
|
||||||
if !markFaulty(time.Now().Add(2 * markRetry)) {
|
|
||||||
t.Fatal("an agent whose mark still cannot be made takes work")
|
|
||||||
}
|
|
||||||
if err := os.RemoveAll(inHandMark()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if markFaulty(time.Now().Add(4 * markRetry)) {
|
|
||||||
t.Fatal("an agent whose mark can be made again takes no work")
|
|
||||||
}
|
|
||||||
if !claim() {
|
|
||||||
t.Fatal("an agent that can mark takes no build")
|
|
||||||
}
|
|
||||||
released()
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1834/AC4: given a node under the floor of free space, when the taking loop asks, then it takes nothing, and takes again
|
|
||||||
// once there is room.
|
|
||||||
func TestANodeUnderTheFloorOfFreeSpaceTakesNoWork(t *testing.T) {
|
|
||||||
was := builder.FreeBytes
|
|
||||||
t.Cleanup(func() { builder.FreeBytes = was })
|
|
||||||
builder.FreeBytes = func(string) (uint64, error) { return 2 << 30, nil }
|
|
||||||
if !noRoom(t.TempDir()) {
|
|
||||||
t.Error("a node with 2 GiB free took work")
|
|
||||||
}
|
|
||||||
builder.FreeBytes = func(string) (uint64, error) { return 200 << 30, nil }
|
|
||||||
if noRoom(t.TempDir()) {
|
|
||||||
t.Error("a node with 200 GiB free refused work")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The build seat hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own.
|
|
||||||
func TestTheBuildSeatHandsTheCheckTheJudgeCacheItDeclares(t *testing.T) {
|
|
||||||
t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache")
|
|
||||||
if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" {
|
|
||||||
t.Fatalf("the judge cache the manifest declares is read as %q", got)
|
|
||||||
}
|
|
||||||
t.Setenv("MESH_JUDGE_CACHE", "")
|
|
||||||
if got := judgeCache(); got != "" {
|
|
||||||
t.Fatalf("with no judge cache declared, %q is used", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The siblings a check is asked to read reach the build seat's spec and, as read, the outcome (novox/hq ADR 0382).
|
|
||||||
func TestTheSiblingsAnAskNamesReachTheSpecAndTheOutcome(t *testing.T) {
|
|
||||||
spec := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main",
|
|
||||||
Siblings: []link.CheckSibling{{Repository: "novox/mesh-controller", Number: 346, Head: "a", Base: "main", Dir: "mesh-controller"}},
|
|
||||||
SiblingsUnknown: "why"}})
|
|
||||||
if len(spec.Siblings) != 1 || spec.Siblings[0] != (builder.Sibling{Repository: "novox/mesh-controller", Number: 346,
|
|
||||||
Head: "a", Base: "main", Dir: "mesh-controller"}) || spec.SiblingsUnknown != "why" {
|
|
||||||
t.Fatalf("the spec reads %+v", spec)
|
|
||||||
}
|
|
||||||
out := siblingsOut([]builder.JudgedSibling{{Repository: "novox/mesh-controller", Number: 346, Head: "a", Base: "main", BaseCommit: "b"}})
|
|
||||||
if len(out) != 1 || out[0].BaseCommit != "b" || out[0].Number != 346 {
|
|
||||||
t.Fatalf("the outcome reads %+v", out)
|
|
||||||
}
|
|
||||||
if plain := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "o", Repo: "r"}}); plain.Siblings != nil {
|
|
||||||
t.Fatalf("a request that names none has siblings %+v", plain.Siblings)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1826/AC2: the held values a check request carries reach the spec; a request that carries none leaves it empty.
|
|
||||||
func TestTheHeldValuesAnAskCarriesReachTheSpec(t *testing.T) {
|
|
||||||
spec := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog",
|
|
||||||
HeldConfig: []link.HeldConfig{{Pull: "novox/mesh-catalog#434", Head: "a", Node: "ace", Module: "n8n", Key: "media-gid",
|
|
||||||
Value: []byte("2000"), Approved: "the operator"}}}})
|
|
||||||
if len(spec.Held) != 1 || spec.Held[0].Node != "ace" || spec.Held[0].Key != "media-gid" || string(spec.Held[0].Value) != "2000" ||
|
|
||||||
spec.Held[0].Approved != "the operator" {
|
|
||||||
t.Fatalf("the spec reads %+v", spec.Held)
|
|
||||||
}
|
|
||||||
if plain := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "o", Repo: "r"}}); plain.Held != nil {
|
|
||||||
t.Fatalf("a request that carries none has %+v", plain.Held)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,433 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/micro"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
|
|
||||||
//
|
|
||||||
// **The queue is the controller's; the build running here is this machine's.** The controller can
|
|
||||||
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
|
|
||||||
// on this machine and containers in this machine's runtime, and only this machine can end them. So
|
|
||||||
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
|
|
||||||
// it, pause, resume.
|
|
||||||
//
|
|
||||||
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
|
|
||||||
// set per build — so "the build running here" is one or none, and kill names it by id so a call
|
|
||||||
// that arrives as one build ends and the next begins cannot end the wrong one.
|
|
||||||
|
|
||||||
// holder is this machine's state as a holder of the build seat.
|
|
||||||
type holder struct {
|
|
||||||
on, seat string
|
|
||||||
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
|
|
||||||
// rather than silently taking work again.
|
|
||||||
workspace string
|
|
||||||
// say publishes this machine's state: whether it takes work (link.HolderState).
|
|
||||||
say func(link.HolderState) error
|
|
||||||
// remove runs what a kill needs outside the build: the containers left behind.
|
|
||||||
remove builder.Runner
|
|
||||||
// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
|
|
||||||
// this machine publishes to no artifact store.
|
|
||||||
scratch builder.Scratch
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
paused bool
|
|
||||||
running *running
|
|
||||||
}
|
|
||||||
|
|
||||||
// running is the build this machine is doing.
|
|
||||||
type running struct {
|
|
||||||
request link.BuildRequest
|
|
||||||
step string
|
|
||||||
started time.Time
|
|
||||||
cancel context.CancelFunc
|
|
||||||
killed bool
|
|
||||||
// returned is the build's own work having ended, before a kill or not; outcome is what was then
|
|
||||||
// announced, and announced whether it went out.
|
|
||||||
returned bool
|
|
||||||
outcome string
|
|
||||||
announced bool
|
|
||||||
done chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// pausedFile is where the flag lives in the workspace.
|
|
||||||
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
|
|
||||||
|
|
||||||
// newHolder reads the paused flag the workspace keeps.
|
|
||||||
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
|
|
||||||
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
|
|
||||||
if _, err := os.Stat(pausedFile(workspace)); err == nil {
|
|
||||||
h.paused = true
|
|
||||||
}
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
// Paused is asked by the taking loop before every fetch.
|
|
||||||
func (h *holder) Paused() bool {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
return h.paused
|
|
||||||
}
|
|
||||||
|
|
||||||
// setPaused records the flag in the workspace first and then in memory, so what this holder says
|
|
||||||
// it is and what it would be after a restart never differ.
|
|
||||||
func (h *holder) setPaused(paused bool) error {
|
|
||||||
path := pausedFile(h.workspace)
|
|
||||||
if paused {
|
|
||||||
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
|
||||||
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
|
|
||||||
}
|
|
||||||
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
|
|
||||||
}
|
|
||||||
h.mu.Lock()
|
|
||||||
h.paused = paused
|
|
||||||
h.mu.Unlock()
|
|
||||||
h.announce()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
|
|
||||||
// controller reading an older state is a plan read as late rather than a build lost.
|
|
||||||
func (h *holder) announce() {
|
|
||||||
if h.say == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
|
|
||||||
// a pause outlives the events stream's retention.
|
|
||||||
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
|
||||||
h.announce()
|
|
||||||
tick := time.NewTicker(every)
|
|
||||||
defer tick.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-tick.C:
|
|
||||||
if h.Paused() {
|
|
||||||
h.announce()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
|
|
||||||
// after. Every holder sweeps the one store; what one let go of, another finds gone.
|
|
||||||
func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
|
|
||||||
sweep := func() {
|
|
||||||
if h.scratch == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
|
|
||||||
switch {
|
|
||||||
case err != nil:
|
|
||||||
fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
|
|
||||||
case n > 0:
|
|
||||||
fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sweep()
|
|
||||||
tick := time.NewTicker(every)
|
|
||||||
defer tick.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-tick.C:
|
|
||||||
sweep()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// begin records the build this machine took, with the cancel that ends it.
|
|
||||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
|
||||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
|
||||||
h.mu.Lock()
|
|
||||||
h.running = r
|
|
||||||
h.mu.Unlock()
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// end clears it, and lets a kill waiting on it know it is over.
|
|
||||||
func (h *holder) end(r *running) {
|
|
||||||
h.mu.Lock()
|
|
||||||
if h.running == r {
|
|
||||||
h.running = nil
|
|
||||||
}
|
|
||||||
h.mu.Unlock()
|
|
||||||
close(r.done)
|
|
||||||
}
|
|
||||||
|
|
||||||
// stepped records the step a build is at, for `current`.
|
|
||||||
func (h *holder) stepped(r *running, step string) {
|
|
||||||
h.mu.Lock()
|
|
||||||
r.step = step
|
|
||||||
h.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// currentBuild is what `current` answers.
|
|
||||||
type currentBuild struct {
|
|
||||||
On string `json:"on"`
|
|
||||||
Paused bool `json:"paused"`
|
|
||||||
Running *struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Repository string `json:"repository"`
|
|
||||||
Path string `json:"path,omitempty"`
|
|
||||||
Ref string `json:"ref,omitempty"`
|
|
||||||
Step string `json:"step,omitempty"`
|
|
||||||
Started string `json:"started"`
|
|
||||||
Elapsed string `json:"elapsed"`
|
|
||||||
} `json:"running,omitempty"`
|
|
||||||
Said string `json:"said"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *holder) current() currentBuild {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
out := currentBuild{On: h.on, Paused: h.paused}
|
|
||||||
taking := "taking builds"
|
|
||||||
if h.paused {
|
|
||||||
taking = "paused, taking no new build"
|
|
||||||
}
|
|
||||||
if h.running == nil {
|
|
||||||
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
r := h.running
|
|
||||||
elapsed := time.Since(r.started).Round(time.Second)
|
|
||||||
out.Running = &struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Repository string `json:"repository"`
|
|
||||||
Path string `json:"path,omitempty"`
|
|
||||||
Ref string `json:"ref,omitempty"`
|
|
||||||
Step string `json:"step,omitempty"`
|
|
||||||
Started string `json:"started"`
|
|
||||||
Elapsed string `json:"elapsed"`
|
|
||||||
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
|
|
||||||
r.started.UTC().Format(time.RFC3339), elapsed.String()}
|
|
||||||
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
|
|
||||||
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
|
|
||||||
// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
|
|
||||||
// (killAnswer in the controller's queue.go, 75s).
|
|
||||||
var (
|
|
||||||
killRemoves = 15 * time.Second
|
|
||||||
killWaits = 20 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// kill ends the build with this id, if it is the one running here and still working: its context
|
|
||||||
// cancelled — which kills each command's process group — and the containers it started removed by
|
|
||||||
// their label, once at once and again after the build has ended, so one created while it was being
|
|
||||||
// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
|
|
||||||
// ask so it is not redelivered; the answer says whether that happened.
|
|
||||||
func (h *holder) kill(id string) (string, error) {
|
|
||||||
h.mu.Lock()
|
|
||||||
r := h.running
|
|
||||||
if r == nil || r.request.ID != id {
|
|
||||||
doing := "nothing"
|
|
||||||
if r != nil {
|
|
||||||
doing = r.request.ID
|
|
||||||
}
|
|
||||||
h.mu.Unlock()
|
|
||||||
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
|
|
||||||
}
|
|
||||||
if r.returned {
|
|
||||||
h.mu.Unlock()
|
|
||||||
return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
|
|
||||||
}
|
|
||||||
r.killed = true
|
|
||||||
cancel, done := r.cancel, r.done
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
cancel()
|
|
||||||
removed, removeErr := h.removeContainers(id)
|
|
||||||
ended := false
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
ended = true
|
|
||||||
case <-time.After(killWaits):
|
|
||||||
}
|
|
||||||
again, againErr := h.removeContainers(id)
|
|
||||||
removed += again
|
|
||||||
if removeErr == nil {
|
|
||||||
removeErr = againErr
|
|
||||||
}
|
|
||||||
containers := fmt.Sprintf("%d container(s) it started removed", removed)
|
|
||||||
if removeErr != nil {
|
|
||||||
containers = "its containers could not all be listed or removed: " + removeErr.Error()
|
|
||||||
}
|
|
||||||
if !ended {
|
|
||||||
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
|
|
||||||
"its outcome is in", id, h.on, containers), nil
|
|
||||||
}
|
|
||||||
h.mu.Lock()
|
|
||||||
outcome, announced := r.outcome, r.announced
|
|
||||||
h.mu.Unlock()
|
|
||||||
if !announced {
|
|
||||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
|
|
||||||
"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
|
|
||||||
containers), nil
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
|
|
||||||
"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeContainers is one pass of removing what the build left, bounded.
|
|
||||||
func (h *holder) removeContainers(id string) (int, error) {
|
|
||||||
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
|
||||||
defer stop()
|
|
||||||
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeLeftBehind removes, once, what earlier holders on this machine left — called before anything is
|
|
||||||
// taken, so none of it is this holder's — and says what it did. A runtime that cannot be asked is said
|
|
||||||
// and the holder takes work all the same: what is left is waste, not a reason to stop building.
|
|
||||||
func (h *holder) removeLeftBehind() int {
|
|
||||||
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
|
||||||
defer stop()
|
|
||||||
n, err := builder.RemoveLeftBehind(cleanup, h.remove)
|
|
||||||
switch {
|
|
||||||
case err != nil:
|
|
||||||
fmt.Fprintf(os.Stderr, "could not look for containers earlier builds left on %s: %v\n", h.on, err)
|
|
||||||
case n > 0:
|
|
||||||
fmt.Fprintf(os.Stderr, "removed %d container(s) earlier builds left on %s\n", n, h.on)
|
|
||||||
}
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
// returned records that the build's work ended, and says whether a kill came first — only then is
|
|
||||||
// the build killed; an error it ended with on its own is its own outcome.
|
|
||||||
func (h *holder) returned(r *running) bool {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
r.returned = true
|
|
||||||
return r.killed
|
|
||||||
}
|
|
||||||
|
|
||||||
// said records the outcome announced, and whether it went out, for a kill to answer with.
|
|
||||||
func (h *holder) said(r *running, outcome string, announced bool) {
|
|
||||||
h.mu.Lock()
|
|
||||||
r.outcome, r.announced = outcome, announced
|
|
||||||
h.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// handlers are the seat's verbs, as this machine answers them.
|
|
||||||
func (h *holder) handlers() map[string]link.ToolHandler {
|
|
||||||
return map[string]link.ToolHandler{
|
|
||||||
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
|
|
||||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
|
||||||
var args struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
|
|
||||||
return nil, errors.New("kill needs the build's id")
|
|
||||||
}
|
|
||||||
said, err := h.kill(strings.TrimSpace(args.ID))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return map[string]any{"said": said}, nil
|
|
||||||
},
|
|
||||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
|
||||||
if err := h.setPaused(true); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
said := h.on + " is paused: it takes no new build until resumed"
|
|
||||||
if c := h.current(); c.Running != nil {
|
|
||||||
said += "; " + c.Running.ID + " runs on and finishes"
|
|
||||||
}
|
|
||||||
return map[string]any{"said": said, "paused": true}, nil
|
|
||||||
},
|
|
||||||
"resume": func(context.Context, json.RawMessage) (any, error) {
|
|
||||||
if err := h.setPaused(false); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func orNothing(s string) string {
|
|
||||||
if s == "" {
|
|
||||||
return "its start"
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
|
|
||||||
// log it would be is the one being ended.
|
|
||||||
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
|
|
||||||
cmd := exec.CommandContext(ctx, name, args...)
|
|
||||||
cmd.Dir = dir
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
|
||||||
}
|
|
||||||
return string(out), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
|
|
||||||
// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
|
|
||||||
// the module answering, the seat, scope node, the machine, its description and argument schema — so
|
|
||||||
// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
|
|
||||||
//
|
|
||||||
// One service per machine, named for the seat and identified by the machine, so the answer is this
|
|
||||||
// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
|
|
||||||
// store, and what it serves is what this binary was built to serve.
|
|
||||||
func announcement(seat, module, node string) micro.Info {
|
|
||||||
s, _ := catalogue.SeatNamed(seat)
|
|
||||||
var endpoints []micro.EndpointInfo
|
|
||||||
for _, v := range s.Serves {
|
|
||||||
schema, _ := json.Marshal(v.Input)
|
|
||||||
endpoints = append(endpoints, micro.EndpointInfo{
|
|
||||||
Name: seat + "__" + v.Name,
|
|
||||||
Subject: link.NodeSeatToolSubject(seat, v.Name, node),
|
|
||||||
// The queue group the verbs are served in, as every runtime announces its own.
|
|
||||||
QueueGroup: "seat." + seat,
|
|
||||||
Metadata: map[string]string{
|
|
||||||
"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
|
|
||||||
"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return micro.Info{
|
|
||||||
ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
|
|
||||||
Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
|
|
||||||
Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
|
|
||||||
Endpoints: endpoints,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
|
|
||||||
func moduleOf(user string) string {
|
|
||||||
if _, module, ok := strings.Cut(user, "."); ok && module != "" {
|
|
||||||
return module
|
|
||||||
}
|
|
||||||
return "build-agent"
|
|
||||||
}
|
|
||||||
@@ -1,172 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
|
||||||
// 0219), and what it says about itself follows.
|
|
||||||
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
|
||||||
workspace := t.TempDir()
|
|
||||||
var said []link.HolderState
|
|
||||||
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
|
||||||
said = append(said, s)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if h.Paused() {
|
|
||||||
t.Fatal("a new holder starts paused")
|
|
||||||
}
|
|
||||||
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
|
||||||
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
|
||||||
}
|
|
||||||
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
|
||||||
if !again.Paused() {
|
|
||||||
t.Fatal("restarted, the holder forgot it was paused")
|
|
||||||
}
|
|
||||||
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
|
||||||
t.Fatal("resumed, the holder came back paused")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
|
||||||
// label, and refuses an id it is not building.
|
|
||||||
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
|
||||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
|
||||||
var removed []string
|
|
||||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
|
||||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
|
||||||
if args[0] == "ps" {
|
|
||||||
return "c1\n", nil
|
|
||||||
}
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
kill := h.handlers()["kill"]
|
|
||||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
|
||||||
t.Fatal("killed a build while none ran")
|
|
||||||
}
|
|
||||||
|
|
||||||
building, cancel := context.WithCancel(context.Background())
|
|
||||||
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
|
||||||
h.stepped(r, "image")
|
|
||||||
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
|
||||||
t.Fatalf("current says %+v", c)
|
|
||||||
}
|
|
||||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), "build-1") {
|
|
||||||
t.Fatalf("killed another id: %v", err)
|
|
||||||
}
|
|
||||||
// The build's own goroutine: it ends when its context does, as a build's commands do, and
|
|
||||||
// announces what came of it.
|
|
||||||
go func() {
|
|
||||||
<-building.Done()
|
|
||||||
if h.returned(r) {
|
|
||||||
h.said(r, link.KilledByHand, true)
|
|
||||||
}
|
|
||||||
h.end(r)
|
|
||||||
}()
|
|
||||||
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if building.Err() == nil {
|
|
||||||
t.Fatal("the build's context was not cancelled")
|
|
||||||
}
|
|
||||||
if !r.killed {
|
|
||||||
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
|
||||||
}
|
|
||||||
said := answer.(map[string]any)["said"].(string)
|
|
||||||
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
|
|
||||||
t.Errorf("kill said %q", said)
|
|
||||||
}
|
|
||||||
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
|
||||||
if len(removed) != 6 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[3], "label=mesh.build=build-1") {
|
|
||||||
t.Errorf("removed %v", removed)
|
|
||||||
}
|
|
||||||
if c := h.current(); c.Running != nil {
|
|
||||||
t.Errorf("after the kill current says %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
|
|
||||||
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
|
|
||||||
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
|
|
||||||
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
|
|
||||||
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
|
|
||||||
}
|
|
||||||
kill := info.Endpoints[1]
|
|
||||||
md := kill.Metadata
|
|
||||||
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
|
|
||||||
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
|
|
||||||
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
|
|
||||||
t.Fatalf("kill is announced as %+v", kill)
|
|
||||||
}
|
|
||||||
body, err := json.Marshal(info)
|
|
||||||
if err != nil || len(body) > 8*1024 {
|
|
||||||
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
|
|
||||||
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
|
|
||||||
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
|
|
||||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
|
||||||
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
|
|
||||||
_, cancel := context.WithCancel(context.Background())
|
|
||||||
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
|
|
||||||
if killed := h.returned(r); killed {
|
|
||||||
t.Fatal("a build nobody killed reads as killed")
|
|
||||||
}
|
|
||||||
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
|
|
||||||
t.Fatalf("killed a build that had ended: %v", err)
|
|
||||||
}
|
|
||||||
h.end(r)
|
|
||||||
|
|
||||||
building, cancel := context.WithCancel(context.Background())
|
|
||||||
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
|
|
||||||
go func() {
|
|
||||||
<-building.Done()
|
|
||||||
if h.returned(r) {
|
|
||||||
h.said(r, link.KilledByHand, false)
|
|
||||||
}
|
|
||||||
h.end(r)
|
|
||||||
}()
|
|
||||||
said, err := h.kill("build-2")
|
|
||||||
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
|
|
||||||
t.Fatalf("kill said %q (%v)", said, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Issue 306**: a holder starting removes the containers earlier builds left on its machine, through
|
|
||||||
// the runner a kill uses, and a runtime that cannot be asked does not stop it.
|
|
||||||
func TestAHolderStartingRemovesWhatEarlierBuildsLeftHere(t *testing.T) {
|
|
||||||
h := newHolder("novox", link.TheBuildMachine, t.TempDir(), nil)
|
|
||||||
var removed []string
|
|
||||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
|
||||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
|
||||||
if args[0] == "ps" {
|
|
||||||
return "c1 build-1\nc2 build-1\nc3 check-here-2\n", nil
|
|
||||||
}
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
if n := h.removeLeftBehind(); n != 2 || len(removed) != 3 || removed[2] != "docker rm -f -v c1 c2" {
|
|
||||||
t.Fatalf("removed %d: %v", n, removed)
|
|
||||||
}
|
|
||||||
h.remove = func(context.Context, string, string, ...string) (string, error) {
|
|
||||||
return "", errors.New("no runtime")
|
|
||||||
}
|
|
||||||
if n := h.removeLeftBehind(); n != 0 {
|
|
||||||
t.Errorf("a runtime that cannot be asked removed %d", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,946 +0,0 @@
|
|||||||
// mesh-builder — the thing a build machine runs.
|
|
||||||
//
|
|
||||||
// It takes work from the mesh, turns a repository into artifacts, publishes them, and says what
|
|
||||||
// came out. It is **not** the control plane and it is **not** the host:
|
|
||||||
//
|
|
||||||
// - the control plane decides and never touches a machine. Building runs commands on one, and
|
|
||||||
// what the control plane may send a machine is bounded by the declaration language
|
|
||||||
// (novox/hq ADR 0005). "Run this build" is not in it, and widening the language so it could
|
|
||||||
// be would make the control plane able to run anything anywhere.
|
|
||||||
// - the host applies declarations and holds no opinion about what they contain. A host that
|
|
||||||
// also built things would need a container runtime and git, on every machine, to do something
|
|
||||||
// almost none of them will ever do.
|
|
||||||
//
|
|
||||||
// So it is a module: a program a machine runs because the mesh told it to, holding its own broker
|
|
||||||
// credential and nothing else. Compromise of a build machine is compromise of a build machine.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// version is set at build time.
|
|
||||||
var version = "development"
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
if err := run(); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-builder: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const usage = `mesh-builder — builds modules for the mesh
|
|
||||||
|
|
||||||
It consumes build requests and answers with what it made. Nothing is listened on and nothing
|
|
||||||
is dialled except the broker.
|
|
||||||
|
|
||||||
MESH_BROKER_FILE a file the mesh sealed to this machine holding the same
|
|
||||||
MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said
|
|
||||||
MESH_BINDING a file the mesh wrote saying where the artifact store is
|
|
||||||
MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is
|
|
||||||
MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it
|
|
||||||
MESH_GENESIS_PACKAGE_REGISTRY the document the installer sealed at genesis, read while no
|
|
||||||
binding is present (novox/hq ADR 0336)
|
|
||||||
MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap)
|
|
||||||
MESH_NPM_TOKEN the token for it, likewise
|
|
||||||
MESH_NPM_SCOPE the scope it answers for (default: @novox)
|
|
||||||
MESH_WORKSPACE where to clone and build (default: a temporary directory)
|
|
||||||
MESH_BUILD_CGROUP_PARENT the slice every container a build or check starts joins, which carries the
|
|
||||||
machine's limits on builds (novox/hq issue 1798; default: the runtime's own)
|
|
||||||
|
|
||||||
It also builds one module and stops, which is how a mesh is raised — before there is a
|
|
||||||
broker to take work from or a registry to publish into:
|
|
||||||
|
|
||||||
mesh-builder build <repository> [--path P] [--ref COMMIT] [--registry HOST:PORT]
|
|
||||||
|
|
||||||
Without --registry the artifacts stay in this machine's container runtime, named by the
|
|
||||||
digest of their own configuration. The result is printed as JSON.
|
|
||||||
|
|
||||||
mesh-builder ready
|
|
||||||
|
|
||||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
|
||||||
1 before: its container's health check.
|
|
||||||
|
|
||||||
mesh-builder busy
|
|
||||||
|
|
||||||
exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
|
|
||||||
from the first ask the agent takes no new build: its container's busy command, asked before the
|
|
||||||
container is replaced (novox/hq ADR 0335).
|
|
||||||
`
|
|
||||||
|
|
||||||
func run() error {
|
|
||||||
if len(os.Args) > 1 {
|
|
||||||
switch os.Args[1] {
|
|
||||||
case "version":
|
|
||||||
fmt.Println(version)
|
|
||||||
return nil
|
|
||||||
case "build":
|
|
||||||
return buildOnce(context.Background(), os.Args[2:])
|
|
||||||
case "ready":
|
|
||||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
|
|
||||||
// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
|
|
||||||
if err := ready(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return drainedTooLong(time.Now())
|
|
||||||
case "busy":
|
|
||||||
// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
|
|
||||||
// nothing is in hand.
|
|
||||||
said, err := busy()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Println(said)
|
|
||||||
return nil
|
|
||||||
case "help", "-h", "--help":
|
|
||||||
fmt.Print(usage)
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
|
|
||||||
// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
|
|
||||||
fmt.Fprint(os.Stderr, usage)
|
|
||||||
return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := notReady(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
|
|
||||||
// differently never takes a build another one made.
|
|
||||||
builder.Version = builder.SelfVersion(version)
|
|
||||||
if err := undrained(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
credential, err := brokerFrom()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
registry, err := whereToPublish()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// A build and a check run below the machine's own work (novox/hq issue 1798): the agent itself at the
|
|
||||||
// lowest priority, which its git and go children inherit, and every container in the build slice.
|
|
||||||
_ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19)
|
|
||||||
builder.CgroupParent = strings.TrimSpace(os.Getenv("MESH_BUILD_CGROUP_PARENT"))
|
|
||||||
workspace := os.Getenv("MESH_WORKSPACE")
|
|
||||||
if workspace == "" {
|
|
||||||
workspace = os.TempDir() + "/mesh-builder"
|
|
||||||
}
|
|
||||||
// **The mesh's name for this machine, not the container's.** A build is reported to the rest
|
|
||||||
// of the mesh, and a report whose origin reads `104cb10e105b` names something no other module
|
|
||||||
// can look up. The mesh already knows the answer and has a way to say it — `${machine:name}`
|
|
||||||
// in the environment file this module is handed — so the hostname is only what is left when
|
|
||||||
// nobody said.
|
|
||||||
on := os.Getenv("MESH_NODE")
|
|
||||||
if on == "" {
|
|
||||||
hostname, err := os.Hostname()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("this build machine has no name: nothing said MESH_NODE and the host would not say either: %w", err)
|
|
||||||
}
|
|
||||||
on = hostname
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
js, seat, err := dialFor(credential)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
|
|
||||||
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
|
|
||||||
// paused flag is read from the workspace before anything is taken, so a holder restarted while
|
|
||||||
// paused takes nothing.
|
|
||||||
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
|
|
||||||
body, err := json.Marshal(state)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
if h.Paused() {
|
|
||||||
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
|
||||||
}
|
|
||||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
|
||||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
|
||||||
h.removeLeftBehind()
|
|
||||||
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
|
||||||
// in hand and then says idle, and its container is replaced between builds.
|
|
||||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
|
||||||
Paused: func() bool { return h.Paused() || drained() || markFaulty(time.Now()) || noRoom(workspace) },
|
|
||||||
Taking: func() { taking(seat) },
|
|
||||||
Claim: claim,
|
|
||||||
})
|
|
||||||
defer machine.Close()
|
|
||||||
|
|
||||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
|
||||||
// one serves none, and a subscription its holder has no grant for would be refused for ever.
|
|
||||||
if seat == link.TheBuildMachine {
|
|
||||||
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
|
|
||||||
log.New(os.Stderr, "", 0))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer stopServing()
|
|
||||||
// And says so, for the console to find (novox/hq ADR 0197).
|
|
||||||
stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
|
|
||||||
announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer stopAnnouncing()
|
|
||||||
go h.stateWhilePaused(ctx, time.Hour)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
|
||||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
|
||||||
// Check builds are let go of after a week (novox/hq ADR 0350 §5).
|
|
||||||
h.scratch = publisher
|
|
||||||
go h.expireChecks(ctx, time.Hour, time.Now)
|
|
||||||
|
|
||||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
|
||||||
defer released()
|
|
||||||
holding(work.Request().Repository + " (" + work.Request().ID + ")")
|
|
||||||
answer(ctx, publisher, on, workspace, work, h)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
|
|
||||||
// holds.
|
|
||||||
//
|
|
||||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
|
||||||
// machine told about both would take work from one and answer on the other, and every log line would
|
|
||||||
// say it was fine.
|
|
||||||
func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
|
||||||
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
|
||||||
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
|
||||||
// and that is enough to dial it, pinned.
|
|
||||||
if !credential.onTheNewBus() {
|
|
||||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
|
||||||
}
|
|
||||||
// Named for what it is, not the controller whose code dials it (novox/hq issue 327).
|
|
||||||
host, _ := os.Hostname()
|
|
||||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint, nats.Name("build agent on "+host))
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
|
||||||
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
|
||||||
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
|
||||||
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
|
||||||
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
|
||||||
return js, seat, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// answer does one build and says what happened, whichever way it went.
|
|
||||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
|
|
||||||
request := work.Request()
|
|
||||||
|
|
||||||
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
|
|
||||||
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
|
|
||||||
// reaches it through the parent, and that keeps today's meaning below.
|
|
||||||
building, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
var mine *running
|
|
||||||
if h != nil {
|
|
||||||
mine = h.begin(request, cancel)
|
|
||||||
defer h.end(mine)
|
|
||||||
}
|
|
||||||
|
|
||||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
|
||||||
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
|
||||||
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
|
||||||
// the handler said nothing until the end.
|
|
||||||
fmt.Fprintf(os.Stderr, "a build request arrived for %s (%s)\n", request.Repository, request.ID)
|
|
||||||
|
|
||||||
// **Everything a build says goes two ways**: to stderr, as always, and onto the bus as the
|
|
||||||
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
|
||||||
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
|
||||||
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
|
||||||
// One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq
|
|
||||||
// issue 716).
|
|
||||||
var saying sync.Mutex
|
|
||||||
say := func(step, message string) {
|
|
||||||
saying.Lock()
|
|
||||||
defer saying.Unlock()
|
|
||||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
|
||||||
work.Say(step, message)
|
|
||||||
if mine != nil && step != "run" && step != "output" {
|
|
||||||
h.stepped(mine, step)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
builder.Said = say
|
|
||||||
defer func() { builder.Said = nil }()
|
|
||||||
if err := work.Began(ctx); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
|
||||||
}
|
|
||||||
// **A running check watches its pull request's key** (novox/hq ADR 0406 §4, issue 974): when the head or the trunk
|
|
||||||
// commit in it moves, the check is for a state nobody wants any more, and ends itself — nothing is asked of the forge
|
|
||||||
// (issue 726). A group's check, a try and a build watch nothing.
|
|
||||||
var superseded atomic.Bool
|
|
||||||
if w, ok := work.(interface {
|
|
||||||
WatchPull(context.Context, func(link.PullKey)) (func(), error)
|
|
||||||
}); ok && request.Check != nil {
|
|
||||||
stopWatching, err := w.WatchPull(building, func(k link.PullKey) {
|
|
||||||
superseded.Store(true)
|
|
||||||
say("superseded", fmt.Sprintf("the pull request's key moved to %.12s on %.12s while this checked %.12s on %.12s: stopped",
|
|
||||||
k.Head, k.Trunk, request.Ref, request.Check.Trunk))
|
|
||||||
cancel()
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "the key of %s/%s#%d could not be watched, so its check runs on if the key moves: %v\n",
|
|
||||||
request.Check.Owner, request.Check.Repo, request.Check.Number, err)
|
|
||||||
} else {
|
|
||||||
defer stopWatching()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
result := link.BuildResult{
|
|
||||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
|
||||||
Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
|
|
||||||
}
|
|
||||||
what := "building " + request.Repository
|
|
||||||
if request.Path != "" {
|
|
||||||
what += " at " + request.Path
|
|
||||||
}
|
|
||||||
if request.Ref != "" {
|
|
||||||
what += " on " + request.Ref
|
|
||||||
}
|
|
||||||
say("build", what)
|
|
||||||
|
|
||||||
npmrc, err := packagesFrom(on)
|
|
||||||
if err == nil {
|
|
||||||
say("packages", saysRegistry(npmrc))
|
|
||||||
}
|
|
||||||
registryHere, isRegistry := publisher.(builder.Registry)
|
|
||||||
var built builder.Result
|
|
||||||
if request.Check != nil {
|
|
||||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
|
||||||
// published or registered; the verdict is the outcome.
|
|
||||||
result.Checked = request.Check
|
|
||||||
registry := ""
|
|
||||||
if r, ok := publisher.(builder.Registry); ok {
|
|
||||||
registry = r.Address
|
|
||||||
}
|
|
||||||
spec := checkSpecOf(request)
|
|
||||||
// The judge cache the build agent declares as its own data, never in the workspace: only judge builds and
|
|
||||||
// mesh-lab's replays mount it, never a check's containers, so no pull request's script can write what a judge
|
|
||||||
// is built with or kept as, or what the replays run (novox/hq ADR 0358 §6, issue 709).
|
|
||||||
spec.JudgeCache = judgeCache()
|
|
||||||
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
|
|
||||||
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
|
|
||||||
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
|
|
||||||
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
|
|
||||||
if isRegistry && len(spec.Builds) > 0 {
|
|
||||||
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
|
|
||||||
Running: request.Check.Running, Seats: request.Seats, Named: namedOf(request.Check.Named)}
|
|
||||||
}
|
|
||||||
var v builder.CheckVerdict
|
|
||||||
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
|
|
||||||
if err == nil {
|
|
||||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
|
||||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
|
||||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built),
|
|
||||||
Validator: v.Validator, Siblings: siblingsOut(v.Siblings), SiblingsUnknown: v.SiblingsUnknown, JudgeFrom: v.JudgeFrom}
|
|
||||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
|
||||||
for _, m := range v.Members {
|
|
||||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
|
||||||
Commit: m.Head, Trunk: m.Trunk})
|
|
||||||
}
|
|
||||||
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
|
|
||||||
if request.Check.Try {
|
|
||||||
if line, err := json.Marshal(triedOf(v)); err == nil {
|
|
||||||
say(link.StepTried, string(line))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if err == nil {
|
|
||||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
|
||||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
|
||||||
// after a clone that then fails at npm ci.
|
|
||||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
|
||||||
// docker client (ADR 0219).
|
|
||||||
if request.Reuse != nil && isRegistry && !request.DryRun {
|
|
||||||
// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
|
|
||||||
reuse := builder.Reuse{Scratch: registryHere}
|
|
||||||
for _, p := range request.Reuse.Pulls {
|
|
||||||
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
|
|
||||||
}
|
|
||||||
// Only what a verdict names is taken (ADR 0350 §2).
|
|
||||||
reuse.Named = namedOf(request.Reuse.Builds)
|
|
||||||
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
|
|
||||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
|
||||||
forgeFrom(), say, request.Seats)
|
|
||||||
} else {
|
|
||||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
|
||||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
|
||||||
forgeFrom(), say, request.Seats)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
|
||||||
result.Mirrored = built.Mirrored
|
|
||||||
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
|
||||||
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
|
||||||
killed := false
|
|
||||||
if mine != nil {
|
|
||||||
killed = h.returned(mine) && err != nil
|
|
||||||
}
|
|
||||||
if superseded.Load() && err != nil {
|
|
||||||
// Ended by its own key moving on (ADR 0406 §4): said as the check's failure, as a kill is.
|
|
||||||
result.Failed = link.SupersededByKey
|
|
||||||
say("failed", link.SupersededByKey)
|
|
||||||
} else if killed {
|
|
||||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
|
||||||
// error it ended with is the kill's consequence, not a fault of the source.
|
|
||||||
result.Failed = link.KilledByHand
|
|
||||||
say("failed", link.KilledByHand)
|
|
||||||
} else if err != nil {
|
|
||||||
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
|
||||||
// builder that is not running, and those want completely different responses.
|
|
||||||
result.Failed = err.Error()
|
|
||||||
say("failed", err.Error())
|
|
||||||
} else if request.Check != nil {
|
|
||||||
say("checked", result.Check.Verdict+": "+result.Check.Summary)
|
|
||||||
} else {
|
|
||||||
manifest, marshalErr := json.Marshal(built.Manifest)
|
|
||||||
if marshalErr != nil {
|
|
||||||
result.Failed = marshalErr.Error()
|
|
||||||
} else {
|
|
||||||
result.Commit = built.Commit
|
|
||||||
result.Manifest = manifest
|
|
||||||
for _, made := range built.Built {
|
|
||||||
result.Made = append(result.Made, link.MadeArtifact{
|
|
||||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
result.Against = built.Against
|
|
||||||
result.SourceFingerprint = built.Source
|
|
||||||
result.SourceInputs = built.SourceInputs
|
|
||||||
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
|
||||||
result.ReuseKey = built.ReuseKey
|
|
||||||
if built.Reused != nil {
|
|
||||||
f := built.Reused.From
|
|
||||||
result.Reused = built.Reused.Words()
|
|
||||||
result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
|
|
||||||
Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
|
|
||||||
} else if request.Reuse != nil {
|
|
||||||
result.BuiltWhy = built.Why
|
|
||||||
if result.BuiltWhy == "" {
|
|
||||||
result.BuiltWhy = "it has no artifact to build"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
|
||||||
for _, r := range built.Read {
|
|
||||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
|
||||||
}
|
|
||||||
for _, s := range built.Sources {
|
|
||||||
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
|
||||||
}
|
|
||||||
// The list of predefined keys the manifest was judged against, for the controller to publish
|
|
||||||
// (novox/hq ADR 0348 §7).
|
|
||||||
if p := built.Predefined; p != nil {
|
|
||||||
result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation}
|
|
||||||
}
|
|
||||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
|
|
||||||
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
|
|
||||||
// to be built again. A machine being stopped is the other case and keeps its meaning: the
|
|
||||||
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
|
|
||||||
announcing := ctx
|
|
||||||
if killed {
|
|
||||||
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
|
||||||
defer stop()
|
|
||||||
announcing = fresh
|
|
||||||
}
|
|
||||||
announceErr := work.Announce(announcing, result)
|
|
||||||
if mine != nil {
|
|
||||||
h.said(mine, result.Failed, announceErr == nil)
|
|
||||||
}
|
|
||||||
if err := announceErr; err != nil {
|
|
||||||
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
|
||||||
// failed is a lie about work that was done — and the request stays unsettled below only if
|
|
||||||
// nothing was said at all, so another machine can try.
|
|
||||||
fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Settled only once the outcome is away, so a machine that dies before answering leaves the work
|
|
||||||
// for another rather than losing it.
|
|
||||||
if err := work.Done(); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkSpecOf is a check request as the builder runs it.
|
|
||||||
func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
|
||||||
c := request.Check
|
|
||||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
|
||||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
|
||||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
|
||||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
|
||||||
Tests: c.Tests, StoreOnly: c.StoreOnly, SiblingsUnknown: c.SiblingsUnknown}
|
|
||||||
for _, h := range c.HeldConfig {
|
|
||||||
spec.Held = append(spec.Held, builder.HeldValue{Pull: h.Pull, Head: h.Head, Node: h.Node, Module: h.Module, Key: h.Key,
|
|
||||||
Value: h.Value, Approved: h.Approved, By: h.By, At: h.At})
|
|
||||||
}
|
|
||||||
for _, s := range c.Siblings {
|
|
||||||
spec.Siblings = append(spec.Siblings, builder.Sibling{Repository: s.Repository, Number: s.Number, Head: s.Head,
|
|
||||||
Base: s.Base, Dir: s.Dir})
|
|
||||||
}
|
|
||||||
for dir, b := range c.Beside {
|
|
||||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
|
||||||
}
|
|
||||||
for _, b := range c.Builds {
|
|
||||||
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
|
|
||||||
Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
|
|
||||||
}
|
|
||||||
for _, m := range c.Members {
|
|
||||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
|
||||||
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
|
|
||||||
}
|
|
||||||
return spec
|
|
||||||
}
|
|
||||||
|
|
||||||
// siblingsOut is the siblings a check read as the outcome carries them (novox/hq ADR 0382 §4).
|
|
||||||
func siblingsOut(read []builder.JudgedSibling) []link.CheckSibling {
|
|
||||||
var out []link.CheckSibling
|
|
||||||
for _, s := range read {
|
|
||||||
out = append(out, link.CheckSibling{Repository: s.Repository, Number: s.Number, Head: s.Head, Base: s.Base,
|
|
||||||
BaseCommit: s.BaseCommit})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgeCache is the judge cache the build agent's manifest declares (MESH_JUDGE_CACHE, novox/hq ADR 0358 §6): a
|
|
||||||
// directory mounted at the same path on the host, so a judge build's mounts name what the agent sees. Empty when the
|
|
||||||
// agent declares none, and every judge is then built cold under its check's root.
|
|
||||||
func judgeCache() string { return os.Getenv("MESH_JUDGE_CACHE") }
|
|
||||||
|
|
||||||
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
|
|
||||||
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
|
|
||||||
func triedOf(v builder.CheckVerdict) link.Tried {
|
|
||||||
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
|
|
||||||
Head: v.Head, Trunk: v.Trunk}
|
|
||||||
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
|
|
||||||
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
|
|
||||||
}
|
|
||||||
if v.Repo != nil && v.Repo.Failed != "" {
|
|
||||||
t.Failed = strings.Split(v.Repo.Failed, "\n")
|
|
||||||
}
|
|
||||||
if t.Verdict == "" {
|
|
||||||
t.Verdict = "error"
|
|
||||||
}
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
// namedOf are the builds the seat's verdicts name, as the builder believes a scratch record by them (novox/hq ADR
|
|
||||||
// 0350 §2, issue 703): each module's key with every artifact's digest.
|
|
||||||
func namedOf(builds []link.CheckedBuild) []builder.NamedBuild {
|
|
||||||
var out []builder.NamedBuild
|
|
||||||
for _, b := range builds {
|
|
||||||
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
|
|
||||||
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
|
|
||||||
Digests: map[string]string{}}
|
|
||||||
for _, m := range b.Built.Made {
|
|
||||||
named.Digests[m.Name] = builder.DigestOf(m.Reference)
|
|
||||||
}
|
|
||||||
out = append(out, named)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
|
||||||
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
|
||||||
var out []link.CheckBuilt
|
|
||||||
for _, b := range built {
|
|
||||||
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(),
|
|
||||||
Kept: b.Kept, Same: b.Same, SameAs: b.SameAs}
|
|
||||||
for _, m := range b.Made {
|
|
||||||
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
|
|
||||||
}
|
|
||||||
out = append(out, c)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// layerOf is one layer of a check as the outcome carries it.
|
|
||||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
|
||||||
if l == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules}
|
|
||||||
}
|
|
||||||
|
|
||||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
|
||||||
// (novox/hq ADR 0076, issue 053) — on this machine.
|
|
||||||
//
|
|
||||||
// **The binding wins, then the genesis document** (novox/hq ADR 0336 §3, §4). The build agent's
|
|
||||||
// npm-package-registry requirement is optional: on genesis's node, before the gitea module is assigned,
|
|
||||||
// nothing provides it, and the installer sealed the forge it raised to this agent as one document
|
|
||||||
// instead. Once the requirement binds, the binding names the endpoint the way the artifact store's
|
|
||||||
// does and a sealed token file the credential, and the genesis document is left unread. A binding
|
|
||||||
// file named and absent is a requirement not bound here, never a fault: the environment names the
|
|
||||||
// path whether or not the mesh wrote it.
|
|
||||||
//
|
|
||||||
// The environment variables remain for a builder run by a person. With none of the three the result
|
|
||||||
// is disabled and says this machine, so a build that needs the registry is refused naming it (§6),
|
|
||||||
// and a build that needs none builds anyway.
|
|
||||||
func packagesFrom(machine string) (builder.Npmrc, error) {
|
|
||||||
scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE"))
|
|
||||||
if scope == "" {
|
|
||||||
scope = "@novox"
|
|
||||||
}
|
|
||||||
none := builder.Npmrc{Machine: machine}
|
|
||||||
|
|
||||||
if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" {
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, os.ErrNotExist):
|
|
||||||
// Not bound on this machine: the genesis document, or nothing.
|
|
||||||
case err != nil:
|
|
||||||
return none, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err)
|
|
||||||
default:
|
|
||||||
return fromBinding(path, raw, scope, machine)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if path := strings.TrimSpace(os.Getenv("MESH_GENESIS_PACKAGE_REGISTRY")); path != "" {
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, os.ErrNotExist):
|
|
||||||
// Not this mesh's genesis node, or the installer has not sealed it yet.
|
|
||||||
case err != nil:
|
|
||||||
return none, fmt.Errorf("cannot read the genesis document for the package registry: %w", err)
|
|
||||||
default:
|
|
||||||
return fromGenesis(path, raw, scope, machine)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A builder run by a person.
|
|
||||||
registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY"))
|
|
||||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
|
||||||
username := strings.TrimSpace(os.Getenv("MESH_NPM_USER"))
|
|
||||||
if registry == "" && secret == "" {
|
|
||||||
return none, nil
|
|
||||||
}
|
|
||||||
n := builder.Npmrc{Scope: scope, Registry: registry, From: builder.FromEnvironment, Machine: machine}
|
|
||||||
if username != "" {
|
|
||||||
n.Username, n.Password = username, secret
|
|
||||||
} else {
|
|
||||||
n.Token = secret
|
|
||||||
}
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// fromBinding composes the registry from the npm-package-registry binding the mesh wrote and the
|
|
||||||
// credential it sealed beside it. The mesh authenticates the ordinary way — a generated password the
|
|
||||||
// provider only applies (novox/hq ADR 0048) — so with a username the secret is a password (basic
|
|
||||||
// auth); without one it is a bearer token a provider minted.
|
|
||||||
func fromBinding(path string, raw []byte, scope, machine string) (builder.Npmrc, error) {
|
|
||||||
none := builder.Npmrc{Machine: machine}
|
|
||||||
var told struct {
|
|
||||||
From string `json:"from"`
|
|
||||||
At string `json:"at"`
|
|
||||||
As string `json:"as"`
|
|
||||||
Serves map[string]any `json:"serves"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &told); err != nil {
|
|
||||||
return none, fmt.Errorf("%s is not a binding: %w", path, err)
|
|
||||||
}
|
|
||||||
if told.At == "" {
|
|
||||||
return none, fmt.Errorf(
|
|
||||||
"%s says the package registry is on %q and gives no address for it", path, told.From)
|
|
||||||
}
|
|
||||||
// Composed from what the provider serves, so nothing here knows gitea's URL shape from
|
|
||||||
// another registry's: it states its port, the path its registry answers on, and the scheme.
|
|
||||||
scheme := "https"
|
|
||||||
if s, ok := told.Serves["scheme"]; ok {
|
|
||||||
scheme = fmt.Sprintf("%v", s)
|
|
||||||
}
|
|
||||||
port, ok := told.Serves["port"]
|
|
||||||
if !ok {
|
|
||||||
return none, fmt.Errorf("%s says nothing about which port the package registry answers on", path)
|
|
||||||
}
|
|
||||||
npmPath, ok := told.Serves["npm-path"]
|
|
||||||
if !ok {
|
|
||||||
return none, fmt.Errorf("%s says nothing about the path the package registry answers on", path)
|
|
||||||
}
|
|
||||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
|
||||||
if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" {
|
|
||||||
raw, err := os.ReadFile(file)
|
|
||||||
if err != nil {
|
|
||||||
return none, fmt.Errorf("cannot read this builder's package-registry credential: %w", err)
|
|
||||||
}
|
|
||||||
secret = strings.TrimSpace(string(raw))
|
|
||||||
}
|
|
||||||
n := builder.Npmrc{Scope: scope, Registry: fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath),
|
|
||||||
From: builder.FromBinding, Machine: machine}
|
|
||||||
if told.As != "" {
|
|
||||||
n.Username, n.Password = told.As, secret
|
|
||||||
} else {
|
|
||||||
n.Token = secret
|
|
||||||
}
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GenesisRegistry is the genesis document (novox/hq ADR 0336 §3): what the installer sealed to the
|
|
||||||
// build agent on its node at genesis's packages step, as the `genesis-package-registry` own-secret —
|
|
||||||
// the forge it raised, the port this node gave the package registry, its npm path, and the account it
|
|
||||||
// made with that account's password. mesh-host writes it; the field names are the two repositories'
|
|
||||||
// agreement.
|
|
||||||
type GenesisRegistry struct {
|
|
||||||
Scheme string `json:"scheme"`
|
|
||||||
At string `json:"at"`
|
|
||||||
Port int `json:"port"`
|
|
||||||
NpmPath string `json:"npm-path"`
|
|
||||||
As string `json:"as"`
|
|
||||||
Password string `json:"password"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// fromGenesis composes the registry from the genesis document.
|
|
||||||
func fromGenesis(path string, raw []byte, scope, machine string) (builder.Npmrc, error) {
|
|
||||||
none := builder.Npmrc{Machine: machine}
|
|
||||||
var g GenesisRegistry
|
|
||||||
if err := json.Unmarshal(raw, &g); err != nil {
|
|
||||||
return none, fmt.Errorf("%s is not the genesis document for the package registry: %w", path, err)
|
|
||||||
}
|
|
||||||
var missing []string
|
|
||||||
for name, value := range map[string]string{"at": g.At, "npm-path": g.NpmPath, "as": g.As, "password": g.Password} {
|
|
||||||
if strings.TrimSpace(value) == "" {
|
|
||||||
missing = append(missing, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if g.Port <= 0 {
|
|
||||||
missing = append(missing, "port")
|
|
||||||
}
|
|
||||||
if len(missing) > 0 {
|
|
||||||
sort.Strings(missing)
|
|
||||||
return none, fmt.Errorf("%s, the genesis document for the package registry, says nothing of %s",
|
|
||||||
path, strings.Join(missing, ", "))
|
|
||||||
}
|
|
||||||
scheme := g.Scheme
|
|
||||||
if scheme == "" {
|
|
||||||
scheme = "http"
|
|
||||||
}
|
|
||||||
return builder.Npmrc{Scope: scope, Registry: fmt.Sprintf("%s://%s:%d%s", scheme, g.At, g.Port, g.NpmPath),
|
|
||||||
Username: g.As, Password: g.Password, From: builder.FromGenesis, Machine: machine}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// saysRegistry is the line each build's log carries about the package registry (novox/hq ADR 0336
|
|
||||||
// §4): which of the binding and the genesis document it resolved from, or that it has neither here.
|
|
||||||
func saysRegistry(n builder.Npmrc) string {
|
|
||||||
if n.Enabled() {
|
|
||||||
return "the package registry is resolved from " + n.From
|
|
||||||
}
|
|
||||||
machine := n.Machine
|
|
||||||
if machine == "" {
|
|
||||||
machine = "this machine"
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("no package registry: %s is unbound on %s and no genesis document is here; "+
|
|
||||||
"a build needing it is refused", builder.RegistryRequirement, machine)
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeFrom is the git credential this builder may offer a clone, composed from the same binding
|
|
||||||
// and sealed secret its package-registry half already reads: the forge that answers npm is the
|
|
||||||
// forge that hosts the repositories, and its provisioner applies one password to one user for
|
|
||||||
// both. Anything missing means no credential, and every clone stays anonymous — which is all a
|
|
||||||
// mesh of public repositories ever needs.
|
|
||||||
//
|
|
||||||
// The URL names the binding's own address — the machine the mesh says the forge is on — so a
|
|
||||||
// private repository is registered and built by that address, and a clone of anything else is
|
|
||||||
// never shown this credential (git's credential store matches the whole origin).
|
|
||||||
func forgeFrom() builder.GitCredential {
|
|
||||||
path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING"))
|
|
||||||
if path == "" {
|
|
||||||
return builder.GitCredential{}
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return builder.GitCredential{}
|
|
||||||
}
|
|
||||||
var told struct {
|
|
||||||
At string `json:"at"`
|
|
||||||
As string `json:"as"`
|
|
||||||
Serves map[string]any `json:"serves"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &told); err != nil || told.At == "" || told.As == "" {
|
|
||||||
return builder.GitCredential{}
|
|
||||||
}
|
|
||||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
|
||||||
if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" {
|
|
||||||
if raw, err := os.ReadFile(file); err == nil {
|
|
||||||
secret = strings.TrimSpace(string(raw))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if secret == "" {
|
|
||||||
return builder.GitCredential{}
|
|
||||||
}
|
|
||||||
scheme := "https"
|
|
||||||
if s, ok := told.Serves["scheme"]; ok {
|
|
||||||
scheme = fmt.Sprintf("%v", s)
|
|
||||||
}
|
|
||||||
host := told.At
|
|
||||||
if port, ok := told.Serves["port"]; ok {
|
|
||||||
host = fmt.Sprintf("%s:%v", told.At, port)
|
|
||||||
}
|
|
||||||
made := url.URL{Scheme: scheme, User: url.UserPassword(told.As, secret), Host: host}
|
|
||||||
return builder.GitCredential{URL: made.String()}
|
|
||||||
}
|
|
||||||
|
|
||||||
func short(commit string) string {
|
|
||||||
if len(commit) > 8 {
|
|
||||||
return commit[:8]
|
|
||||||
}
|
|
||||||
return commit
|
|
||||||
}
|
|
||||||
|
|
||||||
// whereToPublish is the artifact store this builder uses.
|
|
||||||
//
|
|
||||||
// **Preferably from the mesh.** A builder that is a module requires an artifact store, and the
|
|
||||||
// mesh writes it a file saying which machine answers that and on what port — the same binding any
|
|
||||||
// consumer of any provision gets. Reading it means the address is not a setting somebody keeps in
|
|
||||||
// step by hand, and moving the store is an ordinary reassignment rather than an edit on every
|
|
||||||
// build machine.
|
|
||||||
//
|
|
||||||
// The environment variable remains for a builder run by a person, which is how this started and
|
|
||||||
// how it is still run while being developed.
|
|
||||||
func whereToPublish() (string, error) {
|
|
||||||
binding := strings.TrimSpace(os.Getenv("MESH_BINDING"))
|
|
||||||
if binding == "" {
|
|
||||||
registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY"))
|
|
||||||
if registry == "" {
|
|
||||||
return "", fmt.Errorf("neither MESH_BINDING nor MESH_REGISTRY: a built artifact " +
|
|
||||||
"nobody can fetch is not built")
|
|
||||||
}
|
|
||||||
return registry, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
raw, err := os.ReadFile(binding)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("cannot read what the mesh said about the artifact store: %w", err)
|
|
||||||
}
|
|
||||||
var told struct {
|
|
||||||
From string `json:"from"`
|
|
||||||
At string `json:"at"`
|
|
||||||
Serves map[string]any `json:"serves"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &told); err != nil {
|
|
||||||
return "", fmt.Errorf("%s is not a binding: %w", binding, err)
|
|
||||||
}
|
|
||||||
if told.At == "" {
|
|
||||||
// The provider is not on the private network, so there is no name to reach it by. Said
|
|
||||||
// rather than falling back to the machine's own name, which would publish to a store on
|
|
||||||
// the wrong machine and be found out much later.
|
|
||||||
return "", fmt.Errorf(
|
|
||||||
"%s says the artifact store is on %q and gives no address for it", binding, told.From)
|
|
||||||
}
|
|
||||||
port, ok := told.Serves["port"]
|
|
||||||
if !ok {
|
|
||||||
return "", fmt.Errorf("%s says nothing about which port the artifact store answers on",
|
|
||||||
binding)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s:%v", told.At, port), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// brokerFrom is where this builder connects, and with what.
|
|
||||||
//
|
|
||||||
// **Preferably from a file the mesh sealed to this machine.** A builder that is a module is given
|
|
||||||
// its credential the way every other module is given one: generated or accepted centrally, sealed
|
|
||||||
// to the machine, written by the host. Putting it in an environment variable instead would mean
|
|
||||||
// the one copy that matters passing through a terminal and a process listing.
|
|
||||||
//
|
|
||||||
// The variable remains for a builder run by a person.
|
|
||||||
func brokerFrom() (Credential, error) {
|
|
||||||
if path := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); path != "" {
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return Credential{}, fmt.Errorf("cannot read this builder's credential: %w", err)
|
|
||||||
}
|
|
||||||
said := strings.TrimSpace(string(raw))
|
|
||||||
if said == "" {
|
|
||||||
// An empty credential file is a machine that will connect as nobody and be refused,
|
|
||||||
// with the reason three layers away.
|
|
||||||
return Credential{}, fmt.Errorf("%s is empty, so this builder has no credential", path)
|
|
||||||
}
|
|
||||||
var held Credential
|
|
||||||
if err := json.Unmarshal([]byte(said), &held); err == nil && held.URL != "" {
|
|
||||||
return held, nil
|
|
||||||
}
|
|
||||||
// A file holding only a URL, which is what a person writing one by hand produces. The
|
|
||||||
// broker is then verified against whatever this machine already trusts.
|
|
||||||
return Credential{URL: said}, nil
|
|
||||||
}
|
|
||||||
return Credential{}, fmt.Errorf(
|
|
||||||
"no MESH_BROKER_FILE: a build machine with no credential for the bus has nothing to build")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Credential is what a build machine is given so it can reach the broker.
|
|
||||||
//
|
|
||||||
// Two things, because reaching a broker over TLS needs both: who to connect as, and what to check
|
|
||||||
// the certificate against. A mesh's broker presents a certificate of the mesh's own, which is in
|
|
||||||
// no public trust store, so a URL alone can only connect to a broker somebody else vouches for.
|
|
||||||
//
|
|
||||||
// **The same shape a node gets, for the same reason** (novox/hq ADR 0004): the fingerprint travels
|
|
||||||
// out of band — here, sealed with the credential — and the endpoint is verified once at connect.
|
|
||||||
type Credential struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
Fingerprint string `json:"fingerprint,omitempty"`
|
|
||||||
// User and Password ride beside the address on the bus being built (design 25): a credential
|
|
||||||
// embedded in a URL leaks into every log line that prints a connection, so the mesh seals them
|
|
||||||
// as two fields and this machine joins them once, here, to dial.
|
|
||||||
User string `json:"user,omitempty"`
|
|
||||||
Password string `json:"password,omitempty"`
|
|
||||||
// Claims are the seats the module this credential was issued for claims, as the mesh writes
|
|
||||||
// them beside the credential (novox/hq ADR 0159). The first is the build role this machine
|
|
||||||
// serves; a credential naming none is from before claims travelled in it.
|
|
||||||
Claims []struct {
|
|
||||||
Seat string `json:"seat"`
|
|
||||||
} `json:"claims,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// seatsClaimed is the seats the credential names, in order.
|
|
||||||
func (c Credential) seatsClaimed() []string {
|
|
||||||
out := make([]string, 0, len(c.Claims))
|
|
||||||
for _, claim := range c.Claims {
|
|
||||||
out = append(out, claim.Seat)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
|
|
||||||
// mesh only ever seals such a credential with the user and password beside it.
|
|
||||||
func (c Credential) onTheNewBus() bool { return strings.HasPrefix(strings.TrimSpace(c.URL), "nats://") }
|
|
||||||
|
|
||||||
// natsURL is the address with this machine's credential in it, for the one dial that needs it.
|
|
||||||
func (c Credential) natsURL() string {
|
|
||||||
rest := strings.TrimPrefix(strings.TrimSpace(c.URL), "nats://")
|
|
||||||
if c.User == "" {
|
|
||||||
return "nats://" + rest
|
|
||||||
}
|
|
||||||
return "nats://" + c.User + ":" + c.Password + "@" + rest
|
|
||||||
}
|
|
||||||
@@ -1,188 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
)
|
|
||||||
|
|
||||||
// buildOnce is the builder doing one build and stopping, with no broker and no mesh.
|
|
||||||
//
|
|
||||||
// **This is how a mesh is raised** (novox/hq ADR 0073). The installer carries this program and runs
|
|
||||||
// it once, before anything else exists, to produce the control plane from the same repository and
|
|
||||||
// path that every later rebuild of it will use. What raises the mesh is therefore the same thing
|
|
||||||
// that maintains it — not a second mechanism that has to be kept in step with the first and is
|
|
||||||
// exercised once per new mesh, which is how often enough to rot.
|
|
||||||
//
|
|
||||||
// It takes no work from a queue and answers nobody: there is no broker yet, and the only thing
|
|
||||||
// waiting for the answer is the installer that started it. So the result goes to standard output as
|
|
||||||
// JSON, which is what the installer reads.
|
|
||||||
//
|
|
||||||
// With no --registry it publishes nowhere and the image stays in this machine's container runtime,
|
|
||||||
// named by the digest of its own configuration (see builder.Local). That is the genesis case. With
|
|
||||||
// one, it publishes as it always does — the same command is how an operator builds a module by hand
|
|
||||||
// on a mesh that already exists.
|
|
||||||
func buildOnce(ctx context.Context, args []string) error {
|
|
||||||
set := flag.NewFlagSet("build", flag.ContinueOnError)
|
|
||||||
path := set.String("path", "", "the module's directory inside the repository (default: its root)")
|
|
||||||
ref := set.String("ref", "", "the commit to build; a branch is a moving target somebody else controls")
|
|
||||||
registry := set.String("registry", "",
|
|
||||||
"host:port to publish to. Without it the artifacts stay in this machine's container runtime, which is the genesis case")
|
|
||||||
workspace := set.String("workspace", "", "where to clone and build (default: a temporary directory)")
|
|
||||||
on := multiple(set, "on",
|
|
||||||
"a base this build stands on, as <module>/<artifact>=<reference>. Repeatable")
|
|
||||||
positionals, err := parseAround(set, args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(positionals) != 1 {
|
|
||||||
return errors.New("mesh-builder build <repository> [--path P] [--ref COMMIT] [--registry HOST:PORT]")
|
|
||||||
}
|
|
||||||
repository := positionals[0]
|
|
||||||
|
|
||||||
where := *workspace
|
|
||||||
if where == "" {
|
|
||||||
where = os.TempDir() + "/mesh-builder-once"
|
|
||||||
}
|
|
||||||
|
|
||||||
// What this build may stand on, said on the command line because there is no mesh to ask.
|
|
||||||
//
|
|
||||||
// **This is the genesis case and it is meant to be awkward.** On a running mesh the control
|
|
||||||
// plane answers this, because only it knows what this mesh holds. Here nothing has been built
|
|
||||||
// yet, so whoever runs this says what to use — and for the control plane, which is the one
|
|
||||||
// module raised before anything else exists, the answer is ordinarily nothing at all.
|
|
||||||
bases := map[string]string{}
|
|
||||||
for _, pair := range *on {
|
|
||||||
key, reference, found := strings.Cut(pair, "=")
|
|
||||||
if !found || key == "" || reference == "" {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"--on takes <module>/<artifact>=<reference>, and %q is not that", pair)
|
|
||||||
}
|
|
||||||
bases[key] = reference
|
|
||||||
}
|
|
||||||
|
|
||||||
// Local unless told otherwise, because the moment this exists for has nowhere to publish. A
|
|
||||||
// default pointing at a registry would mean genesis failing at a push to something that is not
|
|
||||||
// there yet, one step away from the thing that could explain it.
|
|
||||||
var publisher builder.Publisher = builder.Local{Run: builder.Command}
|
|
||||||
if *registry != "" {
|
|
||||||
publisher = builder.Registry{Address: *registry, Run: builder.Command}
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "building %s", repository)
|
|
||||||
if *path != "" {
|
|
||||||
fmt.Fprintf(os.Stderr, " at %s", *path)
|
|
||||||
}
|
|
||||||
if *ref != "" {
|
|
||||||
fmt.Fprintf(os.Stderr, " at %s", *ref)
|
|
||||||
}
|
|
||||||
fmt.Fprintln(os.Stderr)
|
|
||||||
|
|
||||||
machine := os.Getenv("MESH_NODE")
|
|
||||||
if machine == "" {
|
|
||||||
machine, _ = os.Hostname()
|
|
||||||
}
|
|
||||||
npmrc, err := packagesFrom(machine)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Fprintf(os.Stderr, " [packages] %s\n", saysRegistry(npmrc))
|
|
||||||
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc,
|
|
||||||
forgeFrom(),
|
|
||||||
func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) })
|
|
||||||
if buildErr != nil {
|
|
||||||
return buildErr
|
|
||||||
}
|
|
||||||
|
|
||||||
// To standard output, and everything else to standard error, so the caller can read this
|
|
||||||
// without having to separate it from progress.
|
|
||||||
out := onceResult{
|
|
||||||
Module: built.Manifest.Module,
|
|
||||||
Commit: built.Commit,
|
|
||||||
Repository: repository,
|
|
||||||
Path: *path,
|
|
||||||
Ref: *ref,
|
|
||||||
Manifest: built.Manifest,
|
|
||||||
Against: built.Against,
|
|
||||||
Source: built.Source,
|
|
||||||
}
|
|
||||||
for _, r := range built.Read {
|
|
||||||
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
|
|
||||||
}
|
|
||||||
for _, made := range built.Built {
|
|
||||||
out.Made = append(out.Made, madeArtifact{Name: made.Name, Kind: made.Kind, Reference: made.Reference})
|
|
||||||
}
|
|
||||||
body, err := json.MarshalIndent(out, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Println(string(body))
|
|
||||||
fmt.Fprintf(os.Stderr, " built %s from %s\n", built.Manifest.Module, short(built.Commit))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// onceResult is what one build reports to whoever started it.
|
|
||||||
//
|
|
||||||
// The same fields the mesh records for a build, so a reader comparing a genesis build against an
|
|
||||||
// ordinary one is comparing the same thing said the same way.
|
|
||||||
type onceResult struct {
|
|
||||||
Module string `json:"module"`
|
|
||||||
Commit string `json:"commit"`
|
|
||||||
Repository string `json:"repository"`
|
|
||||||
Path string `json:"path,omitempty"`
|
|
||||||
Ref string `json:"ref,omitempty"`
|
|
||||||
Manifest any `json:"manifest"`
|
|
||||||
Made []madeArtifact `json:"made"`
|
|
||||||
Against []string `json:"against,omitempty"`
|
|
||||||
Read []readRepository `json:"read,omitempty"`
|
|
||||||
Source string `json:"source-fingerprint,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// readRepository is a repository this build read source from besides the module's own.
|
|
||||||
type readRepository struct {
|
|
||||||
Repository string `json:"repository"`
|
|
||||||
Ref string `json:"ref,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type madeArtifact struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
Reference string `json:"reference"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseAround lets flags appear on either side of the repository, because a person writing this by
|
|
||||||
// hand will put them wherever reads best and the standard parser stops at the first thing that is
|
|
||||||
// not a flag. The same helper the control plane's commands use, for the same reason.
|
|
||||||
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
|
||||||
var positionals []string
|
|
||||||
rest := args
|
|
||||||
for {
|
|
||||||
if err := set.Parse(rest); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
rest = set.Args()
|
|
||||||
if len(rest) == 0 {
|
|
||||||
return positionals, nil
|
|
||||||
}
|
|
||||||
positionals = append(positionals, rest[0])
|
|
||||||
rest = rest[1:]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// multiple is a flag that may be given more than once.
|
|
||||||
type repeated []string
|
|
||||||
|
|
||||||
func (r *repeated) String() string { return strings.Join(*r, ", ") }
|
|
||||||
func (r *repeated) Set(v string) error { *r = append(*r, v); return nil }
|
|
||||||
|
|
||||||
func multiple(set *flag.FlagSet, name, usage string) *[]string {
|
|
||||||
var values repeated
|
|
||||||
set.Var(&values, name, usage)
|
|
||||||
return (*[]string)(&values)
|
|
||||||
}
|
|
||||||
@@ -1,102 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The build agent's package registry (novox/hq ADR 0336, issue 121): its npm-package-registry
|
|
||||||
// requirement is optional, so the binding may be absent, and on genesis's node the installer seals
|
|
||||||
// the forge it raised as one genesis document instead.
|
|
||||||
|
|
||||||
// registryFiles names the three paths the build agent's environment always names, writing only those
|
|
||||||
// given.
|
|
||||||
func registryFiles(t *testing.T, binding, token, genesis string) {
|
|
||||||
t.Helper()
|
|
||||||
dir := t.TempDir()
|
|
||||||
for env, content := range map[string]string{
|
|
||||||
"MESH_PACKAGE_BINDING": binding,
|
|
||||||
"MESH_NPM_TOKEN_FILE": token,
|
|
||||||
"MESH_GENESIS_PACKAGE_REGISTRY": genesis,
|
|
||||||
} {
|
|
||||||
path := filepath.Join(dir, env)
|
|
||||||
t.Setenv(env, path)
|
|
||||||
if content != "" {
|
|
||||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, env := range []string{"MESH_NPM_REGISTRY", "MESH_NPM_TOKEN", "MESH_NPM_USER", "MESH_NPM_SCOPE"} {
|
|
||||||
t.Setenv(env, "")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
aBinding = `{"from":"gitea","at":"anchor.mesh","as":"anchor-agent",
|
|
||||||
"serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`
|
|
||||||
aGenesisDocument = `{"scheme":"http","at":"127.0.0.1","port":3100,"npm-path":"/api/packages/novox/npm/",
|
|
||||||
"as":"mesh-builder","password":"genesis-secret"}`
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestUntilTheRequirementBindsTheGenesisDocumentIsRead(t *testing.T) {
|
|
||||||
registryFiles(t, "", "", aGenesisDocument)
|
|
||||||
n, err := packagesFrom("anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("an absent binding beside a genesis document was refused: %v", err)
|
|
||||||
}
|
|
||||||
if n.From != builder.FromGenesis || n.Registry != "http://127.0.0.1:3100/api/packages/novox/npm/" ||
|
|
||||||
n.Username != "mesh-builder" || n.Password != "genesis-secret" || n.Scope != "@novox" {
|
|
||||||
t.Fatalf("the genesis document was not what the registry came from: %+v", n)
|
|
||||||
}
|
|
||||||
file, err := n.File()
|
|
||||||
if err != nil || !strings.Contains(file, "//127.0.0.1:3100/api/packages/novox/npm/:username=mesh-builder") {
|
|
||||||
t.Fatalf("the .npmrc is not written from the genesis document: %v\n%s", err, file)
|
|
||||||
}
|
|
||||||
if got := saysRegistry(n); !strings.Contains(got, "genesis document") {
|
|
||||||
t.Errorf("the log does not say the genesis document: %s", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARealBindingWinsOverTheGenesisDocument(t *testing.T) {
|
|
||||||
registryFiles(t, aBinding, "bound-secret\n", aGenesisDocument)
|
|
||||||
n, err := packagesFrom("anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if n.From != builder.FromBinding || n.Registry != "http://anchor.mesh:3000/api/packages/novox/npm/" ||
|
|
||||||
n.Username != "anchor-agent" || n.Password != "bound-secret" {
|
|
||||||
t.Fatalf("the binding did not win over the genesis document: %+v", n)
|
|
||||||
}
|
|
||||||
if got := saysRegistry(n); !strings.Contains(got, "binding") {
|
|
||||||
t.Errorf("the log does not say the binding: %s", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWithNeitherTheAgentHasNoRegistryAndSaysWhere(t *testing.T) {
|
|
||||||
registryFiles(t, "", "", "")
|
|
||||||
n, err := packagesFrom("workstation")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("an unbound optional requirement was a fault: %v", err)
|
|
||||||
}
|
|
||||||
if n.Enabled() || n.Machine != "workstation" {
|
|
||||||
t.Fatalf("a registry came from nowhere: %+v", n)
|
|
||||||
}
|
|
||||||
got := saysRegistry(n)
|
|
||||||
for _, want := range []string{"npm-package-registry", "unbound", "workstation"} {
|
|
||||||
if !strings.Contains(got, want) {
|
|
||||||
t.Errorf("the log does not say %q: %s", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAGenesisDocumentMissingAFieldIsRefusedByName(t *testing.T) {
|
|
||||||
registryFiles(t, "", "", `{"at":"127.0.0.1","port":3000,"npm-path":"/api/packages/novox/npm/","as":"mesh-builder"}`)
|
|
||||||
if _, err := packagesFrom("anchor"); err == nil || !strings.Contains(err.Error(), "password") {
|
|
||||||
t.Fatalf("a genesis document with no password was taken: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp
|
|
||||||
// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its
|
|
||||||
// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without
|
|
||||||
// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole
|
|
||||||
// grace on each machine.
|
|
||||||
//
|
|
||||||
// The mark lives in the container's own temporary directory, never in the workspace the machine keeps:
|
|
||||||
// a new container starts without it, and a restarted one removes it before it binds again.
|
|
||||||
func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") }
|
|
||||||
|
|
||||||
// notReady removes the mark: a holder starting is not taking work until it says so again.
|
|
||||||
func notReady() error {
|
|
||||||
if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its
|
|
||||||
// check then says it is not ready, which is the truth the gate should hear.
|
|
||||||
func taking(seat string) {
|
|
||||||
if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ready is the health check: nil when this container's agent is bound and pulling build work.
|
|
||||||
func ready() error {
|
|
||||||
if _, err := os.Stat(readyMark()); err != nil {
|
|
||||||
return fmt.Errorf("not taking build work yet: %s is not there", readyMark())
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The health check passes only between the agent saying it takes work and its next start (novox/hq
|
|
||||||
// issue 544).
|
|
||||||
func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) {
|
|
||||||
t.Setenv("TMPDIR", t.TempDir())
|
|
||||||
if err := ready(); err == nil {
|
|
||||||
t.Fatal("an agent that never took work is ready")
|
|
||||||
}
|
|
||||||
taking("mesh-build-machine")
|
|
||||||
if err := ready(); err != nil {
|
|
||||||
t.Fatalf("an agent taking work is not ready: %v", err)
|
|
||||||
}
|
|
||||||
// Started again, it is not ready until it binds again.
|
|
||||||
if err := notReady(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := ready(); err == nil {
|
|
||||||
t.Fatal("an agent starting again is ready before it takes work")
|
|
||||||
}
|
|
||||||
if err := notReady(); err != nil {
|
|
||||||
t.Fatalf("removing a mark that is not there failed: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(readyMark()); err == nil {
|
|
||||||
t.Fatal("the mark is still there")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover).
|
|
||||||
func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) {
|
|
||||||
var held Credential
|
|
||||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x",
|
|
||||||
"claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" {
|
|
||||||
t.Errorf("the old builder's credential serves %q", got)
|
|
||||||
}
|
|
||||||
var bare Credential
|
|
||||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine {
|
|
||||||
t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **The genesis path writes its result to stdout and nothing else there.** The installer captures
|
|
||||||
// stdout alone and parses it as JSON; one stray log line makes that fail with "invalid character"
|
|
||||||
// — exactly what a builder fmt.Printf caused, breaking a build that had worked. This keeps
|
|
||||||
// diagnostics on stderr so a future print cannot repeat it silently.
|
|
||||||
func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
|
||||||
allowed := map[string]bool{
|
|
||||||
"string(body)": true, // once.go: the result JSON, which IS stdout
|
|
||||||
"version)": true, // --version
|
|
||||||
`"stopping")`: true, // the loop.s shutdown line
|
|
||||||
"usage)": true, // --help text, for a human
|
|
||||||
"said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
|
|
||||||
}
|
|
||||||
for _, file := range []string{"once.go", "main.go"} {
|
|
||||||
src, err := os.ReadFile(file)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for i, raw := range strings.Split(string(src), "\n") {
|
|
||||||
line := strings.TrimSpace(raw)
|
|
||||||
if !strings.HasPrefix(line, "fmt.Printf(") &&
|
|
||||||
!strings.HasPrefix(line, "fmt.Println(") &&
|
|
||||||
!strings.HasPrefix(line, "fmt.Print(") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ok := false
|
|
||||||
for token := range allowed {
|
|
||||||
if strings.Contains(line, token) {
|
|
||||||
ok = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("%s:%d writes a diagnostic to stdout, which the installer parses as JSON:\n %s",
|
|
||||||
file, i+1, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
|
|
||||||
// check's verdict, how long it took, and each line naming what failed.
|
|
||||||
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
|
|
||||||
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
|
|
||||||
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
|
|
||||||
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
|
|
||||||
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
|
|
||||||
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
|
|
||||||
tried.Summary != "its merge-check.sh failed: TestX" {
|
|
||||||
t.Fatalf("a failed try: %+v", tried)
|
|
||||||
}
|
|
||||||
// A head that does not merge fails both layers: the check's own word.
|
|
||||||
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
|
|
||||||
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
|
|
||||||
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
|
|
||||||
t.Fatalf("a try that does not merge: %+v", conflict)
|
|
||||||
}
|
|
||||||
// No verdict is an error, never a pass.
|
|
||||||
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
|
|
||||||
t.Fatalf("a try with no verdict: %+v", none)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,225 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/ed25519"
|
|
||||||
"crypto/rand"
|
|
||||||
"crypto/sha256"
|
|
||||||
"crypto/tls"
|
|
||||||
"crypto/x509"
|
|
||||||
"crypto/x509/pkix"
|
|
||||||
"encoding/hex"
|
|
||||||
"math/big"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Where a builder publishes.
|
|
||||||
//
|
|
||||||
// Preferably from the mesh: a builder that is a module requires an artifact store, and the mesh
|
|
||||||
// writes it a binding saying which machine answers and on what port. Reading it means the address
|
|
||||||
// is not a setting somebody keeps in step by hand.
|
|
||||||
|
|
||||||
func binding(t *testing.T, body string) string {
|
|
||||||
t.Helper()
|
|
||||||
path := filepath.Join(t.TempDir(), "artifact-store.json")
|
|
||||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheMeshSaysWhereToPublish(t *testing.T) {
|
|
||||||
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"provision":"artifact-store",
|
|
||||||
"from":"anchor","at":"anchor.internal","serves":{"port":5000,"scheme":"http"}}`))
|
|
||||||
where, err := whereToPublish()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if where != "anchor.internal:5000" {
|
|
||||||
t.Fatalf("got %q", where)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABindingWithNoAddressIsRefused(t *testing.T) {
|
|
||||||
// The provider is not on the private network, so there is no name to reach it by. Falling
|
|
||||||
// back to anything would publish to a store on the wrong machine and be found out much later.
|
|
||||||
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"anchor","serves":{"port":5000}}`))
|
|
||||||
_, err := whereToPublish()
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("a binding with nowhere to reach was accepted")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "anchor") {
|
|
||||||
t.Fatalf("the failure does not name the machine: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABindingWithNoPortIsRefused(t *testing.T) {
|
|
||||||
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"a","at":"a.internal","serves":{}}`))
|
|
||||||
if _, err := whereToPublish(); err == nil {
|
|
||||||
t.Fatal("a binding saying nothing about a port was accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheVariableStillWorksForABuilderRunByAPerson(t *testing.T) {
|
|
||||||
// Which is how this started and how it is still run while being developed.
|
|
||||||
t.Setenv("MESH_BINDING", "")
|
|
||||||
t.Setenv("MESH_REGISTRY", "127.0.0.1:5000")
|
|
||||||
where, err := whereToPublish()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if where != "127.0.0.1:5000" {
|
|
||||||
t.Fatalf("got %q", where)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNeitherIsRefusedRatherThanGuessed(t *testing.T) {
|
|
||||||
t.Setenv("MESH_BINDING", "")
|
|
||||||
t.Setenv("MESH_REGISTRY", "")
|
|
||||||
if _, err := whereToPublish(); err == nil {
|
|
||||||
t.Fatal("a builder with nowhere to publish reported somewhere")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheCredentialComesFromAFileTheMeshSealed(t *testing.T) {
|
|
||||||
// A builder that is a module is given its credential the way every module is: sealed to the
|
|
||||||
// machine and written by the host. An environment variable instead would put the one copy
|
|
||||||
// that matters through a terminal and a process listing.
|
|
||||||
path := filepath.Join(t.TempDir(), "broker")
|
|
||||||
if err := os.WriteFile(path, []byte("amqps://a-builder:secret@broker.internal:5671/\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv("MESH_BROKER_FILE", path)
|
|
||||||
t.Setenv("MESH_BROKER_AMQP", "amqp://should-not-be-used@nowhere/")
|
|
||||||
|
|
||||||
got, err := brokerFrom()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got.URL != "amqps://a-builder:secret@broker.internal:5671/" {
|
|
||||||
t.Fatalf("got %q", got.URL)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The credential the mesh seals carries what to check the broker's certificate against, because a
|
|
||||||
// mesh's broker presents a certificate of the mesh's own and no public trust store has it. A URL
|
|
||||||
// alone can only reach a broker somebody else vouches for.
|
|
||||||
func TestTheSealedCredentialCarriesWhatVerifiesTheBroker(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "broker")
|
|
||||||
if err := os.WriteFile(path, []byte(
|
|
||||||
`{"url":"amqps://a-builder:secret@broker.internal:5671/","fingerprint":"abc123"}`),
|
|
||||||
0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv("MESH_BROKER_FILE", path)
|
|
||||||
t.Setenv("MESH_BROKER_AMQP", "")
|
|
||||||
|
|
||||||
got, err := brokerFrom()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got.URL != "amqps://a-builder:secret@broker.internal:5671/" {
|
|
||||||
t.Fatalf("the url was lost: %q", got.URL)
|
|
||||||
}
|
|
||||||
if got.Fingerprint != "abc123" {
|
|
||||||
t.Fatal("the builder was given nothing to check the broker against, so it can only " +
|
|
||||||
"connect to a broker some public authority vouches for")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnEmptyCredentialFileIsRefused(t *testing.T) {
|
|
||||||
// Otherwise the builder connects as nobody and is refused, with the reason three layers away.
|
|
||||||
path := filepath.Join(t.TempDir(), "broker")
|
|
||||||
if err := os.WriteFile(path, []byte("\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv("MESH_BROKER_FILE", path)
|
|
||||||
t.Setenv("MESH_BROKER_AMQP", "")
|
|
||||||
if _, err := brokerFrom(); err == nil {
|
|
||||||
t.Fatal("an empty credential was accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABuilderWithNoCredentialAtAllSaysSo(t *testing.T) {
|
|
||||||
t.Setenv("MESH_BROKER_FILE", "")
|
|
||||||
t.Setenv("MESH_BROKER_AMQP", "")
|
|
||||||
if _, err := brokerFrom(); err == nil {
|
|
||||||
t.Fatal("a builder with no broker reported one")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The pin is compared in the spelling the mesh writes it.
|
|
||||||
//
|
|
||||||
// A bare digest against a written fingerprint never matches, and the failure is indistinguishable
|
|
||||||
// from being pointed at the wrong broker — which is the one thing this check exists to report
|
|
||||||
// truthfully. It cost a lab run.
|
|
||||||
func TestThePinIsComparedInTheSpellingTheMeshWritesIt(t *testing.T) {
|
|
||||||
certificate, key := aServerCertificate(t)
|
|
||||||
der := certificate.Certificate[0]
|
|
||||||
sum := sha256.Sum256(der)
|
|
||||||
written := "sha256:" + hex.EncodeToString(sum[:])
|
|
||||||
|
|
||||||
listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{
|
|
||||||
Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer listener.Close()
|
|
||||||
go func() {
|
|
||||||
for {
|
|
||||||
conn, err := listener.Accept()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_ = conn.(*tls.Conn).Handshake()
|
|
||||||
conn.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
_ = key
|
|
||||||
|
|
||||||
// The pin the mesh wrote must be accepted.
|
|
||||||
if err := handshakeWith(listener.Addr().String(), written); err != nil {
|
|
||||||
t.Fatalf("the broker this builder was told about was refused: %v", err)
|
|
||||||
}
|
|
||||||
// And a different one refused, or the check reports nothing.
|
|
||||||
other := "sha256:" + strings.Repeat("ab", 32)
|
|
||||||
if err := handshakeWith(listener.Addr().String(), other); err == nil {
|
|
||||||
t.Fatal("a broker this builder was not told about was accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handshakeWith runs the pin check the builder dials with against an address.
|
|
||||||
func handshakeWith(address, pin string) error {
|
|
||||||
conn, err := tls.Dial("tcp", address, broker.PinnedToFingerprint(pin))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return conn.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func aServerCertificate(t *testing.T) (tls.Certificate, ed25519.PrivateKey) {
|
|
||||||
t.Helper()
|
|
||||||
public, private, err := ed25519.GenerateKey(rand.Reader)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
template := &x509.Certificate{
|
|
||||||
SerialNumber: big.NewInt(1),
|
|
||||||
Subject: pkix.Name{CommonName: "a broker"},
|
|
||||||
NotBefore: time.Now().Add(-time.Hour),
|
|
||||||
NotAfter: time.Now().Add(time.Hour),
|
|
||||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
|
||||||
}
|
|
||||||
der, err := x509.CreateCertificate(rand.Reader, template, template, public, private)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: private}, private
|
|
||||||
}
|
|
||||||
@@ -112,11 +112,13 @@ func acceptForProvision(ctx context.Context, provider, providerModule, provision
|
|||||||
if len(accepted.Sealed) == 0 {
|
if len(accepted.Sealed) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := sendTo(ctx, open, accepted.Machines); err != nil {
|
// Composed into each consumer's composition (novox/hq ADR 0539 clause 5): a new revision its want then names.
|
||||||
return fmt.Errorf("%s is sealed and could not be sent: %w; `push %s` sends it", provision, err,
|
if _, err := composeModules(ctx, open, accepted.Machines); err != nil {
|
||||||
strings.Join(accepted.Machines, "`, `push "))
|
return fmt.Errorf("%s is sealed and its compositions on %s could not be written: %w", provision,
|
||||||
|
strings.Join(accepted.Machines, ", "), err)
|
||||||
}
|
}
|
||||||
fmt.Printf(" sent %s; each consumer starts again on it\n", strings.Join(accepted.Machines, ", "))
|
fmt.Printf(" composed for %s; each consumer starts again on it once its want names the new composition\n",
|
||||||
|
strings.Join(accepted.Machines, ", "))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+18
-17
@@ -58,13 +58,6 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
|||||||
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
||||||
// what the reads before it would say about a machine left out is not the answer's.
|
// what the reads before it would say about a machine left out is not the answer's.
|
||||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
|
||||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
|
||||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
// **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue
|
// **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue
|
||||||
// 325): a build in flight keeps the assignment pending, a module known and not built is said with how to
|
// 325): a build in flight keeps the assignment pending, a module known and not built is said with how to
|
||||||
// build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act
|
// build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act
|
||||||
@@ -171,8 +164,21 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
|||||||
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
return answer + composedAfterAct(ctx, open, node) + blockedElsewhere(ctx, open, node), nil
|
||||||
blockedElsewhere(ctx, open, node), nil
|
}
|
||||||
|
|
||||||
|
// composedAfterAct composes the node an assignment or its end changed (novox/hq ADR 0539 clause 5): what the
|
||||||
|
// answer says of it. The assigner then writes the node's wants by its rules; nothing is sent.
|
||||||
|
func composedAfterAct(ctx context.Context, open *stores, node string) string {
|
||||||
|
wrote, err := composeModules(ctx, open, []string{node})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Sprintf("\n but its compositions on %s could not all be written: %v", node, firstLine(err.Error()))
|
||||||
|
}
|
||||||
|
if len(wrote[node]) == 0 {
|
||||||
|
return fmt.Sprintf("\n %s's compositions were already as composed now", node)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("\n composed for %s (%d composition(s) written): its wants follow by the assigner's rules", node,
|
||||||
|
len(wrote[node]))
|
||||||
}
|
}
|
||||||
|
|
||||||
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
||||||
@@ -231,11 +237,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
|||||||
}
|
}
|
||||||
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
||||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
shelf, err := open.inventory.Catalogue(ctx)
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -273,8 +274,8 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
answer := fmt.Sprintf("%s no longer runs %s: its want is written removed by the assigner",
|
||||||
node, strings.Join(modules, ", "), node)
|
node, strings.Join(modules, ", "))
|
||||||
after := resolvedAfter(ctx, open, shelf, node)
|
after := resolvedAfter(ctx, open, shelf, node)
|
||||||
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
||||||
answer += "\n " + line
|
answer += "\n " + line
|
||||||
@@ -295,7 +296,7 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
|||||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||||
answer += "\n " + line
|
answer += "\n " + line
|
||||||
}
|
}
|
||||||
return answer + blockedElsewhere(ctx, open, node), nil
|
return answer + composedAfterAct(ctx, open, node) + blockedElsewhere(ctx, open, node), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
||||||
|
|||||||
@@ -221,8 +221,4 @@ func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
|||||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||||
}
|
}
|
||||||
held := heldBy(ctx)
|
|
||||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
|
||||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,9 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// An address is read from the node's settings where it is used, never recorded with a port
|
// An address is read from the node's settings where it is used, never recorded with a port
|
||||||
@@ -14,62 +11,6 @@ import (
|
|||||||
|
|
||||||
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
||||||
|
|
||||||
// **A build is recorded by digest and path**, whatever address the builder pushed to — what it
|
|
||||||
// made and what it stood on both; an image the module runs from elsewhere is left where it says.
|
|
||||||
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
|
||||||
manifest, _ := json.Marshal(map[string]any{
|
|
||||||
"module": "gitea", "version": "1",
|
|
||||||
"resources": []map[string]any{
|
|
||||||
{"id": "server", "type": "container", "name": "mesh-gitea",
|
|
||||||
"image": "anchor.internal:5100/gitea/server@" + aDigest},
|
|
||||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
|
|
||||||
"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
|
||||||
{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
|
|
||||||
"image": "valkey/valkey@" + aDigest},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
kept := buildFrom(link.BuildResult{
|
|
||||||
ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
|
|
||||||
Manifest: manifest,
|
|
||||||
Made: []link.MadeArtifact{
|
|
||||||
{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
|
|
||||||
{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
|
||||||
},
|
|
||||||
Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
|
|
||||||
})
|
|
||||||
if kept.Module != "gitea" {
|
|
||||||
t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
|
|
||||||
}
|
|
||||||
if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
|
||||||
t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
|
|
||||||
}
|
|
||||||
if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
|
||||||
t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
|
|
||||||
}
|
|
||||||
recorded, err := catalogue.ParseManifest(kept.Manifest)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
|
||||||
t.Errorf("the recorded manifest's image is %v", got)
|
|
||||||
}
|
|
||||||
if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
|
||||||
t.Errorf("the recorded manifest's archive is %v", got)
|
|
||||||
}
|
|
||||||
if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
|
|
||||||
t.Errorf("an image the build did not make was rewritten: %v", got)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
|
|
||||||
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
|
|
||||||
}
|
|
||||||
// What the build stood on is an edge to another module's artifact, and it is recorded the way
|
|
||||||
// that artifact is: by path in the store, so the edge still names the same thing when the
|
|
||||||
// store answers at another address.
|
|
||||||
if kept.Against[0] != catalogue.ArtifactStoreScheme+"mesh-tools/runtime@"+aDigest {
|
|
||||||
t.Errorf("what the build stood on was recorded by address: %v", kept.Against)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// aStore is a module offering the artifact store on 5000, published the long way as the
|
// aStore is a module offering the artifact store on 5000, published the long way as the
|
||||||
// distribution module does, so a node may be given another number for it.
|
// distribution module does, so a node may be given another number for it.
|
||||||
func aStore() catalogue.Manifest {
|
func aStore() catalogue.Manifest {
|
||||||
|
|||||||
@@ -6,6 +6,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
"net/http"
|
"net/http"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -50,6 +54,7 @@ func anAdoptedAnchor(t *testing.T) (*stores, *[]string) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
onTheBusWithItsHaves(t, open)
|
||||||
sent := &[]string{}
|
sent := &[]string{}
|
||||||
saved := sendNodes
|
saved := sendNodes
|
||||||
sendNodes = func(_ context.Context, _ *stores, names []string) error {
|
sendNodes = func(_ context.Context, _ *stores, names []string) error {
|
||||||
@@ -83,13 +88,6 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
record, err := open.inventory.NodeByName(ctx, "anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||||
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
||||||
Firewall: "ufw", Held: held, Reachable: reachable,
|
Firewall: "ufw", Held: held, Reachable: reachable,
|
||||||
@@ -178,7 +176,7 @@ func TestTakingNamesWhatItReplaces(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(said, "container hello-web (hello-web.server)") ||
|
if !strings.Contains(said, "container hello-web (hello-web.server)") ||
|
||||||
!strings.Contains(said, "push anchor") {
|
!strings.Contains(said, "compositions of anchor are written") {
|
||||||
t.Fatalf("taking did not say what it replaces and what to run:\n%s", said)
|
t.Fatalf("taking did not say what it replaces and what to run:\n%s", said)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -250,6 +248,7 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T)
|
|||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
*sent = nil // the take composed the node; what follows is judged from here
|
||||||
reportsHolding(t, open, heldFile)
|
reportsHolding(t, open, heldFile)
|
||||||
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
@@ -438,6 +437,7 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
|||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
*sent = nil // the take composed the node; what follows is judged from here
|
||||||
reportsHolding(t, open, heldFile)
|
reportsHolding(t, open, heldFile)
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -495,52 +495,6 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
|
||||||
// composed after it — never sent after it with the node still adopted. And the send inside the
|
|
||||||
// flip is not made to wait on the flip's own hold.
|
|
||||||
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
|
||||||
open, _ := anAdoptedAnchor(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reportsHolding(t, open, heldFile)
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var heldElsewhere, heldHere error
|
|
||||||
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
|
||||||
// Another caller cannot hold the node while the flip sends it.
|
|
||||||
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
|
||||||
release()
|
|
||||||
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
|
||||||
}
|
|
||||||
// The flip's own send holds it without waiting on itself.
|
|
||||||
_, release, err := holdNodes(inner, open, names)
|
|
||||||
if err != nil {
|
|
||||||
heldHere = err
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
release()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if heldElsewhere != nil || heldHere != nil {
|
|
||||||
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
|
||||||
}
|
|
||||||
// And given back once it is done.
|
|
||||||
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
release()
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
||||||
// not only its files, and the digest changes when any of them does.
|
// not only its files, and the digest changes when any of them does.
|
||||||
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||||
@@ -592,6 +546,7 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
|||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
*sent = nil // the take composed the node; what follows is judged from here
|
||||||
// Only a loopback listener: nothing off the machine, which is the same silence.
|
// Only a loopback listener: nothing off the machine, which is the same silence.
|
||||||
reportsReaching(t, open, []link.Reach{
|
reportsReaching(t, open, []link.Reach{
|
||||||
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
|
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
|
||||||
@@ -612,32 +567,94 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
// onTheBusWithItsHaves gives the controller a bus that holds, for the anchor, a want and a have of its module that agree: the node
|
||||||
// holds the node, so it waits for whatever is converging it.
|
// has everything wanted of it, as read from the haves (novox/hq ADR 0539 §9), so a flip is refused for what it is about and
|
||||||
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
// not because the bus could not be read.
|
||||||
open, _ := anAdoptedAnchor(t)
|
func onTheBusWithItsHaves(t *testing.T, open *stores) {
|
||||||
|
t.Helper()
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reportsHolding(t, open, heldFile)
|
|
||||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
t.Cleanup(js.Close)
|
||||||
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
servingBus.Store(js)
|
||||||
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
t.Cleanup(func() { servingBus.Store(nil) })
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
var whileFlipping error
|
if err != nil {
|
||||||
sendNodes = func(context.Context, *stores, []string) error {
|
|
||||||
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||||
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
composed := "sha256:" + strings.Repeat("b", 64)
|
||||||
|
put := func(bucket, key string, value []byte) {
|
||||||
|
kv, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put(ctx, key, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantKey, err := sdkdelivery.WantKey("anchor", "hello-web")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
haveKey, err := sdkdelivery.HaveKey("anchor", "hello-web")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want, err := sdkdelivery.EncodeWant(sdkdelivery.Want{FP: fp, Composed: composed})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
have, err := sdkdelivery.EncodeHave(sdkdelivery.Have{FP: fp, Composed: composed, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
put(sdkdelivery.AssignerWantsBucket, wantKey, want)
|
||||||
|
put(sdkdelivery.HaveBucket, haveKey, have)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **When the bus cannot be read, whether the machine runs what is wanted is not known, and the flip says exactly that** (issue
|
||||||
|
// 1945): not "push", and not "has not reported".
|
||||||
|
func TestTheFlipIsRefusedWithTheBusUnreachableSayingItCannotTell(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
reportsHolding(t, open, heldContainer, heldFile)
|
||||||
|
servingBus.Store(nil) // this process has no bus now, and no address to dial
|
||||||
|
_, err := converge(t.Context(), open, "anchor", true, "", "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "cannot tell whether anchor runs what is wanted: the bus is not reachable") ||
|
||||||
|
strings.Contains(err.Error(), "push") {
|
||||||
|
t.Fatalf("the flip with the bus out of reach: %v", err)
|
||||||
|
}
|
||||||
|
if len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused flip sent something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A take composes the node only when it is performed** (issue 1945): a preview, a take that names no digest and a take refused
|
||||||
|
// for a changed preview compose nothing; the performed take composes the machine once, and says so.
|
||||||
|
func TestATakeComposesTheNodeOnlyWhenItIsPerformed(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
reportsHolding(t, open, heldContainer, heldFile)
|
||||||
|
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil {
|
||||||
|
t.Fatal("a take of a held module without the preview's digest was performed")
|
||||||
|
}
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: "not-the-digest"}); err == nil {
|
||||||
|
t.Fatal("a take naming another preview's digest was performed")
|
||||||
|
}
|
||||||
|
if len(*sent) != 0 {
|
||||||
|
t.Fatalf("a preview or a refused take composed %v", *sent)
|
||||||
|
}
|
||||||
|
said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: takeDigestIn(t, preview)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(*sent) != 1 || (*sent)[0] != "anchor" || !strings.Contains(said, "compositions of anchor are written") {
|
||||||
|
t.Fatalf("the performed take composed %v and said:\n%s", *sent, said)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -205,9 +205,9 @@ func adoptedNodes(nodes []inventory.Node) []string {
|
|||||||
// The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the
|
// The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the
|
||||||
// command API alike, so a refusal is the same refusal in the same words at both (ADR 0035).
|
// command API alike, so a refusal is the same refusal in the same words at both (ADR 0035).
|
||||||
|
|
||||||
// sendNodes sends the named machines what they should be now. A variable so a test can see what
|
// sendNodes composes the named machines as they should be now (novox/hq ADR 0539 clause 5): each node's want then
|
||||||
// an act would send without a broker.
|
// names what changed. A variable so a test can see what an act would compose without a broker.
|
||||||
var sendNodes = sendTo
|
var sendNodes = composeAndSay
|
||||||
|
|
||||||
// DefaultFilter is the module converging a node assigns to load the mesh's derived filter.
|
// DefaultFilter is the module converging a node assigns to load the mesh's derived filter.
|
||||||
const DefaultFilter = "nftables"
|
const DefaultFilter = "nftables"
|
||||||
@@ -276,8 +276,8 @@ func take(ctx context.Context, open *stores, node, module string, opts takeOptio
|
|||||||
// is still the machine: the same two refusals the flip makes.
|
// is still the machine: the same two refusals the flip makes.
|
||||||
if age := time.Since(c.reported.At); age > reportFreshFor {
|
if age := time.Since(c.reported.At); age > reportFreshFor {
|
||||||
return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+
|
return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+
|
||||||
"an account newer than %s: run `push %s --wait 2m`, then preview again",
|
"an account newer than %s: wait for its next report, then preview again",
|
||||||
node, age.Round(time.Second), reportFreshFor, node)
|
node, age.Round(time.Second), reportFreshFor)
|
||||||
}
|
}
|
||||||
if opts.Digest == "" {
|
if opts.Digest == "" {
|
||||||
return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+
|
return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+
|
||||||
@@ -294,9 +294,12 @@ func take(ctx context.Context, open *stores, node, module string, opts takeOptio
|
|||||||
}
|
}
|
||||||
said := fmt.Sprintf("%s is taken on %s", module, node)
|
said := fmt.Sprintf("%s is taken on %s", module, node)
|
||||||
if holds {
|
if holds {
|
||||||
said += "; the next push replaces what the node found and holds for it:\n" + preview
|
said += "; its next composition replaces what the node found and holds for it:\n" + preview
|
||||||
}
|
}
|
||||||
return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil
|
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||||
|
return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err
|
||||||
|
}
|
||||||
|
return said + fmt.Sprintf("\n the compositions of %s are written; its wants then cut it over", node), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// comparison is everything a take puts beside what the module declares: the machine's account of
|
// comparison is everything a take puts beside what the module declares: the machine's account of
|
||||||
@@ -433,7 +436,7 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
|||||||
fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", "))
|
fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", "))
|
||||||
}
|
}
|
||||||
if c.settingsRefused != "" {
|
if c.settingsRefused != "" {
|
||||||
fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: %s\n", c.settingsRefused)
|
fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the composition leaves it out: %s\n", c.settingsRefused)
|
||||||
}
|
}
|
||||||
preview = strings.TrimRight(b.String(), "\n")
|
preview = strings.TrimRight(b.String(), "\n")
|
||||||
sum := sha256.Sum256([]byte(preview))
|
sum := sha256.Sum256([]byte(preview))
|
||||||
@@ -608,16 +611,6 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
|||||||
if filter == "" {
|
if filter == "" {
|
||||||
filter = DefaultFilter
|
filter = DefaultFilter
|
||||||
}
|
}
|
||||||
if yes {
|
|
||||||
// Held from the checks to the send, so no push composed before the flip is sent after it
|
|
||||||
// and returns the node to adopted.
|
|
||||||
held, release, err := holdNodes(ctx, open, []string{node})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
ctx = held
|
|
||||||
}
|
|
||||||
record, err := inv.NodeByName(ctx, node)
|
record, err := inv.NodeByName(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -625,24 +618,27 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
|||||||
if !record.Adopted {
|
if !record.Adopted {
|
||||||
return "", fmt.Errorf("%s is converged already; there is nothing to flip", node)
|
return "", fmt.Errorf("%s is converged already; there is nothing to flip", node)
|
||||||
}
|
}
|
||||||
reports, err := inv.LastReports(ctx)
|
reports, err := lastReports(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
current := false
|
current, unknown := false, false
|
||||||
for _, r := range reports {
|
for _, r := range reports {
|
||||||
if r.Node == node {
|
if r.Node == node {
|
||||||
current = r.Current
|
current, unknown = r.Current, r.CurrentUnknown
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if unknown {
|
||||||
|
return "", fmt.Errorf("cannot tell whether %s runs what is wanted: the bus is not reachable", node)
|
||||||
|
}
|
||||||
reported, err := inv.AdoptionOf(ctx, node)
|
reported, err := inv.AdoptionOf(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if !current || reported.At.IsZero() {
|
if !current || reported.At.IsZero() {
|
||||||
return "", fmt.Errorf("%s has not reported on the declaration it was last sent, so what it "+
|
return "", fmt.Errorf("%s does not run everything that is wanted of it yet, or has not reported, so what it "+
|
||||||
"says is reachable may not be the machine as it is: run `push %s --wait 2m` and "+
|
"says is reachable may not be the machine as it is: converge once its haves match its wants and it has "+
|
||||||
"converge once it has applied", node, node)
|
"reported", node)
|
||||||
}
|
}
|
||||||
|
|
||||||
assignedWhenPreviewed, err := inv.Assigned(ctx, node)
|
assignedWhenPreviewed, err := inv.Assigned(ctx, node)
|
||||||
@@ -738,13 +734,13 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
|||||||
if yes && countReachable(reported) == 0 {
|
if yes && countReachable(reported) == 0 {
|
||||||
return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+
|
return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+
|
||||||
"up ever is: its account looks partial — whatever reads what is listening, or what "+
|
"up ever is: its account looks partial — whatever reads what is listening, or what "+
|
||||||
"the container runtime publishes, did not answer. Fix that on the machine and run "+
|
"the container runtime publishes, did not answer. Fix that on the machine, wait for its next report, "+
|
||||||
"`push %s --wait 2m`, then preview again", node, node)
|
"then preview again", node)
|
||||||
}
|
}
|
||||||
if age := time.Since(reported.At); age > reportFreshFor {
|
if age := time.Since(reported.At); age > reportFreshFor {
|
||||||
return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+
|
return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+
|
||||||
"only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+
|
"only on an account newer than %s: wait for its next report, then preview again", node,
|
||||||
"then preview again", node, age.Round(time.Second), reportFreshFor, node)
|
age.Round(time.Second), reportFreshFor)
|
||||||
}
|
}
|
||||||
if digest == "" {
|
if digest == "" {
|
||||||
return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+
|
return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+
|
||||||
@@ -787,7 +783,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
|||||||
said += "; took " + strings.Join(took, ", ")
|
said += "; took " + strings.Join(took, ", ")
|
||||||
}
|
}
|
||||||
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||||
return said + "\n and it could not be sent: run `push " + node + "`", err
|
return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err
|
||||||
}
|
}
|
||||||
return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil
|
return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil
|
||||||
}
|
}
|
||||||
@@ -1031,18 +1027,12 @@ func adopt(ctx context.Context, open *stores, node string) (string, error) {
|
|||||||
if record.Adopted {
|
if record.Adopted {
|
||||||
return "", fmt.Errorf("%s is adopted already", node)
|
return "", fmt.Errorf("%s is adopted already", node)
|
||||||
}
|
}
|
||||||
held, release, err := holdNodes(ctx, open, []string{node})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
ctx = held
|
|
||||||
if err := inv.SetAdopted(ctx, node, true); err != nil {
|
if err := inv.SetAdopted(ctx, node, true); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node)
|
said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node)
|
||||||
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||||
return said + "\n and it could not be sent: run `push " + node + "`", err
|
return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err
|
||||||
}
|
}
|
||||||
return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil
|
return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,131 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Issue 1334, the look: when the bus did not answer an ask of any module of a tier, no build is in flight and no event follows,
|
|
||||||
// so the look itself comes back (it fails, and the looker retries it), asks the modules again, and past the bound fails the plan.
|
|
||||||
func TestALookComesBackForATierTheBusRefusedWhole(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
for _, name := range []string{"a", "b"} {
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
p := inventory.Plan{ID: "plan-bus", Repository: "novox/a", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b"}}, Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}}
|
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
bus := false
|
|
||||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
|
||||||
if !bus {
|
|
||||||
return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain)
|
|
||||||
}
|
|
||||||
return "build-" + source.Repository, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
look := func() bool {
|
|
||||||
t.Helper()
|
|
||||||
release, err := inv.HoldPlans(ctx, true)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
return advanceHeld(ctx, open)
|
|
||||||
}
|
|
||||||
read := func() inventory.Plan {
|
|
||||||
t.Helper()
|
|
||||||
got, err := inv.PlanByID(ctx, "plan-bus")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every module refused: the look says it failed, which has the looker come back; the plan is building and says why.
|
|
||||||
if !look() {
|
|
||||||
t.Fatal("a look that left the whole tier unasked for the bus did not fail, so nothing would come back to it")
|
|
||||||
}
|
|
||||||
if got := read(); got.State != inventory.PlanBuilding || !strings.Contains(got.Note, "bus did not answer") ||
|
|
||||||
got.Modules["a"].State != "" || got.Modules["b"].State != "" {
|
|
||||||
t.Fatalf("the plan after a refused tier: %s %q %+v %+v", got.State, got.Note, got.Modules["a"], got.Modules["b"])
|
|
||||||
}
|
|
||||||
// Again, still refused: the same fact is not saved again, and the look still comes back.
|
|
||||||
before := read().Revision
|
|
||||||
if !look() || read().Revision != before {
|
|
||||||
t.Fatalf("a second refused look failed to come back or saved the plan again (revision %d, was %d)", read().Revision, before)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bus answers at the next look: both are asked, and nothing waits any more.
|
|
||||||
bus = true
|
|
||||||
if look() {
|
|
||||||
t.Fatal("a look that asked the whole tier failed")
|
|
||||||
}
|
|
||||||
if got := read(); got.Modules["a"].State != "asked" || got.Modules["b"].State != "asked" || got.Modules["a"].BusWaitFrom != nil {
|
|
||||||
t.Fatalf("the tier was not asked once the bus answered: %+v %+v", got.Modules["a"], got.Modules["b"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALookFailsTheTierTheBusRefusedPastTheBound(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "a", Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/a", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
long := time.Now().UTC().Add(-busWaitBound - time.Minute)
|
|
||||||
p := inventory.Plan{ID: "plan-bus", Repository: "novox/a", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &long}}}
|
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
|
||||||
return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
release, err := inv.HoldPlans(ctx, true)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
release()
|
|
||||||
got, err := inv.PlanByID(ctx, "plan-bus")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got.State != inventory.PlanFailed || !strings.Contains(got.Note, "the bus did not answer an ask for 10m0s (since ") {
|
|
||||||
t.Fatalf("a bus silent past its bound did not fail the plan in words: %s %q", got.State, got.Note)
|
|
||||||
}
|
|
||||||
|
|
||||||
// And retried by hand: the module is named as waiting, not as asked again.
|
|
||||||
got, _ = inv.PlanByID(ctx, "plan-bus")
|
|
||||||
if got.State == inventory.PlanFailed {
|
|
||||||
got.Modules["a"].State, got.Modules["a"].Why = "failed", "x"
|
|
||||||
if err := inv.SavePlan(ctx, &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
said, err := retryPlan(ctx, open, "plan-bus")
|
|
||||||
if err != nil || !strings.Contains(said, "not asked yet: the bus did not answer") {
|
|
||||||
t.Fatalf("a retry left for the bus is said as %q %v", said, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,151 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Issue 1334: a bus that does not answer for a moment (it reloads its user list, and JetStream's API has no responder for a few
|
|
||||||
// seconds) is not a failed build. The module is left unasked and the plan building, and the next look asks it again; only a
|
|
||||||
// module's own fault, or a bus silent for ten minutes, fails the plan.
|
|
||||||
|
|
||||||
// asksOfEveryModule makes every module a test names the builder's own, the one module a walk still asks build-agent
|
|
||||||
// for (ADR 0453 §5): what a test of that ask's waits reads.
|
|
||||||
func asksOfEveryModule(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
was := buildersOwn
|
|
||||||
buildersOwn = func(string) bool { return true }
|
|
||||||
t.Cleanup(func() { buildersOwn = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) {
|
|
||||||
p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b", "c"}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{}}
|
|
||||||
byName := map[string]inventory.Entry{}
|
|
||||||
for _, m := range []string{"a", "b", "c"} {
|
|
||||||
byName[m] = inventory.Entry{Source: inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main"}}
|
|
||||||
}
|
|
||||||
return p, byName
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) {
|
|
||||||
asksOfEveryModule(t)
|
|
||||||
p, byName := aTierOfThree()
|
|
||||||
bus := false
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
|
||||||
if source.Repository == "novox/b" && !bus {
|
|
||||||
return "", fmt.Errorf("%w: the bucket mesh-controller_builds is not on the bus: nats: no responders available for request",
|
|
||||||
inventory.ErrNoChain)
|
|
||||||
}
|
|
||||||
return "build-" + source.Repository, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
|
|
||||||
for _, m := range []string{"a", "b", "c"} {
|
|
||||||
askModule(t.Context(), p, m, byName)
|
|
||||||
}
|
|
||||||
if p.State != inventory.PlanBuilding {
|
|
||||||
t.Fatalf("a bus that did not answer for a moment failed the plan: %s, %q", p.State, p.Note)
|
|
||||||
}
|
|
||||||
if p.Modules["a"].State != "asked" || p.Modules["c"].State != "asked" {
|
|
||||||
t.Fatalf("the modules the bus did answer for were not asked: %+v %+v", p.Modules["a"], p.Modules["c"])
|
|
||||||
}
|
|
||||||
if s := p.Modules["b"]; s.State != "" {
|
|
||||||
t.Fatalf("the module the bus did not answer for is %q, not left unasked", s.State)
|
|
||||||
}
|
|
||||||
if got := unaskedOf(*p); len(got) != 1 || got[0] != "b" {
|
|
||||||
t.Fatalf("the next look would ask %v, not b", got)
|
|
||||||
}
|
|
||||||
if !strings.Contains(p.Note, "bus did not answer") {
|
|
||||||
t.Fatalf("the plan says nothing of the wait: %q", p.Note)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The next look, the bus answering: b is asked and the plan goes on.
|
|
||||||
bus = true
|
|
||||||
for _, m := range unaskedOf(*p) {
|
|
||||||
askModule(t.Context(), p, m, byName)
|
|
||||||
}
|
|
||||||
if p.State != inventory.PlanBuilding || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
|
|
||||||
t.Fatalf("b was not asked once the bus answered: %s %+v", p.State, p.Modules["b"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABusSilentForTenMinutesFailsTheTierAsBefore(t *testing.T) {
|
|
||||||
asksOfEveryModule(t)
|
|
||||||
p, byName := aTierOfThree()
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
|
||||||
return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
askModule(t.Context(), p, "a", byName)
|
|
||||||
if p.State != inventory.PlanBuilding {
|
|
||||||
t.Fatalf("the first refusal failed the plan: %q", p.Note)
|
|
||||||
}
|
|
||||||
long := time.Now().Add(-busWaitBound - time.Minute)
|
|
||||||
p.Modules["a"].BusWaitFrom = &long
|
|
||||||
askModule(t.Context(), p, "a", byName)
|
|
||||||
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" {
|
|
||||||
t.Fatalf("a bus silent past its bound did not fail the plan: %s %+v", p.State, p.Modules["a"])
|
|
||||||
}
|
|
||||||
// Said in plain words: the bus, the wait, and since when.
|
|
||||||
want := "a could not be asked for: the bus did not answer an ask for 10m0s (since " + long.UTC().Format(time.RFC3339) +
|
|
||||||
"): this kind of record lives on the bus"
|
|
||||||
if !strings.HasPrefix(p.Note, want) || p.Modules["a"].Why != strings.TrimPrefix(want, "a could not be asked for: ")+p.Note[len(want):] {
|
|
||||||
t.Fatalf("the plan says %q, not words starting %q", p.Note, want)
|
|
||||||
}
|
|
||||||
// The wait has ended: a Retry made while the bus still reloads waits again, and does not fail at its first refusal.
|
|
||||||
if p.Modules["a"].BusWaitFrom != nil {
|
|
||||||
t.Fatalf("a failed module still holds the start of its wait: %v", p.Modules["a"].BusWaitFrom)
|
|
||||||
}
|
|
||||||
p.State = inventory.PlanBuilding
|
|
||||||
askModule(t.Context(), p, "a", byName)
|
|
||||||
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "" || p.Modules["a"].BusWaitFrom == nil {
|
|
||||||
t.Fatalf("a retry made while the bus is silent did not wait afresh: %s %+v", p.State, p.Modules["a"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAPlanWithAModuleWaitingForTheBusIsOneTheLookComesBackTo(t *testing.T) {
|
|
||||||
p, _ := aTierOfThree()
|
|
||||||
if busWaiting(*p) {
|
|
||||||
t.Fatal("a plan whose modules are not asked yet waits for the bus")
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
p.Modules["b"] = &inventory.PlanModule{BusWaitFrom: &now}
|
|
||||||
p.Modules["a"] = &inventory.PlanModule{State: "asked"}
|
|
||||||
if got := busWaitingIn(*p); len(got) != 1 || got[0] != "b" {
|
|
||||||
t.Fatalf("the modules waiting for the bus are %v, not b", got)
|
|
||||||
}
|
|
||||||
p.State = inventory.PlanFailed
|
|
||||||
if busWaiting(*p) {
|
|
||||||
t.Fatal("a plan that is not open is waited for")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAModulesOwnFaultStillFailsThePlan(t *testing.T) {
|
|
||||||
asksOfEveryModule(t)
|
|
||||||
p, byName := aTierOfThree()
|
|
||||||
delete(byName, "a")
|
|
||||||
askModule(t.Context(), p, "a", byName)
|
|
||||||
if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "no longer in the catalogue") {
|
|
||||||
t.Fatalf("a module gone from the catalogue did not fail the plan: %s %q", p.State, p.Note)
|
|
||||||
}
|
|
||||||
|
|
||||||
p, byName = aTierOfThree()
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
|
||||||
return "", errors.New("git.example/novox/a is no repository the catalogue builds a module from")
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
askModule(t.Context(), p, "a", byName)
|
|
||||||
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" {
|
|
||||||
t.Fatalf("a module's own refusal did not fail the plan: %s %+v", p.State, p.Modules["a"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// busWaitBound is how long a module whose ask the bus did not answer is asked again at every look before the plan fails
|
|
||||||
// (issue 1334).
|
|
||||||
const busWaitBound = 10 * time.Minute
|
|
||||||
|
|
||||||
// busDidNotAnswer says an ask failed because the bus's JetStream had no responder at that moment, as it has for a few seconds
|
|
||||||
// when the bus reloads its user list (novox/hq issue 353), and not because of the module: the same ask is made again.
|
|
||||||
func busDidNotAnswer(err error) bool {
|
|
||||||
if err == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrJetStreamNotEnabled) ||
|
|
||||||
errors.Is(err, jetstream.ErrJetStreamNotEnabled) || errors.Is(err, inventory.ErrNoChain) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// An error that crossed a boundary as text only.
|
|
||||||
text := err.Error()
|
|
||||||
return strings.Contains(text, "no responders available") || strings.Contains(text, "jetstream not enabled")
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForTheBus leaves a module whose ask the bus did not answer unasked, the plan building, and says so in the plan's note: the
|
|
||||||
// next look asks the tier's unasked modules again. It says false, so the caller fails the plan as for any refusal, once the bus
|
|
||||||
// has been silent for busWaitBound.
|
|
||||||
func waitForTheBus(p *inventory.Plan, state *inventory.PlanModule, name string, err error, now time.Time) bool {
|
|
||||||
if state.BusWaitFrom == nil {
|
|
||||||
state.BusWaitFrom = &now
|
|
||||||
}
|
|
||||||
if now.Sub(*state.BusWaitFrom) > busWaitBound {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
state.State, state.Why = "", ""
|
|
||||||
p.Note = fmt.Sprintf("tier %d: %s not asked: the bus did not answer (%v); asked again at the next look, failing after %s",
|
|
||||||
p.Tier, name, firstLine(err.Error()), busWaitBound)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaiting says an open plan has a module of its current tier that the bus did not answer an ask of: the look must come back
|
|
||||||
// for it, as nothing else will when no build of the tier is in flight (issue 1334).
|
|
||||||
func busWaiting(p inventory.Plan) bool {
|
|
||||||
return len(busWaitingIn(p)) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaitingIn are the modules of the plan's current tier left unasked because the bus did not answer, sorted.
|
|
||||||
func busWaitingIn(p inventory.Plan) []string {
|
|
||||||
if !p.Open() || p.Tier >= len(p.Tiers) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var out []string
|
|
||||||
for _, m := range p.Tiers[p.Tier] {
|
|
||||||
if s := p.Modules[m]; s != nil && s.State == "" && s.BusWaitFrom != nil {
|
|
||||||
out = append(out, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaitWords is how a verb says modules are waiting for the bus: not asked yet, and asked again by the plan at its next look.
|
|
||||||
func busWaitWords(p inventory.Plan, modules []string) string {
|
|
||||||
return fmt.Sprintf("%s not asked yet: the bus did not answer; %s asks again at its next look", strings.Join(modules, ", "), p.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// slicesWithout is names without those in drop, in order.
|
|
||||||
func slicesWithout(names, drop []string) []string {
|
|
||||||
var out []string
|
|
||||||
for _, n := range names {
|
|
||||||
if !slices.Contains(drop, n) {
|
|
||||||
out = append(out, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A retry of a plan failed past the bus's bound, whose module the bus still does not answer, names that module as waiting:
|
|
||||||
// read while the plan still says failed, it was named nowhere, and the retry said "asked" with nothing after it (issue
|
|
||||||
// 1403, met again once the hand rebuild no longer resumed the plan first).
|
|
||||||
func TestARetryNamesTheModulesStillWaitingForTheBus(t *testing.T) {
|
|
||||||
since := time.Now().UTC()
|
|
||||||
p := inventory.Plan{ID: "plan-bus", State: inventory.PlanFailed, Tiers: [][]string{{"a"}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &since}}}
|
|
||||||
waiting := busWaitingOnRetry(p)
|
|
||||||
_, said := retryWords(p, []string{"a"}, nil, waiting)
|
|
||||||
if !strings.Contains(said, "a not asked yet: the bus did not answer") || strings.Contains(said, "asked ;") {
|
|
||||||
t.Fatalf("a retry left for the bus is said as %q", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,141 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own
|
|
||||||
// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1).
|
|
||||||
func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
|
|
||||||
twoTiers(t, open)
|
|
||||||
before := time.Now().UTC().Add(-time.Hour)
|
|
||||||
ended := before.Add(time.Minute)
|
|
||||||
failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
|
||||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
|
||||||
Note: "a failed to build in tier 0",
|
|
||||||
Delivery: &inventory.PlanDelivery{},
|
|
||||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
|
||||||
Why: link.KilledByHand}},
|
|
||||||
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
|
|
||||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
conn, err := nats.Connect(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
// The controller's seat answers `plans` as it does: by running the verb.
|
|
||||||
sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) {
|
|
||||||
var args map[string]string
|
|
||||||
_ = json.Unmarshal(m.Data, &args)
|
|
||||||
answer := link.Answer{Result: json.RawMessage(`"ok"`)}
|
|
||||||
if len(args) != 1 || args["retry"] == "" {
|
|
||||||
answer = link.Answer{Error: "the retry was asked with more than the plan"}
|
|
||||||
} else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil {
|
|
||||||
answer = link.Answer{Error: err.Error()}
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(answer)
|
|
||||||
_ = m.Respond(body)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = sub.Unsubscribe() }()
|
|
||||||
|
|
||||||
// What S23 raises for it, read from the store as the watchdog reads it.
|
|
||||||
raised := func(context.Context) ([]conditions.Condition, error) {
|
|
||||||
facts, err := gatherFailedPlans(ctx, open.inventory, time.Now())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []conditions.Condition
|
|
||||||
for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) {
|
|
||||||
out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
|
||||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions})
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
store := memAskedStore{}
|
|
||||||
var sent []published
|
|
||||||
var acts []link.HandAct
|
|
||||||
a := &asker{open: raised, store: store,
|
|
||||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
|
||||||
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
|
||||||
sent = append(sent, published{subject, id, body})
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
call: callAction(conn),
|
|
||||||
record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil },
|
|
||||||
now: func() time.Time { return now },
|
|
||||||
logf: t.Logf}
|
|
||||||
cs, _ := raised(ctx)
|
|
||||||
if len(cs) != 1 || len(cs[0].Actions) != 1 {
|
|
||||||
t.Fatalf("the failed plan raised %+v", cs)
|
|
||||||
}
|
|
||||||
if err := a.reconcile(ctx); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var w asks.Warrant
|
|
||||||
for _, r := range store {
|
|
||||||
for _, o := range r.Ask.Options {
|
|
||||||
if o.Label == "Retry" {
|
|
||||||
w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen,
|
|
||||||
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
|
||||||
AskDigest: r.Ask.Digest(),
|
|
||||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w.Ask == "" {
|
|
||||||
t.Fatalf("no ask offers Retry: %+v", store)
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(w)
|
|
||||||
if err := a.Decided(ctx, body); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.Decided(ctx, body); err != nil { // heard again: nothing more
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
m := p.Modules["a"]
|
|
||||||
if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" {
|
|
||||||
t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
|
|
||||||
}
|
|
||||||
if len(*asked) != 1 {
|
|
||||||
t.Fatalf("the build was asked %d times: %v", len(*asked), *asked)
|
|
||||||
}
|
|
||||||
if got := store[w.Ask]; got.Acted != "done" {
|
|
||||||
t.Fatalf("the ask ended %q", got.Acted)
|
|
||||||
}
|
|
||||||
if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) {
|
|
||||||
t.Fatalf("the warrant was recorded as %+v", acts)
|
|
||||||
}
|
|
||||||
if cs, _ = raised(ctx); len(cs) != 0 {
|
|
||||||
t.Fatalf("the plan-failed condition did not end: %+v", cs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -125,8 +125,15 @@ func newAskerRig(t *testing.T) *askerRig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func heldCondition() conditions.Condition {
|
func heldCondition() conditions.Condition {
|
||||||
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove,
|
||||||
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}}
|
||||||
|
stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove,
|
||||||
|
Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}}
|
||||||
|
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096", Kind: "delivery-stalled",
|
||||||
|
Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||||
|
Headline: "Delivery of hq held for 36 hours",
|
||||||
|
Explanation: "A delivery of hq has been held for 36 hours, past its limit.",
|
||||||
|
Needs: "release it, or stop it, " + FromMeshMCPServer, Actions: []conditions.Action{release, stop}}
|
||||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
||||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||||
}
|
}
|
||||||
@@ -299,35 +306,6 @@ func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
|
||||||
plan := "plan-1791454185265004861"
|
|
||||||
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
|
||||||
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
|
||||||
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
|
||||||
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
|
||||||
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
|
||||||
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
|
||||||
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
|
||||||
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
c conditions.Condition
|
|
||||||
label string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
|
||||||
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
|
||||||
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
|
||||||
} {
|
|
||||||
r := newAskerRig(t)
|
|
||||||
r.open = []conditions.Condition{tc.c}
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
|
||||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
|
||||||
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
||||||
r := newAskerRig(t)
|
r := newAskerRig(t)
|
||||||
r.open = []conditions.Condition{unitsCondition()}
|
r.open = []conditions.Condition{unitsCondition()}
|
||||||
@@ -1026,215 +1004,3 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
|||||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it.
|
|
||||||
func failedPlanCondition(plan string) conditions.Condition {
|
|
||||||
p := aFailedPlan(plan, nil)
|
|
||||||
f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC))
|
|
||||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
|
||||||
o := watchPlansFailed(f)[0]
|
|
||||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
|
||||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again".
|
|
||||||
func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) {
|
|
||||||
r := newAskerRig(t)
|
|
||||||
c := failedPlanCondition("plan-1792433168676870614")
|
|
||||||
r.open = []conditions.Condition{c}
|
|
||||||
if err := r.a.reconcile(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sent := r.asksSent(t)
|
|
||||||
if len(sent) != 1 || len(sent[0].Options) != 1 {
|
|
||||||
t.Fatalf("asked %+v", sent)
|
|
||||||
}
|
|
||||||
o := sent[0].Options[0]
|
|
||||||
if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" {
|
|
||||||
t.Errorf("the option is %+v", o)
|
|
||||||
}
|
|
||||||
if err := sent[0].Check(r.now); err != nil {
|
|
||||||
t.Errorf("the ask is refused as made: %v", err)
|
|
||||||
}
|
|
||||||
if strings.Contains(sent[0].Explanation, "mesh MCP server") {
|
|
||||||
t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1315/AC2: the warrant calls plans with retry=<plan> and nothing else, once, and is recorded as the hand act;
|
|
||||||
// the same warrant heard again does nothing.
|
|
||||||
func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) {
|
|
||||||
plan := "plan-1792433168676870614"
|
|
||||||
r := newAskerRig(t)
|
|
||||||
c := failedPlanCondition(plan)
|
|
||||||
r.open = []conditions.Condition{c}
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
w := r.warrantFor(t, c.Key, "Retry")
|
|
||||||
answerWith(t, r, w)
|
|
||||||
answerWith(t, r, w)
|
|
||||||
if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want {
|
|
||||||
t.Fatalf("called %v, want exactly %s", r.called, want)
|
|
||||||
}
|
|
||||||
if len(r.acts) != 1 {
|
|
||||||
t.Fatalf("hand-acts %+v", r.acts)
|
|
||||||
}
|
|
||||||
act := r.acts[0]
|
|
||||||
if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan ||
|
|
||||||
act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer {
|
|
||||||
t.Errorf("the hand-act is %+v", act)
|
|
||||||
}
|
|
||||||
if got := r.store[w.Ask]; got.Acted != "done" {
|
|
||||||
t.Errorf("kept %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is
|
|
||||||
// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before.
|
|
||||||
func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) {
|
|
||||||
plan := "plan-1792433168676870614"
|
|
||||||
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
|
||||||
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
|
||||||
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
c conditions.Condition
|
|
||||||
label string
|
|
||||||
}{
|
|
||||||
{"retry", failedPlanCondition(plan), "Retry"},
|
|
||||||
{"stop", waiting, "Stop"},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
r := newAskerRig(t)
|
|
||||||
r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"}
|
|
||||||
r.open = []conditions.Condition{tc.c}
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
w := r.warrantFor(t, tc.c.Key, tc.label)
|
|
||||||
answerWith(t, r, w)
|
|
||||||
answerWith(t, r, w) // heard again: told once
|
|
||||||
if len(r.told) != 1 {
|
|
||||||
t.Fatalf("told %+v", r.told)
|
|
||||||
}
|
|
||||||
got := r.told[0]
|
|
||||||
if got.headline != tc.c.Headline {
|
|
||||||
t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline)
|
|
||||||
}
|
|
||||||
said := strings.Join(got.lines, " ")
|
|
||||||
if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) {
|
|
||||||
t.Errorf("the lines do not carry the answer and the refusal's words: %q", said)
|
|
||||||
}
|
|
||||||
if why, ok := conditions.Plain(said); !ok {
|
|
||||||
t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said)
|
|
||||||
}
|
|
||||||
if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") {
|
|
||||||
t.Errorf("the failure is not recorded as before: %+v", r.acts)
|
|
||||||
}
|
|
||||||
if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") {
|
|
||||||
t.Errorf("kept %+v", kept)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here.
|
|
||||||
func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) {
|
|
||||||
r := newAskerRig(t)
|
|
||||||
c := failedPlanCondition("plan-1792433168676870614")
|
|
||||||
r.open = []conditions.Condition{c}
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
|
||||||
if len(r.told) != 0 {
|
|
||||||
t.Fatalf("a retry that was done told %+v", r.told)
|
|
||||||
}
|
|
||||||
|
|
||||||
r = newAskerRig(t)
|
|
||||||
r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"}
|
|
||||||
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
|
||||||
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
|
||||||
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
|
||||||
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
|
||||||
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
|
||||||
r.open = []conditions.Condition{module}
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
answerWith(t, r, r.warrantFor(t, module.Key, "Restart"))
|
|
||||||
if len(r.told) != 0 || len(r.acts) != 1 {
|
|
||||||
t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was
|
|
||||||
// changed"; and the sentence a refusal falls back to fits the answer given.
|
|
||||||
func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) {
|
|
||||||
plan := "plan-1792433168676870614"
|
|
||||||
r := newAskerRig(t)
|
|
||||||
r.refuses = errors.New("mesh-controller did not answer plans within 30s")
|
|
||||||
c := failedPlanCondition(plan)
|
|
||||||
r.open = []conditions.Condition{c}
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
|
||||||
if len(r.told) != 1 {
|
|
||||||
t.Fatalf("told %+v", r.told)
|
|
||||||
}
|
|
||||||
said := strings.Join(r.told[0].lines, " ")
|
|
||||||
if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") {
|
|
||||||
t.Errorf("a call that may have run is told as %q", said)
|
|
||||||
}
|
|
||||||
if why, ok := conditions.Plain(said); !ok {
|
|
||||||
t.Errorf("not plain: %s: %q", why, said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) {
|
|
||||||
plan := "plan-1792433168676870614"
|
|
||||||
for _, tc := range []struct {
|
|
||||||
act conditions.Action
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{retryAction(plan), "a retry would not run now"},
|
|
||||||
{conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"},
|
|
||||||
{conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"},
|
|
||||||
} {
|
|
||||||
// Words the operator's channel withholds: a hash and a dotted name.
|
|
||||||
got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act)
|
|
||||||
if !strings.Contains(got, tc.want) {
|
|
||||||
t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want)
|
|
||||||
}
|
|
||||||
if _, ok := conditions.Plain(got); !ok {
|
|
||||||
t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the
|
|
||||||
// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not.
|
|
||||||
func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) {
|
|
||||||
r := newAskerRig(t)
|
|
||||||
c := failedPlanCondition("plan-1792433168676870614")
|
|
||||||
c.Raised = r.now.Add(-time.Minute)
|
|
||||||
r.open = []conditions.Condition{c}
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
|
||||||
// Still open right after the answer: not asked again at once.
|
|
||||||
r.now = r.now.Add(time.Minute)
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
if n := len(r.asksSent(t)); n != 1 {
|
|
||||||
t.Fatalf("asked again while the same condition stayed open: %d", n)
|
|
||||||
}
|
|
||||||
// The retry ran, the plan left failed and the condition ended.
|
|
||||||
r.now = r.now.Add(time.Minute)
|
|
||||||
r.open = nil
|
|
||||||
_ = r.a.reconcile(context.Background())
|
|
||||||
// It fails again, ten minutes after the answer: a condition raised after the answered ask ended.
|
|
||||||
r.now = r.now.Add(8 * time.Minute)
|
|
||||||
again := failedPlanCondition("plan-1792433168676870614")
|
|
||||||
again.Raised = r.now
|
|
||||||
r.open = []conditions.Condition{again}
|
|
||||||
if err := r.a.reconcile(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if n := len(r.asksSent(t)); n != 2 {
|
|
||||||
t.Fatalf("the second failure was asked %d times in all, want 2", n)
|
|
||||||
}
|
|
||||||
if len(r.called) != 1 {
|
|
||||||
t.Fatalf("something retried without an answer: %v", r.called)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,186 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aKeeper())
|
|
||||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
|
||||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
|
||||||
_, _, err := composeQuestion(ctx, open, "laptop", compose.Reading, false)
|
|
||||||
var notMade *catalogue.NotMadeError
|
|
||||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
|
||||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
|
||||||
}
|
|
||||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
|
||||||
foreseen, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false)
|
|
||||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
|
||||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
|
||||||
}
|
|
||||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
|
||||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
|
||||||
if got := d1(t, open); len(got) != 0 {
|
|
||||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
|
||||||
before := awaitingPushBound
|
|
||||||
awaitingPushBound = -time.Minute
|
|
||||||
t.Cleanup(func() { awaitingPushBound = before })
|
|
||||||
got := d1(t, open)
|
|
||||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
|
||||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
|
||||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pushed: the secret is made and D1 says nothing.
|
|
||||||
pushedBy(t, open, "laptop")
|
|
||||||
if got := d1(t, open); len(got) != 0 {
|
|
||||||
t.Fatalf("a pushed machine still raised %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aTalker())
|
|
||||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got := d1(t, open)
|
|
||||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
|
||||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
|
||||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
|
||||||
// uncomposable for that — the stand-in hides nothing.
|
|
||||||
register(t, open, aKeeper())
|
|
||||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
one, two := rivals()
|
|
||||||
register(t, open, one)
|
|
||||||
register(t, open, two)
|
|
||||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
|
||||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
|
||||||
keys := map[string]conditions.Severity{}
|
|
||||||
for _, o := range d1(t, open) {
|
|
||||||
keys[o.Key()] = o.Severity
|
|
||||||
}
|
|
||||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
|
||||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aKeeper())
|
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got := d1(t, open)
|
|
||||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
|
||||||
t.Fatalf("a given secret under an old key: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
register(t, open, aKeeper())
|
|
||||||
pushedBy(t, open, "laptop")
|
|
||||||
sent := time.Now()
|
|
||||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
|
||||||
if err != nil || since.After(sent) {
|
|
||||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
|
||||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
|
||||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
// What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only
|
|
||||||
// when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown.
|
|
||||||
if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sends, err := readDeliveries(ctx, open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if sends["laptop"].settled("keeper", time.Now()) {
|
|
||||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
|
||||||
}
|
|
||||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
|
||||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
|
||||||
}
|
|
||||||
pushedBy(t, open, "laptop")
|
|
||||||
sends, err = readDeliveries(ctx, open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if sends["laptop"].settled("keeper", time.Now()) {
|
|
||||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
|
||||||
}
|
|
||||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
|
||||||
t.Fatal("pushed past the grace, a holder is not expected")
|
|
||||||
}
|
|
||||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
|
||||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
|
||||||
}
|
|
||||||
pushedAndApplied(t, open, "laptop")
|
|
||||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
|
||||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
|
||||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
|
||||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
|
||||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
|
||||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
|
||||||
|
|
||||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
|
||||||
func aKeeper() catalogue.Manifest {
|
|
||||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
|
||||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
|
||||||
Resources: []map[string]any{
|
|
||||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
|
||||||
// are made — without a bus to carry it.
|
|
||||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
|
||||||
t.Helper()
|
|
||||||
ctx := t.Context()
|
|
||||||
plan, settings, err := resolveNode(ctx, open, node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, err := declared.Body()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, err := open.inventory.NodeByName(ctx, node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
digest := digestOf(body)
|
|
||||||
// A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was
|
|
||||||
// built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109).
|
|
||||||
builds := declared.Builds
|
|
||||||
if builds == nil {
|
|
||||||
current, err := open.inventory.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
builds = map[string]string{}
|
|
||||||
for _, m := range plan.Modules {
|
|
||||||
builds[m.Module] = current[m.Module].Commit
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return digest
|
|
||||||
}
|
|
||||||
|
|
||||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
|
||||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
|
||||||
t.Helper()
|
|
||||||
digest := pushedBy(t, open, node)
|
|
||||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
|
||||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// d1 runs D1 once.
|
|
||||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
|
||||||
t.Helper()
|
|
||||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
|
||||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
|
||||||
|
|
||||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
|
||||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
|
||||||
|
|
||||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
|
||||||
// refused on it, so D1 is too.
|
|
||||||
|
|
||||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
|
||||||
|
|
||||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
|
||||||
// it and had time to report — never between assign and push.
|
|
||||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
|
||||||
now := time.Now()
|
|
||||||
sent := now.Add(-time.Minute)
|
|
||||||
long := now.Add(-time.Hour)
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
send lastSend
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"never sent", lastSend{}, false},
|
|
||||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
|
||||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
|
||||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
|
||||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
|
||||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
|
||||||
}
|
|
||||||
for _, c := range cases {
|
|
||||||
if got := c.send.settled("keeper", now); got != c.want {
|
|
||||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
|
||||||
// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send
|
|
||||||
// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent
|
|
||||||
// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so.
|
|
||||||
|
|
||||||
// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine
|
|
||||||
// is away once presence says so). A machine
|
|
||||||
// never heard from has not joined and is not away.
|
|
||||||
func awayOf(machines []machineFacts, now time.Time) map[string]bool {
|
|
||||||
out := map[string]bool{}
|
|
||||||
for _, m := range machines {
|
|
||||||
if m.silent(now) {
|
|
||||||
out[m.name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and
|
|
||||||
// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did
|
|
||||||
// before: a machine is not passed on a reading that was not made.
|
|
||||||
var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) {
|
|
||||||
nodes, err := open.inventory.Nodes(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
address, err := broker.BusAddress()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
js, err := broker.Dial(address)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
machines := make([]machineFacts, len(nodes))
|
|
||||||
for i, n := range nodes {
|
|
||||||
machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen}
|
|
||||||
}
|
|
||||||
if err := withPresence(ctx, js.Conn(), machines, now); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return awayOf(machines, now), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// rememberAway keeps on the plan the machines of running that are away: sorted, each once.
|
|
||||||
func rememberAway(p *inventory.Plan, running []string, away map[string]bool) {
|
|
||||||
for _, n := range running {
|
|
||||||
if away[n] && !slices.Contains(p.Away, n) {
|
|
||||||
p.Away = append(p.Away, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(p.Away)
|
|
||||||
}
|
|
||||||
|
|
||||||
// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and
|
|
||||||
// those it did not pass as away.
|
|
||||||
func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string {
|
|
||||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool {
|
|
||||||
if !slices.Contains(p.Away, n) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_, sent := s.Rest[n]
|
|
||||||
return !sent
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// awaySaying is what a plan says of the machines it passed as away, empty when it passed none.
|
|
||||||
func awaySaying(p inventory.Plan) string {
|
|
||||||
var said []string
|
|
||||||
for _, n := range p.Away {
|
|
||||||
said = append(said, n+" was away: sent when it is heard again")
|
|
||||||
}
|
|
||||||
return strings.Join(said, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot.
|
|
||||||
func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool {
|
|
||||||
away, err := judgeAway(ctx, open, now)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err)
|
|
||||||
return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above
|
|
||||||
}
|
|
||||||
return away
|
|
||||||
}
|
|
||||||
@@ -1,229 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it
|
|
||||||
// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard
|
|
||||||
// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's
|
|
||||||
// builds, so the send it gets when it is heard again carries them (AC3).
|
|
||||||
func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
|
|
||||||
// laptop is away as every tier is sent; anchor is heard.
|
|
||||||
wasAway := judgeAway
|
|
||||||
judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) {
|
|
||||||
return map[string]bool{"laptop": true}, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { judgeAway = wasAway })
|
|
||||||
|
|
||||||
js, err := broker.Dial(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
if err := broker.AssertMeshStreams(js); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := js.EnsureControllerBuckets(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
legacy, err := js.Conn().JetStream()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
|
||||||
t.Cleanup(func() { checkEvents = nil })
|
|
||||||
sentHeard := make(chan *nats.Msg, 32)
|
|
||||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
|
||||||
|
|
||||||
wasSaved := inventory.PlanSaved
|
|
||||||
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
|
||||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
|
||||||
|
|
||||||
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
|
||||||
var mu sync.Mutex
|
|
||||||
var asks []string
|
|
||||||
var sends [][]string
|
|
||||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
|
||||||
wasAsk := askABuild
|
|
||||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
|
|
||||||
asks = append(asks, source.Repository+"#"+id)
|
|
||||||
return id, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = wasAsk })
|
|
||||||
digests := 0
|
|
||||||
wasSend := sendRollout
|
|
||||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
sends = append(sends, append([]string(nil), names...))
|
|
||||||
current, err := open.inventory.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
builds := map[string]string{}
|
|
||||||
for _, m := range []string{"a", "b"} {
|
|
||||||
builds[m] = current[m].Commit
|
|
||||||
}
|
|
||||||
for _, node := range names {
|
|
||||||
digests++
|
|
||||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { sendRollout = wasSend })
|
|
||||||
|
|
||||||
register := func(m, commit string, asked time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
|
||||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
|
||||||
Asked: asked}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
record := func(id, m, commit string, at time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
|
||||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
|
||||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
old := time.Now().Add(-2 * time.Hour)
|
|
||||||
for _, m := range []string{"a", "b"} {
|
|
||||||
record("build-"+m+"-1", m, "c1", old)
|
|
||||||
register(m, "c1", old)
|
|
||||||
for _, n := range []string{"anchor", "laptop"} {
|
|
||||||
if _, err := inv.Assign(ctx, n, m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, n := range []string{"anchor", "laptop"} {
|
|
||||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
|
||||||
register("a", "c2", time.Now().Add(-time.Minute))
|
|
||||||
|
|
||||||
// A machine's report on what it was sent, heard as the controller hears it.
|
|
||||||
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
|
||||||
seq := int64(0)
|
|
||||||
reports := func(node string) {
|
|
||||||
t.Helper()
|
|
||||||
last, found, err := inv.SentTo(ctx, node)
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("%s was sent nothing: %v", node, err)
|
|
||||||
}
|
|
||||||
seq++
|
|
||||||
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
|
||||||
"report_sequence": seq})
|
|
||||||
var report link.Report
|
|
||||||
if err := json.Unmarshal(body, &report); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := listener.Heard(ctx, report); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
plan := func() inventory.Plan {
|
|
||||||
t.Helper()
|
|
||||||
p, err := inv.PlanByID(ctx, "plan-1160")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
built := time.Now().UTC()
|
|
||||||
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
|
||||||
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
|
|
||||||
Modules: map[string]*inventory.PlanModule{
|
|
||||||
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
|
||||||
"b": {},
|
|
||||||
}}
|
|
||||||
first, stopFirst := context.WithCancel(ctx)
|
|
||||||
defer stopFirst()
|
|
||||||
waitFirst := startRolloutEvents(first, open, js)
|
|
||||||
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
|
||||||
t.Cleanup(func() { stopFirst(); waitFirst() })
|
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
eventually(t, "tier 0 sent to anchor alone", func() bool {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor"
|
|
||||||
})
|
|
||||||
reports("anchor")
|
|
||||||
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
|
|
||||||
var moved string
|
|
||||||
eventually(t, "tier 1 reached on anchor's report alone", func() bool {
|
|
||||||
if q := plan(); q.Tier == 1 {
|
|
||||||
moved = q.Note
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
})
|
|
||||||
if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
|
|
||||||
t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
|
|
||||||
}
|
|
||||||
// The ask for tier 1's build is the next step, and records its build id on the module.
|
|
||||||
eventually(t, "tier 1 asked", func() bool { m := plan().Modules["b"]; return m != nil && m.Build != "" })
|
|
||||||
b := plan().Modules["b"]
|
|
||||||
askedAt, _ := link.BuildAskedAt(b.Build)
|
|
||||||
record(b.Build, "b", "c2", askedAt)
|
|
||||||
register("b", "c2", askedAt)
|
|
||||||
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
|
|
||||||
eventually(t, "tier 1 sent to anchor alone", func() bool {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
|
|
||||||
})
|
|
||||||
// 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again.
|
|
||||||
was, known, err := inv.SentBuilds(ctx, "laptop")
|
|
||||||
if err != nil || !known {
|
|
||||||
t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known)
|
|
||||||
}
|
|
||||||
current, err := inv.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if was["a"] == current["a"].Commit || was["a"] != "c1" {
|
|
||||||
t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit)
|
|
||||||
}
|
|
||||||
if got := plan().Away; strings.Join(got, ",") != "laptop" {
|
|
||||||
t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
|
||||||
// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so.
|
|
||||||
|
|
||||||
// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away.
|
|
||||||
func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) {
|
|
||||||
now := time.Now().UTC()
|
|
||||||
machines := []machineFacts{
|
|
||||||
{name: "heard", lastHeard: now.Add(-time.Minute)},
|
|
||||||
{name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)},
|
|
||||||
{name: "stuck", lastHeard: now, presence: presenceSilent},
|
|
||||||
{name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard},
|
|
||||||
{name: "new"},
|
|
||||||
}
|
|
||||||
got := awayOf(machines, now)
|
|
||||||
for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false,
|
|
||||||
"new": false} {
|
|
||||||
if got[node] != want {
|
|
||||||
t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once.
|
|
||||||
func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) {
|
|
||||||
p := &inventory.Plan{}
|
|
||||||
away := map[string]bool{"g14": true, "ace": true}
|
|
||||||
rememberAway(p, []string{"anchor", "g14"}, away)
|
|
||||||
rememberAway(p, []string{"g14", "ace"}, away)
|
|
||||||
if !slices.Equal(p.Away, []string{"ace", "g14"}) {
|
|
||||||
t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1161/AC2: the plan says it, where its other passed machines are said.
|
|
||||||
func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) {
|
|
||||||
say := awaySaying(inventory.Plan{Away: []string{"g14"}})
|
|
||||||
if !strings.Contains(say, "g14 was away: sent when it is heard again") {
|
|
||||||
t.Fatalf("1161/AC2: the plan says %q", say)
|
|
||||||
}
|
|
||||||
if got := awaySaying(inventory.Plan{}); got != "" {
|
|
||||||
t.Fatalf("1161/AC2: a plan that passed no machine says %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for.
|
|
||||||
func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) {
|
|
||||||
p := inventory.Plan{Away: []string{"g14"}}
|
|
||||||
none := &inventory.PlanModule{}
|
|
||||||
if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) {
|
|
||||||
t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got)
|
|
||||||
}
|
|
||||||
sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}}
|
|
||||||
if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) {
|
|
||||||
t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,308 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and
|
|
||||||
// answers `backed-up` with the last night it is told (novox/hq ADR 0404).
|
|
||||||
type aBackupSeatOnTheBus struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
nows []string
|
|
||||||
night map[string]any // module → lastNight
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) {
|
|
||||||
t.Helper()
|
|
||||||
reply := func(m *nats.Msg, result any) {
|
|
||||||
body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node})
|
|
||||||
_ = m.Respond(body)
|
|
||||||
}
|
|
||||||
now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) {
|
|
||||||
var a struct {
|
|
||||||
Module string `json:"module"`
|
|
||||||
}
|
|
||||||
_ = json.Unmarshal(m.Data, &a)
|
|
||||||
f.mu.Lock()
|
|
||||||
f.nows = append(f.nows, a.Module)
|
|
||||||
f.mu.Unlock()
|
|
||||||
reply(m, map[string]any{"started": a.Module})
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
var out []map[string]any
|
|
||||||
for module, n := range f.night {
|
|
||||||
out = append(out, map[string]any{"module": module, "lastNight": n})
|
|
||||||
}
|
|
||||||
reply(m, out)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() })
|
|
||||||
if err := conn.Flush(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *aBackupSeatOnTheBus) asks() int {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
return len(f.nows)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits
|
|
||||||
// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the
|
|
||||||
// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the
|
|
||||||
// backups again, and the plan goes on. One bus and one store for the scenarios.
|
|
||||||
func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
js, err := broker.Dial(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
before := handActConn
|
|
||||||
handActConn = js.Conn()
|
|
||||||
t.Cleanup(func() { handActConn = before })
|
|
||||||
|
|
||||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
|
||||||
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
|
|
||||||
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
|
|
||||||
`"resources":[{"id":"files","type":"directory"}]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
|
||||||
Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, node := range []string{"anchor", "laptop"} {
|
|
||||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := assign(ctx, open, node, "notes"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}}
|
|
||||||
for node, s := range seats {
|
|
||||||
s.serve(t, js.Conn(), node)
|
|
||||||
}
|
|
||||||
// Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup.
|
|
||||||
wasWithin, wasEvery := backupsWithin, backupsPollEvery
|
|
||||||
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
|
|
||||||
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
|
|
||||||
|
|
||||||
newPlan := func(id string) {
|
|
||||||
t.Helper()
|
|
||||||
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}}
|
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
read := func(id string) inventory.Plan {
|
|
||||||
t.Helper()
|
|
||||||
p, err := inv.PlanByID(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Both machines are asked; while one has not answered, no build is asked.
|
|
||||||
newPlan("plan-one")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 {
|
|
||||||
t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked)
|
|
||||||
}
|
|
||||||
seats["anchor"].answers("notes", true, "")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending {
|
|
||||||
t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups)
|
|
||||||
}
|
|
||||||
seats["laptop"].answers("notes", true, "")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone {
|
|
||||||
t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups)
|
|
||||||
}
|
|
||||||
finish := read("plan-one")
|
|
||||||
finish.State = inventory.PlanDone
|
|
||||||
if err := inv.SavePlan(ctx, &finish); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised.
|
|
||||||
seats["anchor"].night, seats["laptop"].night = nil, nil
|
|
||||||
newPlan("plan-two")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
seats["anchor"].answers("notes", true, "")
|
|
||||||
seats["laptop"].answers("notes", false, "no space left on device")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
held := read("plan-two")
|
|
||||||
if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 {
|
|
||||||
t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked)
|
|
||||||
}
|
|
||||||
var failed *inventory.PlanBackup
|
|
||||||
for i := range held.Backups.Asked {
|
|
||||||
if held.Backups.Asked[i].State == inventory.BackupFailed {
|
|
||||||
failed = &held.Backups.Asked[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" ||
|
|
||||||
strings.Contains(held.Note, "tried again") {
|
|
||||||
t.Fatalf("the failed backup: %+v note=%q", failed, held.Note)
|
|
||||||
}
|
|
||||||
advancePlans(ctx, open) // not retried on the next look
|
|
||||||
if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 ||
|
|
||||||
seats["laptop"].asks() != 2 {
|
|
||||||
t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks())
|
|
||||||
}
|
|
||||||
said, err := probeBackupHeld(ctx, &doctor{open: open})
|
|
||||||
if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") {
|
|
||||||
t.Fatalf("the condition: %+v %v", said, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on.
|
|
||||||
seats["laptop"].answers("notes", false, "no space left on device")
|
|
||||||
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by",
|
|
||||||
"mesh-delivery for jochen"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil {
|
|
||||||
t.Fatal("a plan with no failed backup was asked again")
|
|
||||||
}
|
|
||||||
seats["laptop"].night = nil
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 {
|
|
||||||
t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks())
|
|
||||||
}
|
|
||||||
seats["laptop"].answers("notes", true, "")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 2 {
|
|
||||||
t.Fatalf("the plan did not go on: %+v builds=%v", got.Backups, *asked)
|
|
||||||
}
|
|
||||||
if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 {
|
|
||||||
t.Fatalf("the condition stays: %+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Issue 1086: a plan cut from a merge asked its first tier at the cut, so a module keeping data was sent without the
|
|
||||||
// backup the plan promised. The cut now takes the backups first (askFirstTierAtCut, batches.go), and the walk's own
|
|
||||||
// step asks the tier once they are done.
|
|
||||||
func TestAPlanCutFromAMergeOfABackedUpModuleAsksItsBackupAndNotItsTierUntilItIsDone(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
settings := t.TempDir() + "/merge-window.json"
|
|
||||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv(mergeWindowFileVar, settings)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
js, err := broker.Dial(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
before := handActConn
|
|
||||||
handActConn = js.Conn()
|
|
||||||
t.Cleanup(func() { handActConn = before })
|
|
||||||
|
|
||||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
|
|
||||||
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
|
|
||||||
`"resources":[{"id":"files","type":"directory"}]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/notes", Seat: "git", Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, m := range []string{"keeper", "notes"} {
|
|
||||||
if _, err := assign(ctx, open, "anchor", m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
seat := &aBackupSeatOnTheBus{}
|
|
||||||
seat.serve(t, js.Conn(), "anchor")
|
|
||||||
wasWithin, wasEvery := backupsWithin, backupsPollEvery
|
|
||||||
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
|
|
||||||
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
|
|
||||||
|
|
||||||
// 1. A merge of notes is heard and its window closes: the batch is cut through cutBatch.
|
|
||||||
hear(t, open, repoMerge("notes", "c-notes", t0), t0.Add(time.Second))
|
|
||||||
cutAt(t, open, t0.Add(time.Second+90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("%d walk(s)", len(ws))
|
|
||||||
}
|
|
||||||
// 2. Right after the cut no tier is asked, the record has anchor/notes pending, the seat heard one `now`.
|
|
||||||
cut, err := inv.PlanByID(ctx, ws[0].ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(*asked) != 0 || askedInTier(cut) {
|
|
||||||
t.Fatalf("the first tier was asked at the cut: %v %+v", *asked, cut.Modules)
|
|
||||||
}
|
|
||||||
if cut.Backups == nil || cut.Backups.State != inventory.BackupPending || len(cut.Backups.Asked) != 1 ||
|
|
||||||
cut.Backups.Asked[0].Machine != "anchor" || cut.Backups.Asked[0].Module != "notes" {
|
|
||||||
t.Fatalf("the backups of the cut: %+v", cut.Backups)
|
|
||||||
}
|
|
||||||
if seat.asks() != 1 {
|
|
||||||
t.Fatalf("the seat heard %d `now`", seat.asks())
|
|
||||||
}
|
|
||||||
// 3. The backup answers done: the walk's own step asks the first tier, once, and asks no second backup.
|
|
||||||
seat.answers("notes", true, "")
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
after, err := inv.PlanByID(ctx, ws[0].ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(*asked) != 1 || after.Backups.State != inventory.BackupDone || seat.asks() != 1 {
|
|
||||||
t.Fatalf("builds=%v backups=%+v asks=%d", *asked, after.Backups, seat.asks())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,698 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
func windowed(t *testing.T) *stores {
|
|
||||||
t.Helper()
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
settings := t.TempDir() + "/merge-window.json"
|
|
||||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv(mergeWindowFileVar, settings)
|
|
||||||
noGroups(t)
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
|
||||||
Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, m := range []struct{ module, repository, path string }{
|
|
||||||
{"app", "novox/mesh-catalog", "modules/app"},
|
|
||||||
{"notes", "novox/mesh-catalog", "modules/notes"},
|
|
||||||
{"one", "novox/one", ""},
|
|
||||||
{"two", "novox/two", ""},
|
|
||||||
{"three", "novox/three", ""},
|
|
||||||
} {
|
|
||||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
|
||||||
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
|
||||||
Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return open
|
|
||||||
}
|
|
||||||
|
|
||||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func cutAt(t *testing.T, open *stores, now time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksWithPaths(t)
|
|
||||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
|
||||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
|
||||||
hear(t, open, claude, t0.Add(time.Second))
|
|
||||||
hear(t, open, dunst, t0.Add(18*time.Second))
|
|
||||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
|
||||||
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
|
||||||
t.Fatalf("cut before the window closed: %+v", ws)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 {
|
|
||||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
w := ws[0]
|
|
||||||
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
|
||||||
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
|
||||||
}
|
|
||||||
for _, m := range []string{"app", "notes"} {
|
|
||||||
if _, in := w.Modules[m]; !in {
|
|
||||||
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
|
||||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
|
||||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
|
||||||
}
|
|
||||||
if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
|
|
||||||
t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
|
|
||||||
}
|
|
||||||
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
|
||||||
hear(t, open, claude, t0.Add(5*time.Minute))
|
|
||||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
|
||||||
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/three")
|
|
||||||
asksRecorded(t)
|
|
||||||
for i, r := range []string{"one", "two", "three"} {
|
|
||||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
|
||||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 {
|
|
||||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
w := ws[0]
|
|
||||||
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
|
||||||
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
|
||||||
}
|
|
||||||
for _, r := range []string{"one", "two", "three"} {
|
|
||||||
if w.CommitOf("novox/"+r) != "c-"+r {
|
|
||||||
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
|
||||||
}
|
|
||||||
if _, in := w.Modules[r]; !in {
|
|
||||||
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Each module's ask is recorded at its own repository's commit.
|
|
||||||
for _, r := range []string{"one", "two", "three"} {
|
|
||||||
q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
|
|
||||||
if err != nil || !found || q.Commit != "c-"+r {
|
|
||||||
t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/both", "feat/busy")
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
|
||||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(100*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
|
||||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(170*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
|
||||||
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
|
||||||
}
|
|
||||||
|
|
||||||
busy := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/both", "feat/busy")
|
|
||||||
asksWithPaths(t)
|
|
||||||
var last time.Time
|
|
||||||
for i := 0; i < 12; i++ {
|
|
||||||
last = t0.Add(time.Duration(i) * time.Minute)
|
|
||||||
repo := []string{"one", "two", "three"}[i%3]
|
|
||||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
|
||||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
|
||||||
if i != 10 {
|
|
||||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ws, _ := walks(t, busy)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "pr/553b7191claude")
|
|
||||||
asksWithPaths(t)
|
|
||||||
var said []inventory.Plan
|
|
||||||
was := inventory.PlanSaved
|
|
||||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
|
||||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
|
||||||
now := time.Now().UTC()
|
|
||||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
|
||||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
|
||||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
|
||||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
|
||||||
hear(t, open, claude, now.Add(-19*time.Second))
|
|
||||||
hear(t, open, dunst, now.Add(-time.Second))
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
|
||||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
|
||||||
lines := strings.Split(out, "\n")
|
|
||||||
first := lines[0]
|
|
||||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
|
||||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
|
||||||
if !strings.Contains(first, want) {
|
|
||||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
|
||||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
|
||||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
|
||||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
|
||||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
|
||||||
}
|
|
||||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
|
||||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
|
||||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func captured(t *testing.T, run func() error) string {
|
|
||||||
t.Helper()
|
|
||||||
r, w, err := os.Pipe()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
was := os.Stdout
|
|
||||||
os.Stdout = w
|
|
||||||
runErr := run()
|
|
||||||
os.Stdout = was
|
|
||||||
_ = w.Close()
|
|
||||||
var b bytes.Buffer
|
|
||||||
_, _ = io.Copy(&b, r)
|
|
||||||
if runErr != nil {
|
|
||||||
t.Fatal(runErr)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || !ws[0].Open() {
|
|
||||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
|
||||||
}
|
|
||||||
first := ws[0]
|
|
||||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
|
||||||
cutAt(t, open, t0.Add(5*time.Minute))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
|
||||||
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
|
||||||
t.Fatalf("the started walk was %s by the next merge", got.State)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
|
||||||
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
|
||||||
}
|
|
||||||
first.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(6*time.Minute))
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
|
||||||
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || !ws[0].Waiting() {
|
|
||||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
|
||||||
}
|
|
||||||
waiting := ws[0]
|
|
||||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(bs) != 0 || len(ws) != 2 {
|
|
||||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
newer := ws[0]
|
|
||||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
|
||||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
|
||||||
}
|
|
||||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" {
|
|
||||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
|
||||||
}
|
|
||||||
if _, in := newer.Modules["one"]; !in {
|
|
||||||
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
|
||||||
hear(t, open, later, t0.Add(18*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
w := ws[0]
|
|
||||||
w.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
|
||||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
|
||||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
|
||||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
|
||||||
Carried: later.Commit}) {
|
|
||||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
// Moves app, which that walk never built: the next batch walks it.
|
|
||||||
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
|
||||||
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
|
||||||
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
|
||||||
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
|
||||||
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
|
||||||
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
|
||||||
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
failed := ws[0]
|
|
||||||
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
|
||||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(3*time.Minute))
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
|
||||||
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
|
||||||
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
alone := ws[0]
|
|
||||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
|
||||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
|
||||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
|
||||||
}
|
|
||||||
if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
|
|
||||||
middle.Commit} {
|
|
||||||
t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
|
|
||||||
}
|
|
||||||
// Retried, the failed walk would name merges the search answers now.
|
|
||||||
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
|
||||||
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
|
||||||
}
|
|
||||||
alone.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(4*time.Minute))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if ws[0].ID != alone.ID || len(bs) != 0 {
|
|
||||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
|
||||||
}
|
|
||||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
|
||||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
|
||||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
|
||||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
|
||||||
}
|
|
||||||
// A stopped walk starts no search: a person ended it.
|
|
||||||
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
|
||||||
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
|
||||||
cutAt(t, open, t0.Add(8*time.Minute))
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(9*time.Minute))
|
|
||||||
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
|
||||||
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
|
||||||
for _, grouped := range []bool{true, false} {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
for _, m := range []string{"mesh-host"} {
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
host := repoMerge("mesh-host", "h1", t0)
|
|
||||||
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
|
||||||
host.Head, ctl.Head = "feat/together", "feat/together"
|
|
||||||
if !grouped {
|
|
||||||
ctl.Head = "feat/alone"
|
|
||||||
}
|
|
||||||
hear(t, open, ctl, t0.Add(2*time.Second))
|
|
||||||
hear(t, open, host, t0.Add(3*time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("%d walks", len(ws))
|
|
||||||
}
|
|
||||||
tiers := ws[0].Tiers
|
|
||||||
if grouped {
|
|
||||||
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
|
||||||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
|
||||||
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
|
||||||
}
|
|
||||||
} else if len(tiers) != 1 {
|
|
||||||
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABatchSurvivesARestart(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
groupsWaiting(t, "feat/both")
|
|
||||||
asksWithPaths(t)
|
|
||||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
|
||||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
|
||||||
|
|
||||||
// A new controller: nothing of the first one's but the store.
|
|
||||||
again, err := openStores(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(again.Close)
|
|
||||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
|
||||||
cutAt(t, again, t0.Add(119*time.Second))
|
|
||||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
cutAt(t, again, t0.Add(120*time.Second))
|
|
||||||
ws, bs := walks(t, again)
|
|
||||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
|
||||||
hear(t, open, c2, t0)
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
w := ws[0]
|
|
||||||
w.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
|
||||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
|
||||||
Commit: "c1", Carried: "c2"}) {
|
|
||||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk of two that built only two: a late merge of one is not its to name.
|
|
||||||
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
|
||||||
cutAt(t, open, t0.Add(18*time.Minute))
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
w = ws[0]
|
|
||||||
delete(w.Modules, "two")
|
|
||||||
w.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
|
||||||
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
|
||||||
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
|
||||||
}
|
|
||||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
|
||||||
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asked := asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
started, _ := walks(t, open)
|
|
||||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0,
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}},
|
|
||||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
|
||||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
|
||||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
|
||||||
}
|
|
||||||
before := len(*asked)
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
|
||||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
|
||||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
|
||||||
}
|
|
||||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
|
||||||
later := time.Now().Add(time.Hour)
|
|
||||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
|
||||||
t.Fatalf("a deferred walk reads %q", line)
|
|
||||||
}
|
|
||||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, f := range facts {
|
|
||||||
if f.id == got.ID {
|
|
||||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
done := started[0]
|
|
||||||
done.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
if len(*asked) != before+1 {
|
|
||||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
|
||||||
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
|
||||||
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
var ownBatch, catalogueBatch inventory.Plan
|
|
||||||
for _, b := range bs {
|
|
||||||
if b.OwnPath() {
|
|
||||||
ownBatch = b
|
|
||||||
} else {
|
|
||||||
catalogueBatch = b
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
|
||||||
len(ownBatch.Delivery.Merges) != 1 {
|
|
||||||
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
|
||||||
}
|
|
||||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
|
||||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
|
||||||
}
|
|
||||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
|
||||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
|
||||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
|
||||||
}
|
|
||||||
ownWalk := ws[0]
|
|
||||||
for _, m := range []string{"app", "notes"} {
|
|
||||||
if _, in := ownWalk.Modules[m]; in {
|
|
||||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
|
||||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
|
||||||
}
|
|
||||||
ownWalk.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(3*time.Minute))
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
|
||||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
|
||||||
}
|
|
||||||
for _, m := range []string{"app", "notes"} {
|
|
||||||
if _, in := ws[0].Modules[m]; !in {
|
|
||||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, a := range *asked {
|
|
||||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
|
||||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
|
||||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
|
||||||
catalogueWalk := ws[0]
|
|
||||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
|
||||||
cutAt(t, open, t0.Add(6*time.Minute))
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
|
||||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
|
||||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
|
||||||
}
|
|
||||||
if _, in := ws[0].Modules["app"]; in {
|
|
||||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
|
||||||
}
|
|
||||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
before := len(*asked)
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
if len(*asked) != before {
|
|
||||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
|
||||||
}
|
|
||||||
own2 := ws[0]
|
|
||||||
own2.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
|
||||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
for _, m := range []struct {
|
|
||||||
module string
|
|
||||||
claims []catalogue.Claim
|
|
||||||
}{
|
|
||||||
{"nats", nil},
|
|
||||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
|
||||||
} {
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
|
||||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
|
||||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
|
||||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
|
||||||
ModuleDirsSaid: true}
|
|
||||||
hear(t, open, mixed, t0.Add(time.Second))
|
|
||||||
cutAt(t, open, t0.Add(2*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 || ws[0].Waiting() {
|
|
||||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
|
||||||
}
|
|
||||||
done := ws[0]
|
|
||||||
done.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
|
||||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
|
||||||
_, bs := walks(t, open)
|
|
||||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
|
||||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
|
||||||
got.Delivery.Merges, bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,293 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
|
|
||||||
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
|
|
||||||
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
|
|
||||||
func init() {
|
|
||||||
mergeWindowDefault = 0
|
|
||||||
// No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change
|
|
||||||
// (delivery_edges.go). A test that wants some replaces these (forgeHolding).
|
|
||||||
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
|
|
||||||
return catalogue.Manifest{}, false, nil
|
|
||||||
}
|
|
||||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
|
||||||
}
|
|
||||||
|
|
||||||
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
|
|
||||||
// settings file says them, and
|
|
||||||
// modules built from the catalogue (app, notes) and from three repositories of their own.
|
|
||||||
|
|
||||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
|
||||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
|
||||||
|
|
||||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull
|
|
||||||
// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group
|
|
||||||
// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test
|
|
||||||
// whose merges wait for the window as a group's says so (groupsWaiting).
|
|
||||||
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
|
||||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit,
|
|
||||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
|
||||||
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge).
|
|
||||||
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
|
||||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit,
|
|
||||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// noGroups stubs mesh-delivery's word on groups for a test: no head names a group.
|
|
||||||
func noGroups(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil }
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named
|
|
||||||
// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and
|
|
||||||
// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds
|
|
||||||
// its batch open however long it takes (ADR 0418 clause 3), is groupStates'.
|
|
||||||
func groupsWaiting(t *testing.T, heads ...string) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
|
|
||||||
if slices.Contains(heads, head) {
|
|
||||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
|
||||||
}
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
|
|
||||||
// before novox/hq ADR 0304: mesh-delivery cannot say any head's group.
|
|
||||||
func everyHeadWaits(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
|
||||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the
|
|
||||||
// members merge.
|
|
||||||
func groupStates(t *testing.T, head string, states map[string]string) {
|
|
||||||
t.Helper()
|
|
||||||
groupStatesFrom(t, head, states, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the
|
|
||||||
// composed check's: only the first holds the group's batch open for its last member.
|
|
||||||
func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) {
|
|
||||||
t.Helper()
|
|
||||||
was := deliveryGroupOf
|
|
||||||
deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) {
|
|
||||||
if asked != head {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
var members []groupMember
|
|
||||||
for repository, state := range states {
|
|
||||||
members = append(members, groupMember{Repository: repository, State: state, Composed: composed})
|
|
||||||
}
|
|
||||||
return members, true, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { deliveryGroupOf = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group,
|
|
||||||
// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share.
|
|
||||||
func fromHead(m link.SourceMoved, head string) link.SourceMoved {
|
|
||||||
m.Head = head
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is
|
|
||||||
// related to a batch assembling (ADR 0304 decision 1).
|
|
||||||
func lone(m link.SourceMoved) link.SourceMoved {
|
|
||||||
m.Head = ""
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// hear is a merge heard at a moment.
|
|
||||||
|
|
||||||
// cutAt is the cutter looking at a moment.
|
|
||||||
|
|
||||||
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
|
|
||||||
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
|
||||||
t.Helper()
|
|
||||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, p := range recent {
|
|
||||||
if p.Batch() {
|
|
||||||
batches = append(batches, p)
|
|
||||||
} else {
|
|
||||||
walks = append(walks, p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return walks, batches
|
|
||||||
}
|
|
||||||
|
|
||||||
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
|
|
||||||
// later commit, which names both merges — the earlier carried by the later.
|
|
||||||
|
|
||||||
// Merges of three repositories in one window are one walk, one commit for each.
|
|
||||||
|
|
||||||
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
|
|
||||||
// closes at most ten minutes after its first merge however busy.
|
|
||||||
|
|
||||||
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
|
|
||||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
|
||||||
|
|
||||||
// captured is what a command printed.
|
|
||||||
|
|
||||||
// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is
|
|
||||||
// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4).
|
|
||||||
|
|
||||||
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
|
|
||||||
// names the walk that took it over.
|
|
||||||
|
|
||||||
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
|
|
||||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
|
||||||
|
|
||||||
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
|
|
||||||
// first; the first delivered answers the older ones, and the search stops.
|
|
||||||
|
|
||||||
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
|
|
||||||
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
|
|
||||||
|
|
||||||
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
|
|
||||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
|
||||||
|
|
||||||
// The window's settings reach the controller in its settings file, read at every look; one that says no
|
|
||||||
// duration, or no file, is the default.
|
|
||||||
func TestTheWindowIsTheControllersSetting(t *testing.T) {
|
|
||||||
file := t.TempDir() + "/merge-window.json"
|
|
||||||
t.Setenv(mergeWindowFileVar, file)
|
|
||||||
for _, c := range []struct {
|
|
||||||
content string
|
|
||||||
window, atMost time.Duration
|
|
||||||
}{
|
|
||||||
{`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute},
|
|
||||||
{`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault},
|
|
||||||
{`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault},
|
|
||||||
} {
|
|
||||||
if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if w, m := mergeWindowOf(); w != c.window || m != c.atMost {
|
|
||||||
t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Setenv(mergeWindowFileVar, file+".gone")
|
|
||||||
if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault {
|
|
||||||
t.Errorf("no file reads as %s and %s", w, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362).
|
|
||||||
func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
|
||||||
f := calm(t0)
|
|
||||||
f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery",
|
|
||||||
since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"},
|
|
||||||
{Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}}
|
|
||||||
got := watchWaits(f)
|
|
||||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") ||
|
|
||||||
!strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") {
|
|
||||||
t.Fatalf("the waiting walk does not name both merges: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
|
|
||||||
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
|
|
||||||
// nothing, and the merge joins the next batch.
|
|
||||||
|
|
||||||
// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to
|
|
||||||
// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before.
|
|
||||||
|
|
||||||
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
|
|
||||||
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
|
||||||
ev := func(commit string, paths, removed []string) inventory.BatchedMerge {
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed}
|
|
||||||
b, _ := json.Marshal(m)
|
|
||||||
return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b,
|
|
||||||
Merged: t0.Add(time.Duration(len(commit)) * time.Second)}
|
|
||||||
}
|
|
||||||
got := combinedMerges([]inventory.BatchedMerge{
|
|
||||||
ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}),
|
|
||||||
ev("bb", []string{"x/module.json"}, nil),
|
|
||||||
ev("ccc", []string{"z.go"}, nil)})
|
|
||||||
if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) ||
|
|
||||||
len(got[0].Paths) != 3 {
|
|
||||||
t.Fatalf("combined as %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
|
||||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
|
||||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
|
||||||
// still waits for the word, so no catalogue delivery skips its turn.
|
|
||||||
|
|
||||||
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
|
||||||
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
|
||||||
// request is named by its commit, and a title is cut at fifty runes.
|
|
||||||
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
|
|
||||||
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
|
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
|
|
||||||
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
|
|
||||||
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
|
|
||||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
|
|
||||||
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
|
|
||||||
running := func(module string) []string {
|
|
||||||
if module == "messenger" || module == "telegram" {
|
|
||||||
return []string{"novox"}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if got, want := planLineOn(p, now, pauseView{}, tierAtLeast, running),
|
|
||||||
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
|
|
||||||
t.Fatalf("the line reads %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
|
|
||||||
t.Fatalf("without the machines the line reads %q", got)
|
|
||||||
}
|
|
||||||
old := p
|
|
||||||
old.Commits, old.Delivery = nil, nil
|
|
||||||
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
|
|
||||||
t.Fatalf("a record naming no pull request reads %q", got)
|
|
||||||
}
|
|
||||||
long := p
|
|
||||||
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
|
|
||||||
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
|
|
||||||
t.Fatalf("a long title is not cut at fifty runes: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
|
|
||||||
func sameMerge(a, b inventory.PlanMerge) bool {
|
|
||||||
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
|
|
||||||
}
|
|
||||||
|
|
||||||
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
|
||||||
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
|
||||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
|
||||||
@@ -1,153 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
|
|
||||||
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
|
|
||||||
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
|
|
||||||
|
|
||||||
// The kinds S18 and S19 raise.
|
|
||||||
const (
|
|
||||||
kindBatchNotCut = "batch-not-cut"
|
|
||||||
kindBatchBehindWalk = "batch-behind-walk"
|
|
||||||
// kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after
|
|
||||||
// its first (novox/hq ADR 0418 clause 3).
|
|
||||||
kindGroupNotMerged = "group-not-merged"
|
|
||||||
)
|
|
||||||
|
|
||||||
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
|
||||||
type batchFacts struct {
|
|
||||||
id, state, grouped string
|
|
||||||
// atMost is when its window closes at the latest; closed when it closed.
|
|
||||||
atMost, closed time.Time
|
|
||||||
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
|
|
||||||
// tiers.
|
|
||||||
behind string
|
|
||||||
walkBound time.Duration
|
|
||||||
// holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first
|
|
||||||
// merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting).
|
|
||||||
holds map[string]string
|
|
||||||
created time.Time
|
|
||||||
atMostWindow time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
|
||||||
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
|
|
||||||
batches, err := inv.Batches(ctx)
|
|
||||||
if err != nil || len(batches) == 0 {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
bounds, err := measuredTierBounds(ctx, inv, now)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var out []batchFacts
|
|
||||||
for _, b := range batches {
|
|
||||||
_, atMostWindow := mergeWindowOf()
|
|
||||||
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow}
|
|
||||||
if w := b.Delivery; w != nil && w.Batch != nil {
|
|
||||||
f.holds = w.Batch.Holds
|
|
||||||
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
|
||||||
if f.atMost.Before(f.closed) {
|
|
||||||
f.closed = f.atMost
|
|
||||||
}
|
|
||||||
if w.Batch.CutAtOnce && !b.Created.IsZero() {
|
|
||||||
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
|
|
||||||
f.atMost, f.closed = b.Created, b.Created
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if f.behind != "" {
|
|
||||||
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
|
||||||
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out = append(out, f)
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
|
|
||||||
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, b := range f.batches {
|
|
||||||
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter ||
|
|
||||||
len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
late := f.now.Sub(b.atMost)
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
|
|
||||||
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
|
|
||||||
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
|
|
||||||
Headline: "Merged changes are not being delivered",
|
|
||||||
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
|
|
||||||
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
|
|
||||||
Resolved: "Merged changes are being delivered again"})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
|
|
||||||
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, b := range f.batches {
|
|
||||||
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
in := f.now.Sub(b.closed)
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
|
|
||||||
Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
|
|
||||||
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
|
|
||||||
b.grouped, b.behind),
|
|
||||||
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
|
|
||||||
Headline: "Merged changes wait behind a slow delivery",
|
|
||||||
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
|
|
||||||
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
|
|
||||||
Resolved: "Merged changes no longer wait behind a slow delivery"})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge
|
|
||||||
// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of
|
|
||||||
// the group is delivered while it waits.
|
|
||||||
func watchGroupsNotMerged(f *signalFacts) []conditions.Observation {
|
|
||||||
var out []conditions.Observation
|
|
||||||
for _, b := range f.batches {
|
|
||||||
if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
heads := make([]string, 0, len(b.holds))
|
|
||||||
for h := range b.holds {
|
|
||||||
heads = append(heads, h)
|
|
||||||
}
|
|
||||||
sort.Strings(heads)
|
|
||||||
var why []string
|
|
||||||
for _, h := range heads {
|
|
||||||
why = append(why, b.holds[h])
|
|
||||||
}
|
|
||||||
waited := f.now.Sub(b.created)
|
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id,
|
|
||||||
Kind: kindGroupNotMerged, Severity: conditions.Warning,
|
|
||||||
Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+
|
|
||||||
"of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id),
|
|
||||||
Said: "a group waits for a member that has not merged",
|
|
||||||
Headline: "A change waits for the rest of its group to merge",
|
|
||||||
Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+
|
|
||||||
"One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.",
|
|
||||||
humanDuration(waited)),
|
|
||||||
Resolved: "The group's last change merged"})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,108 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once
|
|
||||||
// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery.
|
|
||||||
func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() {
|
|
||||||
t.Fatalf("the first walk is not open: %+v", ws)
|
|
||||||
}
|
|
||||||
heard := t0.Add(100 * time.Second)
|
|
||||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
|
||||||
advanceHeld(ctx, open)
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 0 {
|
|
||||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
for _, w := range ws {
|
|
||||||
if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) {
|
|
||||||
t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(),
|
|
||||||
deferred(w), askedAny(w), w.Note)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is
|
|
||||||
// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973).
|
|
||||||
func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }()
|
|
||||||
heard := t0.Add(100 * time.Second)
|
|
||||||
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
|
||||||
hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second))
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 1 {
|
|
||||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs))
|
|
||||||
}
|
|
||||||
b := bs[0]
|
|
||||||
words := batchWords(b, heard)
|
|
||||||
if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" ||
|
|
||||||
!strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") {
|
|
||||||
t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words)
|
|
||||||
}
|
|
||||||
if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() {
|
|
||||||
t.Fatalf("the started walk was %s by the related merge", got.State)
|
|
||||||
}
|
|
||||||
first.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, heard.Add(2*time.Second))
|
|
||||||
if _, bs := walks(t, open); len(bs) != 0 {
|
|
||||||
t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever
|
|
||||||
// the two share (issue 973): it replaces the controller, so it goes alone.
|
|
||||||
func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
asksWithPaths(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
|
||||||
cutAt(t, open, t0.Add(90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
first := ws[0]
|
|
||||||
heard := t0.Add(100 * time.Second)
|
|
||||||
hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard)
|
|
||||||
ws, bs := walks(t, open)
|
|
||||||
if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued ||
|
|
||||||
bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") {
|
|
||||||
t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
first.State = inventory.PlanDone
|
|
||||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cutAt(t, open, heard.Add(time.Second))
|
|
||||||
ws, _ = walks(t, open)
|
|
||||||
if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
|
||||||
t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws)
|
|
||||||
}
|
|
||||||
own := ws[0]
|
|
||||||
// Unrelated to the own-path walk by every footprint, a merge still waits for it.
|
|
||||||
later := heard.Add(time.Minute)
|
|
||||||
hear(t, open, lone(repoMerge("two", "c2", later)), later)
|
|
||||||
ws, bs = walks(t, open)
|
|
||||||
if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID {
|
|
||||||
t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -54,21 +54,6 @@ func otherMachines(machine string, nodes ...string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
|
|
||||||
// condition at once where this process keeps the conditions: a push is when a person is looking.
|
|
||||||
func reportKept(ctx context.Context, plan catalogue.Resolution) {
|
|
||||||
for _, k := range plan.Kept {
|
|
||||||
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
|
|
||||||
if conditionsFrom != nil {
|
|
||||||
o := keptObservation(k)
|
|
||||||
o.Source = probeBindingsID
|
|
||||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
|
||||||
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
|
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
|
||||||
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
|
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
|
||||||
// moves it (said, so the data goes first), and never anything else.
|
// moves it (said, so the data goes first), and never anything else.
|
||||||
|
|||||||
@@ -75,19 +75,3 @@ func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
|
|||||||
t.Error("two consumers, one condition")
|
t.Error("two consumers, one condition")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A push says the move it refused, and raises its condition at once.
|
|
||||||
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
|
|
||||||
k, _ := withConditionsInMemory(t)
|
|
||||||
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
|
|
||||||
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
|
|
||||||
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
|
|
||||||
open[0].Source != probeBindingsID {
|
|
||||||
t.Fatalf("raised %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,24 +2,15 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// asking a build machine for a module, and what came back.
|
// asking a build machine for a module, and what came back.
|
||||||
@@ -28,80 +19,14 @@ import (
|
|||||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||||
|
|
||||||
// buildFrom turns what a builder said into what the mesh keeps.
|
|
||||||
//
|
|
||||||
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
|
|
||||||
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
|
|
||||||
// reference and composes the store's address back in where a reference is used. `against` — what
|
|
||||||
// the build stood on, the catalogue's edge — is recorded the same way, so an edge names a module's
|
|
||||||
// artifact and not the machine it was pulled from.
|
|
||||||
func buildFrom(result link.BuildResult) inventory.Build {
|
|
||||||
kept := inventory.Build{
|
|
||||||
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
|
|
||||||
Commit: result.Commit, On: result.On, Failed: result.Failed,
|
|
||||||
// **What the announcement carries, kept rather than discarded** (novox/hq 04-ISSUES/050).
|
|
||||||
// The catalogue turns the manifest into requires/provides edges and `against` into build
|
|
||||||
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
|
|
||||||
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
|
||||||
// rebuild the graph rather than a list of names.
|
|
||||||
Path: result.Path,
|
|
||||||
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
|
||||||
SourceFingerprint: result.SourceFingerprint,
|
|
||||||
SourceInputs: result.SourceInputs,
|
|
||||||
// And its reuse key (novox/hq ADR 0350 §1), which the chain names (ADR 0408 §1).
|
|
||||||
ReuseKey: result.ReuseKey,
|
|
||||||
}
|
|
||||||
// When it was asked, which is what orders it against another build of the same module
|
|
||||||
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
|
||||||
if asked, ok := link.BuildAskedAt(result.ID); ok {
|
|
||||||
kept.Asked = asked
|
|
||||||
}
|
|
||||||
for _, ref := range result.Against {
|
|
||||||
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
|
||||||
}
|
|
||||||
for _, ref := range result.Mirrored {
|
|
||||||
kept.Mirrored = append(kept.Mirrored, catalogue.Recorded(ref))
|
|
||||||
}
|
|
||||||
for _, r := range result.Read {
|
|
||||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
|
||||||
}
|
|
||||||
// What it was made from, as files (novox/hq ADR 0267): the planner maps the next merge onto it.
|
|
||||||
for _, s := range result.Sources {
|
|
||||||
kept.Sources = append(kept.Sources, inventory.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
|
|
||||||
}
|
|
||||||
var announced []inventory.Artifact
|
|
||||||
for _, made := range result.Made {
|
|
||||||
announced = append(announced, inventory.Artifact{
|
|
||||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
|
||||||
})
|
|
||||||
kept.Made = append(kept.Made, inventory.Artifact{
|
|
||||||
Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
|
||||||
// The module name comes from the manifest, which only exists when the build got that far.
|
|
||||||
if len(kept.Manifest) > 0 {
|
|
||||||
if list, err := judgedWith(result); err == nil {
|
|
||||||
if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil {
|
|
||||||
kept.Module = m.Module
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return kept
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildsCommand says what has been built lately.
|
// buildsCommand says what has been built lately.
|
||||||
func buildsCommand(ctx context.Context, args []string) error {
|
func buildsCommand(ctx context.Context, args []string) error {
|
||||||
set := flag.NewFlagSet("builds", flag.ContinueOnError)
|
set := flag.NewFlagSet("builds", flag.ContinueOnError)
|
||||||
limit := set.Int("n", 20, "how many to show")
|
limit := set.Int("n", 20, "how many to show")
|
||||||
logOf := set.String("log", "", "a build's id: print what the build machine said, line by line")
|
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if *logOf != "" {
|
|
||||||
return buildLog(ctx, *logOf)
|
|
||||||
}
|
|
||||||
module := ""
|
module := ""
|
||||||
if len(positionals) == 1 {
|
if len(positionals) == 1 {
|
||||||
module = positionals[0]
|
module = positionals[0]
|
||||||
@@ -182,365 +107,6 @@ func printBuild(w io.Writer, b inventory.Build) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// builderCommand issues a build machine its own broker credential.
|
|
||||||
//
|
|
||||||
// **A build machine is not a node**, and giving it a node's account would let it read another
|
|
||||||
// machine's declarations. This is narrower and different: read the build queue, write the
|
|
||||||
// exchange and an asker's reply queue, and nothing else.
|
|
||||||
//
|
|
||||||
// Issued rather than assumed, because until this the builder used whatever credential it was
|
|
||||||
// handed — which in practice meant the broker's own administrative one. A program documented as
|
|
||||||
// holding its own credential and given somebody else's is worse than one with no story at all.
|
|
||||||
func builderCommand(ctx context.Context, args []string) error {
|
|
||||||
set := flag.NewFlagSet("builder issue", flag.ContinueOnError)
|
|
||||||
// Which machine will use it. Given, the credential is delivered by the mesh rather than
|
|
||||||
// printed for somebody to carry — which is the difference between the builder being a module
|
|
||||||
// and being a program somebody configures.
|
|
||||||
forNode := set.String("node", "",
|
|
||||||
"the machine that will run it, so the mesh delivers the credential instead of printing it")
|
|
||||||
module := set.String("module", "builder", "the module on that machine that will read it")
|
|
||||||
positionals, err := parseAround(set, args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(positionals) != 2 || positionals[0] != "issue" {
|
|
||||||
return errors.New("builder issue <name> [--node <machine>]")
|
|
||||||
}
|
|
||||||
name := positionals[1]
|
|
||||||
|
|
||||||
// **The build machine's credential is a module's credential** (novox/hq ADR 0131, design 28
|
|
||||||
// task 5.5): minted into the mesh's records and sealed to the machine as the builder module's
|
|
||||||
// broker secret, usable at the next push — the same act `module issue` performs, and the same
|
|
||||||
// account the composed user list carries. Nothing is created on a server; the bus reads the list.
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
inv := open.inventory
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
m, known := shelf[*module]
|
|
||||||
if !known {
|
|
||||||
return fmt.Errorf("%s is not in the catalogue; `module add` it first", *module)
|
|
||||||
}
|
|
||||||
fmt.Printf("build machine %s: ", name)
|
|
||||||
node := *forNode
|
|
||||||
if node == "" {
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Manifest.Module == *module && len(e.On) > 0 {
|
|
||||||
node = e.On[0]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if node == "" {
|
|
||||||
return fmt.Errorf("%s is assigned nowhere; `assign <machine> %s` first, or say --node", *module, *module)
|
|
||||||
}
|
|
||||||
address, err := broker.BusAddress()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return issueOnTheNewBus(ctx, inv, m, node, address)
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordAsked keeps a build request once it is asked (novox/hq issue 325), with who asked it, in a store
|
|
||||||
// opened for the purpose: the asks reach here from processes that hold none.
|
|
||||||
func recordAsked(ctx context.Context, r inventory.BuildRequest) {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", r.ID, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
recordBuildRequest(ctx, open.inventory, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
// markWaitFailed says what became of a kept build request whose waited ask failed: never handed over, so
|
|
||||||
// nothing runs; or handed over and no longer waited for, so asked with its outcome unknown — still read as in
|
|
||||||
// flight until its outcome or its bound (novox/hq issue 325).
|
|
||||||
func markWaitFailed(ctx context.Context, id string, why error) {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "build %s could not be marked: %v\n", id, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
mark := open.inventory.MarkOutcomeUnknown
|
|
||||||
if errors.Is(why, link.ErrNotHandedOver) {
|
|
||||||
mark = open.inventory.MarkNotAsked
|
|
||||||
}
|
|
||||||
if err := mark(ctx, id, why.Error()); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "build %s could not be marked: %v\n", id, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
|
||||||
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
|
||||||
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
|
||||||
//
|
|
||||||
// asker is who asked, for the build request kept once the ask is made (novox/hq issue 325); empty for an
|
|
||||||
// asker that keeps the request itself, with its own name, once it knows the ask was made.
|
|
||||||
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
|
||||||
dryRun bool, asker string) (string, error) {
|
|
||||||
if dryRun && wait != 0 {
|
|
||||||
return "", errors.New("a dry run waited for is `build --dry-run`")
|
|
||||||
}
|
|
||||||
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
|
||||||
// the reason, rather than sent to a machine to fail at `git clone`.
|
|
||||||
repository, err := cloneFrom(ctx, source)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
// Through a verb, only a repository the catalogue builds from (novox/hq ADR 0266).
|
|
||||||
if err := verbMayAsk(ctx, source, repository, path); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
ident, err := openIdentity(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer ident.Close()
|
|
||||||
|
|
||||||
server, err := connectLink(ctx, nil, nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
// Correlated by something the control plane makes, not by the module's name: two builds of one
|
|
||||||
// module can be in flight, and the second answer is not the first one's.
|
|
||||||
request := link.BuildRequest{
|
|
||||||
ID: link.NewBuildID(time.Now()),
|
|
||||||
Repository: repository,
|
|
||||||
Path: path,
|
|
||||||
Ref: ref,
|
|
||||||
Held: heldBy(ctx),
|
|
||||||
Seats: seatBases(ctx),
|
|
||||||
DryRun: dryRun,
|
|
||||||
}
|
|
||||||
fmt.Printf("asked for %s", source)
|
|
||||||
if source.Seat != "" {
|
|
||||||
fmt.Printf(" (%s)", repository)
|
|
||||||
}
|
|
||||||
// The id is how a person follows this build while it runs: `builds --log <id>`.
|
|
||||||
fmt.Printf(" as %s", request.ID)
|
|
||||||
if path != "" {
|
|
||||||
fmt.Printf(" at %s", path)
|
|
||||||
}
|
|
||||||
if ref != "" {
|
|
||||||
fmt.Printf(" on %s", ref)
|
|
||||||
}
|
|
||||||
fmt.Println()
|
|
||||||
|
|
||||||
seat := buildSeatHeld(ctx)
|
|
||||||
ask, err := askOverOn(seat)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer ask.Close()
|
|
||||||
fmt.Printf(" of %s\n", seat)
|
|
||||||
// Kept once it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325);
|
|
||||||
// never before, so an ask that failed never reads as a build in flight. A dry run registers nothing, and
|
|
||||||
// is not kept.
|
|
||||||
keep := !dryRun && asker != ""
|
|
||||||
// Asked at the terminal is what lets its outcome register a module from a repository the catalogue does
|
|
||||||
// not build it from (novox/hq ADR 0266); never through a verb.
|
|
||||||
asked := inventory.BuildRequest{ID: request.ID, Repository: source.Repository, Seat: source.Seat, Path: path,
|
|
||||||
Ref: ref, For: asker, AtTerminal: startedAtTheTerminal()}
|
|
||||||
|
|
||||||
if wait == 0 {
|
|
||||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
|
||||||
// controller takes it in — records the build, registers the module — whether or not anybody
|
|
||||||
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
|
|
||||||
// follows the build by its id instead.
|
|
||||||
if err := ask.Ask(ctx, request); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if keep {
|
|
||||||
recordAsked(ctx, asked)
|
|
||||||
}
|
|
||||||
if dryRun {
|
|
||||||
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
|
||||||
"its outcome is not taken in\n", request.ID)
|
|
||||||
return request.ID, nil
|
|
||||||
}
|
|
||||||
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
|
|
||||||
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
|
|
||||||
return request.ID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Waited for: kept while it is waited for, since that can take minutes, and marked when the hand-over
|
|
||||||
// failed (not asked) or the wait did (asked, outcome unknown) — an outcome heard later is the last word.
|
|
||||||
if keep {
|
|
||||||
recordAsked(ctx, asked)
|
|
||||||
}
|
|
||||||
result, err := ask.Submit(ctx, request, wait)
|
|
||||||
if err != nil {
|
|
||||||
if keep {
|
|
||||||
markWaitFailed(ctx, request.ID, err)
|
|
||||||
}
|
|
||||||
return request.ID, err
|
|
||||||
}
|
|
||||||
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return request.ID, err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
|
||||||
// The assignments pending on this build, made or ended (novox/hq issue 325).
|
|
||||||
for _, line := range settlePendingOnBuild(ctx, open, result, manifest.Module, err) {
|
|
||||||
fmt.Printf(" %s\n", line)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return request.ID, err
|
|
||||||
}
|
|
||||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
|
||||||
for _, made := range kept.Made {
|
|
||||||
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
|
|
||||||
}
|
|
||||||
fmt.Printf("\n%s %s, built on %s from %s\n",
|
|
||||||
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
|
||||||
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
|
|
||||||
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
|
||||||
return request.ID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
|
|
||||||
// result on at once (novox/hq issue 126, ADR 0163): a person choreographing a data move must
|
|
||||||
// know which module will not wait for them.
|
|
||||||
func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module string) {
|
|
||||||
if u, err := inv.UpgradeOf(ctx, module); err == nil && u.RollOut {
|
|
||||||
how := "one machine at a time"
|
|
||||||
if u.Together {
|
|
||||||
how = "every machine at once"
|
|
||||||
}
|
|
||||||
fmt.Printf(" %s rolls out on build: the machines running it are sent this now, %s — "+
|
|
||||||
"`upgrade %s record` first if something must move before it does\n", module, how, module)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeIn is what the mesh does with a build's outcome, whoever hears it: the waiting command and
|
|
||||||
// the daemon that follows the role's events both come here (novox/hq issue 176), so a build's
|
|
||||||
// result reaches the catalogue whether or not the asker was still listening.
|
|
||||||
//
|
|
||||||
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
|
|
||||||
// nobody asked for, and the difference is the whole of whether somebody should be looking at
|
|
||||||
// something. Then parsed with the same parser a hand-written manifest goes through — a second path
|
|
||||||
// would be a second thing to disagree about what a manifest is — and registered with where it came
|
|
||||||
// from: **for a source on a seat, as the path and the seat, never the URL just cloned** (ADR 0111),
|
|
||||||
// which the request carried and the outcome echoes. A definition naming an installation is refused
|
|
||||||
// here, where it would enter the catalogue; the build stays recorded and the refusal says which.
|
|
||||||
//
|
|
||||||
// Idempotent: the same outcome taken in twice registers the same module twice, which is one row
|
|
||||||
// written with the same values.
|
|
||||||
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
|
|
||||||
catalogue.Manifest, inventory.Build, error) {
|
|
||||||
// The list of predefined keys as the store keeps it now: another controller process may have kept a newer one.
|
|
||||||
if err := loadPredefined(ctx, inv); err != nil {
|
|
||||||
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+
|
|
||||||
"read: %w", err)
|
|
||||||
}
|
|
||||||
kept := buildFrom(result)
|
|
||||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
|
||||||
return catalogue.Manifest{}, kept, err
|
|
||||||
}
|
|
||||||
if result.Failed != "" {
|
|
||||||
// The builder's own words. Wrapping them in something about the control plane would put
|
|
||||||
// two explanations between a person and a build log.
|
|
||||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
|
||||||
result.On, result.Repository, result.Failed)
|
|
||||||
}
|
|
||||||
// Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq
|
|
||||||
// ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the
|
|
||||||
// mesh publishes, before it is registered.
|
|
||||||
list, err := judgedWith(result)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
|
||||||
result.Repository, short(result.Commit), err)
|
|
||||||
}
|
|
||||||
manifest, err := catalogue.ParseManifestWith(kept.Manifest, list)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
|
||||||
result.On, result.Repository, err)
|
|
||||||
}
|
|
||||||
recorded := inventory.Source{
|
|
||||||
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
|
|
||||||
BuiltFrom: result.Commit, Head: result.Commit,
|
|
||||||
// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
|
|
||||||
Against: kept.Against,
|
|
||||||
// When it was asked, so an older request heard later does not replace a newer one
|
|
||||||
// (novox/hq 04-ISSUES/219).
|
|
||||||
Asked: kept.Asked,
|
|
||||||
}
|
|
||||||
if result.Source != nil && result.Source.Seat != "" {
|
|
||||||
recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
|
|
||||||
}
|
|
||||||
// **A build at a commit does not change the branch a module follows** (novox/hq 04-ISSUES/215):
|
|
||||||
// the commit is built and recorded as what it was built from, and the module keeps following
|
|
||||||
// what it followed before — the repository's default branch for one new to the catalogue.
|
|
||||||
if followedBranch(result.Ref) == "" && result.Ref != "" {
|
|
||||||
recorded.Ref = ""
|
|
||||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
|
||||||
recorded.Ref = followedBranch(was.Ref)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
trunk := result.Trunk
|
|
||||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil && followedBranch(was.Ref) != "" {
|
|
||||||
trunk = followedBranch(was.Ref)
|
|
||||||
}
|
|
||||||
if trunk == "" {
|
|
||||||
trunk = "main"
|
|
||||||
}
|
|
||||||
j := takenBuild{kept: kept, manifest: manifest, recorded: recorded, ask: byBuild(result.ID), path: result.Path,
|
|
||||||
trunk: trunk, by: result.On, what: result.Repository, commit: result.Commit}
|
|
||||||
// The checks only an outcome of build-agent needs: a key is a trunk build by construction (ADR 0501 §2), in no
|
|
||||||
// scratch namespace, and carries no list of its own.
|
|
||||||
j.published = func() error {
|
|
||||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
|
|
||||||
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
|
|
||||||
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
|
|
||||||
// The branch the module already follows is its trunk — never the branch a build was asked at, or a
|
|
||||||
// build of a feature branch by name would make that branch its trunk.
|
|
||||||
follows := ""
|
|
||||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
|
||||||
follows = followedBranch(was.Ref)
|
|
||||||
}
|
|
||||||
if err := publishable(result, follows); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
|
|
||||||
// registered only as the walk's copy in the module's own namespace.
|
|
||||||
if scratch := inScratch(result.Made); scratch != "" {
|
|
||||||
return fmt.Errorf("%s is in the scratch namespace, which only a walk's copy leaves (ADR 0350)", scratch)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
j.carried = func() error {
|
|
||||||
// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
|
|
||||||
// catalogue that may be registered keeps its list when its commit is newer than the one kept.
|
|
||||||
carried, err := carriedList(result)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if carried != nil {
|
|
||||||
if err := publishPredefined(ctx, inv, result, carried); err != nil {
|
|
||||||
fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
manifest, err = registerJudged(ctx, inv, j)
|
|
||||||
return manifest, kept, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// takenBuild is a build the controller judges and registers, whatever told it of the build: build-agent's outcome (the
|
// takenBuild is a build the controller judges and registers, whatever told it of the build: build-agent's outcome (the
|
||||||
// builder's own module's, until R4g, ADR 0453 §5) or the builder's done key (ADR 0501 §1). **One judging for both**:
|
// builder's own module's, until R4g, ADR 0453 §5) or the builder's done key (ADR 0501 §1). **One judging for both**:
|
||||||
// the rules of who may register, the gate a build failed, the predefined keys and the rule that an unchanged build
|
// the rules of who may register, the gate a build failed, the predefined keys and the rule that an unchanged build
|
||||||
@@ -584,16 +150,6 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild)
|
|||||||
return notRegistered(err)
|
return notRegistered(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
|
||||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
|
||||||
// and the next push would send it.
|
|
||||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
|
||||||
return manifest, err
|
|
||||||
} else if failed {
|
|
||||||
return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
|
||||||
"back: it is recorded and not registered again — a newer build is", j.by, manifest.Module,
|
|
||||||
short(j.commit))
|
|
||||||
}
|
|
||||||
// Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a
|
// Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a
|
||||||
// key that list does not hold, or of another type, is refused here and never enters the catalogue.
|
// key that list does not hold, or of another type, is refused here and never enters the catalogue.
|
||||||
if j.carried != nil {
|
if j.carried != nil {
|
||||||
@@ -643,44 +199,6 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild)
|
|||||||
return manifest, nil
|
return manifest, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
|
|
||||||
func inScratch(made []link.MadeArtifact) string {
|
|
||||||
for _, m := range made {
|
|
||||||
if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) {
|
|
||||||
return m.Reference
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
|
||||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
|
||||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
|
||||||
|
|
||||||
// publishable says whether a build's outcome may become a module's version: its commit on the module's
|
|
||||||
// trunk, as the build seat read the forge at the build — the branch the module follows when its source
|
|
||||||
// names one, else its repository's default branch. A build seat that could not say — one older than the
|
|
||||||
// rule, building its own successor — is let through and said, so the rule can reach the mesh.
|
|
||||||
func publishable(result link.BuildResult, follows string) error {
|
|
||||||
if result.Check != nil || result.Checked != nil || result.DryRun {
|
|
||||||
return errors.New("a check or a dry run is never published")
|
|
||||||
}
|
|
||||||
if result.Trunk == "" {
|
|
||||||
fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+
|
|
||||||
"registered as before\n", result.ID, short(result.Commit))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
trunk := result.Trunk
|
|
||||||
if follows != "" {
|
|
||||||
trunk = follows
|
|
||||||
}
|
|
||||||
on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk)
|
|
||||||
if !on {
|
|
||||||
return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// answers is what the three questions came back with, read once.
|
// answers is what the three questions came back with, read once.
|
||||||
type answers struct {
|
type answers struct {
|
||||||
// notAnnounced are the owners some module reads the state of that have announced nothing yet (novox/hq ADR 0495,
|
// notAnnounced are the owners some module reads the state of that have announced nothing yet (novox/hq ADR 0495,
|
||||||
@@ -701,12 +219,6 @@ type answers struct {
|
|||||||
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
||||||
// recorded at send, not at apply).
|
// recorded at send, not at apply).
|
||||||
reported []inventory.Reported
|
reported []inventory.Reported
|
||||||
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
|
||||||
plans []inventory.Plan
|
|
||||||
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
|
||||||
paused pauseView
|
|
||||||
// tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296).
|
|
||||||
tierBounds tierBounds
|
|
||||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||||
// other answer here: those are about a machine that was told something, and this is about one
|
// other answer here: those are about a machine that was told something, and this is about one
|
||||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||||
@@ -761,190 +273,4 @@ type answers struct {
|
|||||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||||
pending []inventory.PendingAssignment
|
pending []inventory.PendingAssignment
|
||||||
pendingUnread string
|
pendingUnread string
|
||||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
|
||||||
// issue 485); lastSentUnread why it could not be read.
|
|
||||||
lastSent map[string]inventory.Send
|
|
||||||
lastSentUnread string
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
|
||||||
//
|
|
||||||
// **A failure here is not a failure to build.** A module that names no base does not need this at
|
|
||||||
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
|
|
||||||
// than a build command refusing to start because a query did not run. So the store not opening is
|
|
||||||
// reported and the build goes ahead without it.
|
|
||||||
//
|
|
||||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
|
||||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
|
||||||
// runningBy is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module, as a
|
|
||||||
// builder reaches the artifact store (novox/hq issue 721): what a check compares its builds with. Empty when it cannot
|
|
||||||
// be read, said, and then nothing is said unchanged.
|
|
||||||
func runningBy(ctx context.Context) map[string]string {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
|
||||||
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
running, err := open.inventory.RegisteredHeld(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
|
||||||
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
|
||||||
if err != nil || address == "" {
|
|
||||||
return running
|
|
||||||
}
|
|
||||||
routed := make(map[string]string, len(running))
|
|
||||||
for key, reference := range running {
|
|
||||||
routed[key] = catalogue.Rerouted(reference, address)
|
|
||||||
}
|
|
||||||
return routed
|
|
||||||
}
|
|
||||||
|
|
||||||
func heldBy(ctx context.Context) map[string]string {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
|
|
||||||
"base will be told that base is missing: %v\n", err)
|
|
||||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
held, err := open.inventory.Held(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
|
||||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
|
|
||||||
"module naming a base will be handed a reference nothing can fetch: %v\n", err)
|
|
||||||
return held
|
|
||||||
}
|
|
||||||
if address == "" {
|
|
||||||
return held
|
|
||||||
}
|
|
||||||
routed := make(map[string]string, len(held))
|
|
||||||
for repository, reference := range held {
|
|
||||||
routed[repository] = catalogue.Rerouted(reference, address)
|
|
||||||
}
|
|
||||||
return routed
|
|
||||||
}
|
|
||||||
|
|
||||||
// askOver opens the way a build is asked for, on whichever bus the mesh is on.
|
|
||||||
//
|
|
||||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
|
||||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
|
||||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
|
||||||
func askOverOn(seat string) (link.Builders, error) {
|
|
||||||
// The serving controller asks on its own connection (novox/hq issue 327).
|
|
||||||
if serving := servingBus.Load(); serving != nil {
|
|
||||||
return link.BuildsOn(serving, seat), nil
|
|
||||||
}
|
|
||||||
address, err := broker.BusAddress()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return link.BuildsOverNATSOn(address, seat)
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
|
|
||||||
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
|
|
||||||
// moment the first build-agent is assigned — and a controller that asked the new role before then
|
|
||||||
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
|
|
||||||
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
|
|
||||||
func buildSeatHeld(ctx context.Context) string {
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
|
|
||||||
link.TheBuildMachine, err)
|
|
||||||
return link.TheBuildMachine
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
entries, err := open.inventory.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
|
|
||||||
link.TheBuildMachine, err)
|
|
||||||
return link.TheBuildMachine
|
|
||||||
}
|
|
||||||
return buildSeatAmong(entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
|
|
||||||
// assigned module claims it; else the retired role while an assigned module still claims that; else
|
|
||||||
// the current role, which is where every ask goes once the handover is done.
|
|
||||||
func buildSeatAmong(entries []inventory.Entry) string {
|
|
||||||
heldBefore := false
|
|
||||||
for _, e := range entries {
|
|
||||||
if len(e.On) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
|
|
||||||
return link.TheBuildMachine
|
|
||||||
}
|
|
||||||
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
|
|
||||||
heldBefore = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if heldBefore {
|
|
||||||
return link.TheBuildMachineBefore
|
|
||||||
}
|
|
||||||
return link.TheBuildMachine
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildLog prints everything a build machine said about one build, read back from the bus.
|
|
||||||
//
|
|
||||||
// **From the stream, not from a record** (novox/hq ADR 0157). A build's lines are the role's own
|
|
||||||
// events under the build's id, retained with every other event; the mesh keeps no second copy. Read
|
|
||||||
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
|
|
||||||
// for the reading, and filtered by subject, so one build's lines are all that travel.
|
|
||||||
func buildLog(ctx context.Context, id string) error {
|
|
||||||
js, err := aBus()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
|
|
||||||
// Under whichever build role did it: a build asked of the retired role during the handover
|
|
||||||
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
|
|
||||||
lines := link.BuildLogOf("*", id)
|
|
||||||
sub, err := js.Context().PullSubscribe(lines, "",
|
|
||||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
|
|
||||||
}
|
|
||||||
defer func() { _ = sub.Unsubscribe() }()
|
|
||||||
|
|
||||||
printed := 0
|
|
||||||
for {
|
|
||||||
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
|
|
||||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
return fmt.Errorf("reading a build's log: %w", err)
|
|
||||||
}
|
|
||||||
for _, msg := range batch {
|
|
||||||
var line link.BuildLine
|
|
||||||
if err := json.Unmarshal(msg.Data, &line); err != nil {
|
|
||||||
fmt.Printf(" ? %s\n", string(msg.Data))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
at := line.At
|
|
||||||
if t, err := time.Parse(time.RFC3339Nano, line.At); err == nil {
|
|
||||||
at = t.Local().Format("15:04:05")
|
|
||||||
}
|
|
||||||
fmt.Printf("%s %4d [%s] %s\n", at, line.Seq, line.Step, line.Message)
|
|
||||||
printed++
|
|
||||||
}
|
|
||||||
if len(batch) < 200 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if printed == 0 {
|
|
||||||
fmt.Printf("nothing on the bus for build %s: no build by that id in the last week, or a build "+
|
|
||||||
"machine older than this that said nothing while building\n", id)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,43 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
func claiming(module, seat string, on ...string) inventory.Entry {
|
|
||||||
return inventory.Entry{
|
|
||||||
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
|
|
||||||
On: on,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
|
|
||||||
// one while only the builder is assigned, the current one from the first build-agent on, and the
|
|
||||||
// current one when nothing holds either — where every ask goes once the handover is done.
|
|
||||||
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
|
|
||||||
onlyTheBuilder := []inventory.Entry{
|
|
||||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
|
||||||
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
|
|
||||||
}
|
|
||||||
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
|
|
||||||
t.Errorf("with only the builder assigned, asked %q", got)
|
|
||||||
}
|
|
||||||
bothHeld := []inventory.Entry{
|
|
||||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
|
||||||
claiming("build-agent", link.TheBuildMachine, "home-server"),
|
|
||||||
}
|
|
||||||
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
|
|
||||||
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
|
|
||||||
}
|
|
||||||
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
|
|
||||||
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
|
|
||||||
t.Errorf("with no holder of either, asked %q, want the current role", got)
|
|
||||||
}
|
|
||||||
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
|
|
||||||
t.Errorf("an empty catalogue asks %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -188,25 +189,13 @@ func (f *sourceForms) same(aRepository, aSeat, bRepository, bSeat string) bool {
|
|||||||
return f.canonical(aRepository, aSeat) == f.canonical(bRepository, bSeat)
|
return f.canonical(aRepository, aSeat) == f.canonical(bRepository, bSeat)
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildsFrom says the catalogue builds some module from this repository.
|
|
||||||
func (f *sourceForms) buildsFrom(entries []inventory.Entry, repository, seat string) bool {
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Provided || e.Source.Repository == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if f.same(e.Source.Repository, e.Source.Seat, repository, seat) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgeFacts is what the mesh's forge says of a repository a module is registered from (novox/hq ADR 0266).
|
// forgeFacts is what the mesh's forge says of a repository a module is registered from (novox/hq ADR 0266).
|
||||||
type forgeFacts struct {
|
type forgeFacts struct {
|
||||||
// ID is the forge's own id of the repository: what tells it from one deleted and made again by its name.
|
// ID is the forge's own id of the repository: what tells it from one deleted and made again by its name.
|
||||||
ID int64
|
ID int64
|
||||||
// Guarded is whether the branch is protected as a module's trunk must be: no direct push, at least one
|
// Guarded is whether the branch is protected as a module's trunk must be: no direct push, a merge whitelist naming
|
||||||
// required status, and no administrator merging past one. Why says what is missing when it is not.
|
// the tools' account and no agent's account, and no administrator merging past the rule (novox/hq ADR 0534 §6; no
|
||||||
|
// required status, which the forge no longer asks, §1). Why says what is missing when it is not.
|
||||||
Guarded bool
|
Guarded bool
|
||||||
Why string
|
Why string
|
||||||
}
|
}
|
||||||
@@ -361,27 +350,48 @@ func askTheForgeOnce(ctx context.Context, owner, repo, branch string) (forgeFact
|
|||||||
return facts, nil
|
return facts, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// forgeToolsAccount is the forge tools' account, which main's merge whitelist names for mesh-delivery (novox/hq ADR 0398
|
||||||
|
// §2, the gitea module's tools-account.ts TOOLS_ACCOUNT); beside it the whitelist names only people's own logins, the
|
||||||
|
// operator's (ADR 0534 §2), never an agent's account.
|
||||||
|
const forgeToolsAccount = "mesh-tools"
|
||||||
|
|
||||||
|
// isAgentAccount is whether a forge login is a node's agent's account, agent-<node> (novox/hq ADR 0398 §1): one that
|
||||||
|
// may never merge into a module's trunk by itself.
|
||||||
|
func isAgentAccount(login string) bool { return strings.HasPrefix(login, "agent-") }
|
||||||
|
|
||||||
// protectionRule is a branch protection rule as the forge's tool summarises it.
|
// protectionRule is a branch protection rule as the forge's tool summarises it.
|
||||||
type protectionRule struct {
|
type protectionRule struct {
|
||||||
Rule string `json:"rule"`
|
Rule string `json:"rule"`
|
||||||
Push bool `json:"push"`
|
Push bool `json:"push"`
|
||||||
RequiredStatuses []string `json:"required_statuses"`
|
RequiredStatuses []string `json:"required_statuses"`
|
||||||
AdminMayOverride bool `json:"admin_may_override"`
|
AdminMayOverride bool `json:"admin_may_override"`
|
||||||
|
// MergeWhitelist is who may merge into the branch; nil when the rule keeps no merge whitelist (anyone who writes).
|
||||||
|
MergeWhitelist []string `json:"merge_whitelist"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// judgedRule judges the one rule the forge applies to a branch, as the forge picks it: the rule named for the
|
// judgedRule judges the one rule the forge applies to a branch, as the forge picks it: the rule named for the
|
||||||
// branch, else the first glob rule, in the forge's order, that covers it (gitea's first matching rule). Never a
|
// branch, else the first glob rule, in the forge's order, that covers it (gitea's first matching rule). Never a
|
||||||
// later rule that happens to be stronger: the forge does not read it.
|
// later rule that happens to be stronger: the forge does not read it. A trunk is trusted for who can land a commit on
|
||||||
|
// it, not for a status it requires (novox/hq ADR 0534 §6): no push, a merge whitelist naming the tools' account and
|
||||||
|
// beside it only people's own logins, never an agent's, and no administrator merging past the rule; then a commit lands
|
||||||
|
// there through mesh-delivery, on a passing mesh/check and the operator's Approve (ADR 0534 §4), or by the operator's
|
||||||
|
// own hand at the forge (§2).
|
||||||
func judgedRule(rules []protectionRule, branch string) forgeFacts {
|
func judgedRule(rules []protectionRule, branch string) forgeFacts {
|
||||||
switch r := appliedRule(rules, branch); {
|
switch r := appliedRule(rules, branch); {
|
||||||
case r == nil:
|
case r == nil:
|
||||||
return forgeFacts{Why: fmt.Sprintf("no protection rule covers %s", branch)}
|
return forgeFacts{Why: fmt.Sprintf("no protection rule covers %s", branch)}
|
||||||
case r.Push:
|
case r.Push:
|
||||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets a person push to %s directly", r.Rule, branch)}
|
return forgeFacts{Why: fmt.Sprintf("the rule %s lets a person push to %s directly", r.Rule, branch)}
|
||||||
case len(r.RequiredStatuses) == 0:
|
case r.MergeWhitelist == nil || len(r.MergeWhitelist) == 0:
|
||||||
return forgeFacts{Why: fmt.Sprintf("the rule %s requires no status before a merge into %s", r.Rule, branch)}
|
return forgeFacts{Why: fmt.Sprintf("the rule %s keeps no merge whitelist naming %s alone for %s", r.Rule, forgeToolsAccount, branch)}
|
||||||
|
case !slices.Contains(r.MergeWhitelist, forgeToolsAccount):
|
||||||
|
return forgeFacts{Why: fmt.Sprintf("the rule %s lets %s merge into %s, and not %s", r.Rule,
|
||||||
|
strings.Join(r.MergeWhitelist, ", "), branch, forgeToolsAccount)}
|
||||||
|
case slices.ContainsFunc(r.MergeWhitelist, isAgentAccount):
|
||||||
|
return forgeFacts{Why: fmt.Sprintf("the rule %s lets an agent's account merge into %s: %s", r.Rule, branch,
|
||||||
|
strings.Join(r.MergeWhitelist, ", "))}
|
||||||
case r.AdminMayOverride:
|
case r.AdminMayOverride:
|
||||||
return forgeFacts{Why: fmt.Sprintf("the rule %s lets an administrator merge into %s past a status", r.Rule, branch)}
|
return forgeFacts{Why: fmt.Sprintf("the rule %s lets an administrator merge into %s past it", r.Rule, branch)}
|
||||||
}
|
}
|
||||||
return forgeFacts{Guarded: true}
|
return forgeFacts{Guarded: true}
|
||||||
}
|
}
|
||||||
@@ -464,7 +474,7 @@ func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, o
|
|||||||
// - from the module's registered repository — the same repository by the forge's own id, not only its name; or,
|
// - from the module's registered repository — the same repository by the forge's own id, not only its name; or,
|
||||||
// for a module new to the catalogue, from a repository the catalogue builds another module from;
|
// for a module new to the catalogue, from a repository the catalogue builds another module from;
|
||||||
// - and from a repository on the mesh's forge whose trunk is protected as a trunk must be: no direct push, at
|
// - and from a repository on the mesh's forge whose trunk is protected as a trunk must be: no direct push, at
|
||||||
// least one required status, no administrator merging past one.
|
// a merge whitelist naming the tools' account and no agent's, no administrator merging past the rule (ADR 0534 §6).
|
||||||
//
|
//
|
||||||
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
||||||
// repository and path; or, for the build of a merge, when mesh-delivery announced it on the operator's Approve of
|
// repository and path; or, for the build of a merge, when mesh-delivery announced it on the operator's Approve of
|
||||||
@@ -489,9 +499,6 @@ type registrationAsk struct {
|
|||||||
Terminal bool
|
Terminal bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// byBuild is the warrant of a build's outcome: read from the request kept under its id.
|
|
||||||
func byBuild(id string) registrationAsk { return registrationAsk{Build: id} }
|
|
||||||
|
|
||||||
// String is the ask as the registration's words name it.
|
// String is the ask as the registration's words name it.
|
||||||
func (a registrationAsk) String() string {
|
func (a registrationAsk) String() string {
|
||||||
switch {
|
switch {
|
||||||
@@ -517,7 +524,6 @@ type registrations interface {
|
|||||||
SourceIdentity(ctx context.Context, module string) (int64, error)
|
SourceIdentity(ctx context.Context, module string) (int64, error)
|
||||||
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
||||||
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
||||||
MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
||||||
@@ -697,29 +703,3 @@ func sourceWords(repository, seat string) string {
|
|||||||
}
|
}
|
||||||
return buildSource{Repository: repository, Seat: seat}.String()
|
return buildSource{Repository: repository, Seat: seat}.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
|
||||||
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
|
||||||
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
|
||||||
func verbMayAsk(ctx context.Context, source buildSource, url, path string) error {
|
|
||||||
if startedAtTheTerminal() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
open, err := openStores(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer open.Close()
|
|
||||||
entries, err := open.inventory.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
forms := newSourceForms(ctx, open.inventory)
|
|
||||||
if forms.buildsFrom(entries, source.Repository, source.Seat) || forms.buildsFrom(entries, url, "") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return terminalRefusal("%s is no repository the catalogue builds a module from, and a build of any other is "+
|
|
||||||
"asked at the controller's terminal only, never through a verb: a build node runs what the repository "+
|
|
||||||
"says, and its outcome would register a module the next push sends — whoever may call a verb includes "+
|
|
||||||
"agents (novox/hq ADR 0266). Nothing was asked", source)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -129,62 +129,6 @@ func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) {
|
|
||||||
theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
t.Setenv(verbVar, "build")
|
|
||||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
|
||||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git", "")
|
|
||||||
var policy *heldAtTheTerminal
|
|
||||||
if !errors.As(err, &policy) {
|
|
||||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
|
||||||
}
|
|
||||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
|
||||||
"http://forge.internal:20000/novox/mesh-catalog.git", ""); err != nil {
|
|
||||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
|
||||||
}
|
|
||||||
t.Setenv(verbVar, "")
|
|
||||||
t.Setenv(link.CallerVar, "")
|
|
||||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "", ""); err != nil {
|
|
||||||
t.Fatalf("the terminal was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
|
||||||
open := theCatalogue(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
|
||||||
map[string]any{"module": "sudo", "version": "evil"})
|
|
||||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
|
||||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
|
||||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
|
||||||
t.Fatalf("the fork's build was taken in: %v", err)
|
|
||||||
}
|
|
||||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
|
||||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
|
||||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
|
||||||
map[string]any{"module": "sudo", "version": "2"})
|
|
||||||
failed.Commit = "badbadbad0123456"
|
|
||||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if found && previous.ID == fork.ID {
|
|
||||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
|
||||||
}
|
|
||||||
if !found || previous.ID != "build-sudo" {
|
|
||||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||||
open := theCatalogue(t)
|
open := theCatalogue(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -326,10 +270,11 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
|||||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
var forgeMerge keptMerge
|
||||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
_ = json.Unmarshal(event, &forgeMerge)
|
||||||
t.Fatal(err)
|
wasForge := keptForgeMerge
|
||||||
}
|
t.Cleanup(func() { keptForgeMerge = wasForge })
|
||||||
|
keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil }
|
||||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
||||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
||||||
|
|||||||
@@ -102,16 +102,17 @@ func init() {
|
|||||||
// The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger
|
// The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger
|
||||||
// rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review).
|
// rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review).
|
||||||
func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) {
|
func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) {
|
||||||
guarded := protectionRule{Rule: "*", RequiredStatuses: []string{"mesh/merge-gate"}}
|
tools := []string{forgeToolsAccount}
|
||||||
open := protectionRule{Rule: "main", Push: true, RequiredStatuses: []string{"mesh/merge-gate"}}
|
guarded := protectionRule{Rule: "*", MergeWhitelist: tools}
|
||||||
|
open := protectionRule{Rule: "main", Push: true, MergeWhitelist: tools}
|
||||||
if f := judgedRule([]protectionRule{guarded, open}, "main"); f.Guarded {
|
if f := judgedRule([]protectionRule{guarded, open}, "main"); f.Guarded {
|
||||||
t.Error("an exact rule letting pushes was passed over for a glob that guards")
|
t.Error("an exact rule letting pushes was passed over for a glob that guards")
|
||||||
}
|
}
|
||||||
if f := judgedRule([]protectionRule{{Rule: "ma*", RequiredStatuses: nil}, {Rule: "*", RequiredStatuses: []string{"x"}}},
|
if f := judgedRule([]protectionRule{{Rule: "ma*", MergeWhitelist: nil}, {Rule: "*", MergeWhitelist: tools}},
|
||||||
"main"); f.Guarded || !strings.Contains(f.Why, "ma*") {
|
"main"); f.Guarded || !strings.Contains(f.Why, "ma*") {
|
||||||
t.Errorf("the first glob covering the branch was not the one judged: %+v", f)
|
t.Errorf("the first glob covering the branch was not the one judged: %+v", f)
|
||||||
}
|
}
|
||||||
if f := judgedRule([]protectionRule{{Rule: "release/*"}, {Rule: "main", RequiredStatuses: []string{"x"}}}, "main"); !f.Guarded {
|
if f := judgedRule([]protectionRule{{Rule: "release/*"}, {Rule: "main", MergeWhitelist: tools}}, "main"); !f.Guarded {
|
||||||
t.Errorf("the rule named for the branch was not judged: %+v", f)
|
t.Errorf("the rule named for the branch was not judged: %+v", f)
|
||||||
}
|
}
|
||||||
if f := judgedRule(nil, "main"); f.Guarded {
|
if f := judgedRule(nil, "main"); f.Guarded {
|
||||||
@@ -119,6 +120,39 @@ func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A module's trunk is trusted when it takes no push, only the tools' account and people's own logins merge into it,
|
||||||
|
// and no administrator merges past its rule** (novox/hq ADR 0534 §2, §6): the forge requires no status to merge (§1), so
|
||||||
|
// a rule requiring none is trusted; one with no merge whitelist, no tools' account in it, an agent's account in it, a
|
||||||
|
// push allowed or an administrator's override is not.
|
||||||
|
func TestATrunkIsTrustedForItsMergersNotForAStatus(t *testing.T) {
|
||||||
|
tools := []string{forgeToolsAccount}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
rule protectionRule
|
||||||
|
want bool
|
||||||
|
why string
|
||||||
|
}{
|
||||||
|
{"no status, no push, the tools alone merge", protectionRule{Rule: "main", MergeWhitelist: tools}, true, ""},
|
||||||
|
{"a status required as well", protectionRule{Rule: "main", MergeWhitelist: tools, RequiredStatuses: []string{"mesh/check"}}, true, ""},
|
||||||
|
{"no merge whitelist", protectionRule{Rule: "main"}, false, "merge whitelist"},
|
||||||
|
{"an empty merge whitelist", protectionRule{Rule: "main", MergeWhitelist: []string{}}, false, "merge whitelist"},
|
||||||
|
{"the operator's own login beside the tools", protectionRule{Rule: "main", MergeWhitelist: []string{forgeToolsAccount, "jschoubben"}}, true, ""},
|
||||||
|
{"an agent's account in the whitelist", protectionRule{Rule: "main", MergeWhitelist: []string{forgeToolsAccount, "agent-ace"}}, false, "agent-ace"},
|
||||||
|
{"an agent's account alone", protectionRule{Rule: "main", MergeWhitelist: []string{"agent-ace"}}, false, "agent-ace"},
|
||||||
|
{"a person's login without the tools", protectionRule{Rule: "main", MergeWhitelist: []string{"jschoubben"}}, false, forgeToolsAccount},
|
||||||
|
{"a push allowed", protectionRule{Rule: "main", Push: true, MergeWhitelist: tools}, false, "push"},
|
||||||
|
{"an administrator's override", protectionRule{Rule: "main", MergeWhitelist: tools, AdminMayOverride: true}, false, "administrator"},
|
||||||
|
} {
|
||||||
|
f := judgedRule([]protectionRule{c.rule}, "main")
|
||||||
|
if f.Guarded != c.want || !strings.Contains(f.Why, c.why) {
|
||||||
|
t.Errorf("%s: %+v, want guarded %v saying %q", c.name, f, c.want, c.why)
|
||||||
|
}
|
||||||
|
if strings.Contains(f.Why, "requires no status") {
|
||||||
|
t.Errorf("%s: a trunk is still judged on a required status: %s", c.name, f.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// An id the forge could not give keeps the id already recorded.
|
// An id the forge could not give keeps the id already recorded.
|
||||||
|
|
||||||
// **The operator's Approve of a reviewed group registers a moved module from its new repository** (novox/hq ADR 0420 §7,
|
// **The operator's Approve of a reviewed group registers a moved module from its new repository** (novox/hq ADR 0420 §7,
|
||||||
|
|||||||
@@ -2,12 +2,10 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"reflect"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -68,12 +66,3 @@ func TestBuildsSaysTheSourceFingerprint(t *testing.T) {
|
|||||||
t.Errorf("the other lines changed:\n%s\nwant\n%s", plain.String(), want)
|
t.Errorf("the other lines changed:\n%s\nwant\n%s", plain.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1215/AC4: what the builder said of the inputs is what the controller keeps of the build.
|
|
||||||
func TestTheControllerKeepsTheInputsABuilderSaid(t *testing.T) {
|
|
||||||
in := &sdkdelivery.SourceInputs{Module: "mailu", Tree: "t", Toolchains: map[string]string{"code": "ts sha256:aa 01"}}
|
|
||||||
kept := buildFrom(link.BuildResult{ID: "build-1", Repository: "r", Commit: "c", SourceFingerprint: "src1:x", SourceInputs: in})
|
|
||||||
if !reflect.DeepEqual(kept.SourceInputs, in) {
|
|
||||||
t.Errorf("the controller keeps %+v, the builder said %+v", kept.SourceInputs, in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -48,6 +47,11 @@ func aBuilderOnTheBus(t *testing.T) builderOnTheBus {
|
|||||||
}
|
}
|
||||||
*c.into = kv
|
*c.into = kv
|
||||||
}
|
}
|
||||||
|
// And the controller's own buckets, as its start asserts them: a merge it hears is kept in the bucket of the merges
|
||||||
|
// (ADR 0539 §17), which no other test need have made on this bus first.
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
servingBus.Store(js)
|
servingBus.Store(js)
|
||||||
t.Cleanup(func() { servingBus.Store(nil) })
|
t.Cleanup(func() { servingBus.Store(nil) })
|
||||||
withTrunkList(t, true)
|
withTrunkList(t, true)
|
||||||
@@ -106,34 +110,5 @@ func (b builderOnTheBus) built(t *testing.T, module, version, fp, manifest, stat
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// noBuildAsked fails the test if anything asks build-agent for a build (ADR 0501 §6, ADR 0503 §6).
|
|
||||||
func noBuildAsked(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
|
||||||
t.Errorf("a build was asked of build-agent, at %q", path)
|
|
||||||
return "", context.Canceled
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// asksWithPaths records every build a merge asks for, as repository, path and ref. The merge window's goroutine and
|
|
||||||
// the test's own may ask at once, so the record is written under a lock (issue 1543).
|
|
||||||
func asksWithPaths(t *testing.T) *[][3]string {
|
|
||||||
t.Helper()
|
|
||||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): these tests are of what a walk does around it
|
|
||||||
var asked [][3]string
|
|
||||||
var mu sync.Mutex
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
asked = append(asked, [3]string{source.Repository, path, ref})
|
|
||||||
return "b-" + path, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
return &asked
|
|
||||||
}
|
|
||||||
|
|
||||||
// theIndexedModule is the look at the indexer as the controller makes it, kept before any test replaces it.
|
// theIndexedModule is the look at the indexer as the controller makes it, kept before any test replaces it.
|
||||||
var theIndexedModule = indexedModule
|
var theIndexedModule = indexedModule
|
||||||
|
|||||||
@@ -1,112 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A12 (ADR 0453 §5, ADR 0498 §7; issue 1851): the one function every ask goes through refuses any module but the
|
|
||||||
// builder's own, naming the builder's key it is taken from instead; the builder's own module passes the refusal.
|
|
||||||
func TestBuildAgentIsAskedForTheBuildersOwnModuleAlone(t *testing.T) {
|
|
||||||
was := askedModuleOf
|
|
||||||
t.Cleanup(func() { askedModuleOf = was })
|
|
||||||
askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) {
|
|
||||||
return strings.TrimPrefix(path, "modules/"), nil
|
|
||||||
}
|
|
||||||
_, err := askABuild(t.Context(), buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, "modules/sudo", "main")
|
|
||||||
if !errors.Is(err, errNotTheBuildersOwn) || !strings.Contains(err.Error(), "sudo is taken from the builder's done key") ||
|
|
||||||
!strings.Contains(err.Error(), "mesh-delivery-builder_builds") {
|
|
||||||
t.Fatalf("an ask of another module: %v", err)
|
|
||||||
}
|
|
||||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/mesh-delivery", Seat: "git"},
|
|
||||||
"modules/mesh-delivery-builder"); err != nil {
|
|
||||||
t.Fatalf("the builder's own module is refused: %v", err)
|
|
||||||
}
|
|
||||||
askedModuleOf = func(context.Context, buildSource, string) (string, error) { return "", errors.New("no store") }
|
|
||||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x"}, ""); !errors.Is(err, errNotTheBuildersOwn) {
|
|
||||||
t.Fatalf("an ask whose module cannot be read went on: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A12: a walk of the builder's own module and one other asks build-agent for the first alone; the other is taken from
|
|
||||||
// its key.
|
|
||||||
func TestAWalkAsksBuildAgentForTheBuildersOwnModuleAlone(t *testing.T) {
|
|
||||||
var asked []string
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
|
||||||
asked = append(asked, path)
|
|
||||||
return "build-1", nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
p, byName := aWalkOfOne("sudo")
|
|
||||||
p.Tiers = [][]string{{"mesh-delivery-builder", "sudo"}}
|
|
||||||
byName["mesh-delivery-builder"] = byName["sudo"]
|
|
||||||
e := byName["mesh-delivery-builder"]
|
|
||||||
e.Source.Repository, e.Source.Path = "novox/mesh-delivery", "modules/mesh-delivery-builder"
|
|
||||||
byName["mesh-delivery-builder"] = e
|
|
||||||
for _, m := range p.Tiers[0] {
|
|
||||||
askModule(t.Context(), p, m, byName)
|
|
||||||
}
|
|
||||||
if len(asked) != 1 || asked[0] != "modules/mesh-delivery-builder" {
|
|
||||||
t.Fatalf("build-agent was asked for %v, want the builder's own module alone", asked)
|
|
||||||
}
|
|
||||||
if s := p.Modules["mesh-delivery-builder"]; s.Key || s.Build != "build-1" {
|
|
||||||
t.Fatalf("the builder's own module is not asked: %+v", s)
|
|
||||||
}
|
|
||||||
if s := p.Modules["sudo"]; !s.Key || s.Build != "" {
|
|
||||||
t.Fatalf("the other module is not taken from its key: %+v", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ADR 0520 §1 (issue 1851): until R4g build-agent and mesh-tools, the toolchain image its builds stand on, are build-agent's
|
|
||||||
// own, as the builder's module is: the builder writes no key of them, so a walk asks build-agent for them and reads no key;
|
|
||||||
// mesh-tools-go (its key from mesh-build-image) and any other module of the same tier are still taken from their keys.
|
|
||||||
func TestAWalkAsksBuildAgentForItselfAndMeshToolsAndReadsNoKeyOfThem(t *testing.T) {
|
|
||||||
own := []string{"build-agent", "mesh-tools"}
|
|
||||||
keyed := []string{"mesh-tools-go", "sudo"}
|
|
||||||
wasModule := askedModuleOf
|
|
||||||
askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) {
|
|
||||||
return strings.TrimPrefix(path, "modules/"), nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askedModuleOf = wasModule })
|
|
||||||
for _, m := range own {
|
|
||||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x", Seat: "git"}, "modules/"+m); err != nil {
|
|
||||||
t.Fatalf("an ask of %s is refused: %v", m, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x", Seat: "git"}, "modules/mesh-tools-go"); !errors.Is(err, errNotTheBuildersOwn) {
|
|
||||||
t.Fatalf("an ask of mesh-tools-go went on: %v", err)
|
|
||||||
}
|
|
||||||
var asked []string
|
|
||||||
was := askABuild
|
|
||||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
|
||||||
asked = append(asked, path)
|
|
||||||
return "build-" + strings.TrimPrefix(path, "modules/"), nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { askABuild = was })
|
|
||||||
p, byName := aWalkOfOne("sudo")
|
|
||||||
p.Tiers = [][]string{append(append([]string{}, own...), keyed...)}
|
|
||||||
for _, m := range append(append([]string{}, own...), keyed...) {
|
|
||||||
e := byName["sudo"]
|
|
||||||
e.Source.Path = "modules/" + m
|
|
||||||
byName[m] = e
|
|
||||||
}
|
|
||||||
for _, m := range p.Tiers[0] {
|
|
||||||
askModule(t.Context(), p, m, byName)
|
|
||||||
}
|
|
||||||
if strings.Join(asked, " ") != "modules/build-agent modules/mesh-tools" {
|
|
||||||
t.Fatalf("build-agent was asked for %v", asked)
|
|
||||||
}
|
|
||||||
for _, m := range own {
|
|
||||||
if s := p.Modules[m]; s.Key || s.Build != "build-"+m || s.State != "asked" {
|
|
||||||
t.Fatalf("%s is not asked of build-agent, or is read from a key: %+v", m, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, m := range keyed {
|
|
||||||
if s := p.Modules[m]; !s.Key || s.Build != "" {
|
|
||||||
t.Fatalf("%s is not taken from its key: %+v", m, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1945 (dev-r6-controller's B8 follows it): what a bus machine runs is read from its have, not from a send kept, so a
|
||||||
|
// machine whose have names the build the mesh holds is not moved by a bus upgrade, and one whose have names an older
|
||||||
|
// build is.
|
||||||
|
func TestABusMachineRunsWhatItsHaveNames(t *testing.T) {
|
||||||
|
was := busRuns
|
||||||
|
t.Cleanup(func() { busRuns = was })
|
||||||
|
busRuns = func(_ context.Context, node, _ string) (string, bool, error) {
|
||||||
|
return map[string]string{"novox": "src2:new", "ace": "src2:old"}[node], node != "g14", nil
|
||||||
|
}
|
||||||
|
b := busPending{module: "nats", to: "c-new", from: map[string]string{}, same: map[string]bool{}}
|
||||||
|
commitOf := map[string]string{"src2:new": "c-new", "src2:old": "c-old"}
|
||||||
|
for _, n := range []string{"novox", "ace", "g14"} {
|
||||||
|
fp, known, _ := busRuns(context.Background(), n, "nats")
|
||||||
|
if c, ok := commitOf[fp]; known && ok {
|
||||||
|
b.from[n] = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b.moves("novox") || !b.moves("ace") || !b.moves("g14") {
|
||||||
|
t.Errorf("novox moves %v (runs the new build), ace %v (an older one), g14 %v (says none)", b.moves("novox"),
|
||||||
|
b.moves("ace"), b.moves("g14"))
|
||||||
|
}
|
||||||
|
}
|
||||||
+105
-11
@@ -10,8 +10,10 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -111,20 +113,44 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro
|
|||||||
b.same[commit] = true
|
b.same[commit] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// What each machine runs is its have's fingerprint (novox/hq ADR 0539 clause 9), the commit of the build of it.
|
||||||
|
commitOf := map[string]string{}
|
||||||
|
for commit, src := range sources {
|
||||||
|
if src != "" {
|
||||||
|
commitOf[src] = commit
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, n := range b.machines {
|
for _, n := range b.machines {
|
||||||
sent, known, err := inv.SentBuilds(ctx, n)
|
fp, known, err := busRuns(ctx, n, b.module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return b, err
|
return b, err
|
||||||
}
|
}
|
||||||
if known {
|
if c, ok := commitOf[fp]; known && ok {
|
||||||
if c, carried := sent[b.module]; carried {
|
|
||||||
b.from[n] = c
|
b.from[n] = c
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return b, nil
|
return b, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// busRuns is the fingerprint a machine's node-engine says it runs of a module, from its have; known is false when it
|
||||||
|
// says none. A variable so a test reads it without a bus.
|
||||||
|
var busRuns = func(ctx context.Context, node, module string) (string, bool, error) {
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
h, found, err := haveOn(ctx, js, node, module)
|
||||||
|
if err != nil || !found || h.FP == "" {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
return h.FP, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
// busHeld names the machines a push may not send because sending them would replace the bus: the
|
// busHeld names the machines a push may not send because sending them would replace the bus: the
|
||||||
// planned step's, not a push's (ADR 0236). Said with the remedy.
|
// planned step's, not a push's (ADR 0236). Said with the remedy.
|
||||||
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
|
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
|
||||||
@@ -229,17 +255,85 @@ func busCommand(ctx context.Context, args []string) error {
|
|||||||
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
|
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
|
||||||
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
|
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
|
||||||
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
|
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
|
||||||
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
|
// **Nothing is sent, and the assigner never moves the bus** (novox/hq ADR 0539 clause 15): the bus's machines are
|
||||||
|
// composed at the build the mesh holds, and the operator writes each machine's want with the operator's credential,
|
||||||
|
// recorded, naming that composition.
|
||||||
|
wants, err := busWantsToWrite(withBusStep(ctx), open, b.module, moving)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
|
_ = inv.EndBusStep(ctx, step.ID, "failed", "the bus's composition could not be written: "+err.Error())
|
||||||
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
|
return fmt.Errorf("the bus's machines could not be composed at its new build: %w — nothing was replaced", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("composed %s at the bus's new build. Write each want below with the operator's credential, and record it "+
|
||||||
|
"in the hand-act log; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
||||||
|
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot as the "+
|
||||||
|
"way back. `bus` says how it went\n", strings.Join(moving, ", "), busStepBound)
|
||||||
|
for _, w := range wants {
|
||||||
|
fmt.Println(" " + w)
|
||||||
}
|
}
|
||||||
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
|
||||||
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
|
|
||||||
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// busWantsToWrite composes the bus's machines and answers, for each, the want the operator writes: its key in the
|
||||||
|
// assigner's bucket and its value, naming the fingerprint the bus module is registered at and that machine's
|
||||||
|
// composition of it.
|
||||||
|
func busWantsToWrite(ctx context.Context, open *stores, module string, machines []string) ([]string, error) {
|
||||||
|
if _, err := composeModules(ctx, open, machines); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
fps, err := currentFingerprints(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
digestOf := func(node string) (string, error) {
|
||||||
|
key, err := sdkdelivery.ComposedKey(node, module, fps[module])
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
e, err := digests.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%s: %w", key, err)
|
||||||
|
}
|
||||||
|
v, err := sdkdelivery.DecodeComposedDigest(e.Value())
|
||||||
|
return v.Digest, err
|
||||||
|
}
|
||||||
|
return busWants(module, fps[module], machines, digestOf)
|
||||||
|
}
|
||||||
|
|
||||||
|
// busWants is each machine's want for the bus module, as the operator writes it: `<bucket> <key> <value>`.
|
||||||
|
func busWants(module, fp string, machines []string, digestOf func(node string) (string, error)) ([]string, error) {
|
||||||
|
var out []string
|
||||||
|
for _, node := range machines {
|
||||||
|
digest, err := digestOf(node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
key, err := sdkdelivery.WantKey(node, module)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
value, err := sdkdelivery.EncodeWant(sdkdelivery.Want{FP: fp, Composed: digest,
|
||||||
|
Words: "the bus moved by a person's step (bus upgrade, ADR 0539 clause 15)"})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, fmt.Sprintf("%s %s %s", sdkdelivery.AssignerWantsBucket, key, value))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// busStatus is `bus`: what an upgrade would do, and the last step.
|
// busStatus is `bus`: what an upgrade would do, and the last step.
|
||||||
func busStatus(ctx context.Context) error {
|
func busStatus(ctx context.Context) error {
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
@@ -297,7 +391,7 @@ func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
reports, err := inv.LastReports(ctx)
|
reports, err := lastReports(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1945/AC7: a bus upgrade answers, for each machine, the want the operator writes: the assigner's bucket, the
|
||||||
|
// machine's key for the bus module, and a want naming the bus's fingerprint and that machine's composition of it. It
|
||||||
|
// sends nothing: nothing it calls publishes on a node's declaration subject (busWants composes no send at all).
|
||||||
|
func TestABusUpgradeAnswersTheWantTheOperatorWrites(t *testing.T) {
|
||||||
|
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||||
|
digests := map[string]string{"novox": "sha256:" + strings.Repeat("1", 64), "ace": "sha256:" + strings.Repeat("2", 64)}
|
||||||
|
lines, err := busWants("nats", fp, []string{"ace", "novox"}, func(node string) (string, error) { return digests[node], nil })
|
||||||
|
if err != nil || len(lines) != 2 {
|
||||||
|
t.Fatalf("busWants = %v, %v", lines, err)
|
||||||
|
}
|
||||||
|
for i, node := range []string{"ace", "novox"} {
|
||||||
|
parts := strings.SplitN(lines[i], " ", 3)
|
||||||
|
if len(parts) != 3 || parts[0] != sdkdelivery.AssignerWantsBucket || parts[1] != "want."+node+".nats" {
|
||||||
|
t.Fatalf("line %q", lines[i])
|
||||||
|
}
|
||||||
|
w, err := sdkdelivery.DecodeWant([]byte(parts[2]))
|
||||||
|
if err != nil || w.FP != fp || w.Composed != digests[node] {
|
||||||
|
t.Errorf("%s's want = %+v, %v", node, w, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := busWants("nats", fp, []string{"ace"}, func(string) (string, error) { return "", nil }); err == nil {
|
||||||
|
t.Errorf("a want naming no composition was answered")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,195 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A send held for the bus's planned step (novox/hq issue 336).
|
|
||||||
//
|
|
||||||
// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned
|
|
||||||
// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every
|
|
||||||
// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only
|
|
||||||
// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move.
|
|
||||||
// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what
|
|
||||||
// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it
|
|
||||||
// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the
|
|
||||||
// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks'
|
|
||||||
// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB).
|
|
||||||
|
|
||||||
// kindBusStepWaiting is S17's kind: bus.<module>.step-waiting.
|
|
||||||
const kindBusStepWaiting = "bus-step-waiting"
|
|
||||||
|
|
||||||
// busFacts is a new bus build waiting for its planned step, and the sends held for it.
|
|
||||||
type busFacts struct {
|
|
||||||
module, to string
|
|
||||||
// from is the build each machine of the bus runs; machines those whose bus the step would replace.
|
|
||||||
from map[string]string
|
|
||||||
machines []string
|
|
||||||
waits []busWaitFacts
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaitFacts is one walk whose send was refused because it would replace the bus.
|
|
||||||
type busWaitFacts struct {
|
|
||||||
plan, repository, commit string
|
|
||||||
// modules are what its tier sends: what waits.
|
|
||||||
modules []string
|
|
||||||
// since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal.
|
|
||||||
since time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note
|
|
||||||
// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save.
|
|
||||||
var busRefusedFirst = &firstSeen{at: map[string]time.Time{}}
|
|
||||||
|
|
||||||
type firstSeen struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
at map[string]time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// mark keeps the first moment an id was seen.
|
|
||||||
func (s *firstSeen) mark(id string, at time.Time) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if _, seen := s.at[id]; !seen {
|
|
||||||
s.at[id] = at
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// of is when an id was first seen; zero when it was not.
|
|
||||||
func (s *firstSeen) of(id string) time.Time {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return s.at[id]
|
|
||||||
}
|
|
||||||
|
|
||||||
// keepOnly forgets every id not given: a walk no longer held is not held since then.
|
|
||||||
func (s *firstSeen) keepOnly(ids map[string]bool) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
for id := range s.at {
|
|
||||||
if !ids[id] {
|
|
||||||
delete(s.at, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step.
|
|
||||||
func refusedForTheBus(err error) bool {
|
|
||||||
return err != nil && strings.Contains(err.Error(), errBusWaits.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its
|
|
||||||
// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace
|
|
||||||
// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not.
|
|
||||||
func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts {
|
|
||||||
f := busFacts{module: b.module, to: b.to, from: b.from}
|
|
||||||
for _, n := range b.machines {
|
|
||||||
if b.moves(n) {
|
|
||||||
f.machines = append(f.machines, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(f.machines) == 0 {
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
for _, p := range plans {
|
|
||||||
if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
since := p.Updated
|
|
||||||
if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) {
|
|
||||||
since = seen
|
|
||||||
}
|
|
||||||
var modules []string
|
|
||||||
if p.Tier >= 0 && p.Tier < len(p.Tiers) {
|
|
||||||
modules = append(modules, p.Tiers[p.Tier]...)
|
|
||||||
} else {
|
|
||||||
modules = planModules(p)
|
|
||||||
}
|
|
||||||
sort.Strings(modules)
|
|
||||||
f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules,
|
|
||||||
since: since})
|
|
||||||
}
|
|
||||||
sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) })
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out.
|
|
||||||
func (b busFacts) heldByTheBus() map[string]bool {
|
|
||||||
out := map[string]bool{}
|
|
||||||
for _, w := range b.waits {
|
|
||||||
out[w.plan] = true
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with
|
|
||||||
// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true").
|
|
||||||
const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`"
|
|
||||||
|
|
||||||
// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator.
|
|
||||||
func watchBusWaits(f *signalFacts) []conditions.Observation {
|
|
||||||
b := f.bus
|
|
||||||
if len(b.waits) == 0 || len(b.machines) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
since := b.waits[0].since
|
|
||||||
var walks, what []string
|
|
||||||
for _, w := range b.waits {
|
|
||||||
walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit),
|
|
||||||
strings.Join(w.modules, ", ")))
|
|
||||||
what = append(what, deliveryWhat(w.modules, w.repository))
|
|
||||||
}
|
|
||||||
var from []string
|
|
||||||
for _, n := range b.machines {
|
|
||||||
from = append(from, short(orNotKnown(b.from[n])))
|
|
||||||
}
|
|
||||||
machines := namesWords(b.machines, 3)
|
|
||||||
in := f.now.Sub(since)
|
|
||||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting",
|
|
||||||
Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
|
||||||
Machine: b.machines[0], Also: b.machines[1:],
|
|
||||||
Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+
|
|
||||||
"the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "),
|
|
||||||
b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb,
|
|
||||||
since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")),
|
|
||||||
Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)),
|
|
||||||
Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax),
|
|
||||||
Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+
|
|
||||||
"installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines,
|
|
||||||
namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax),
|
|
||||||
Needs: "start the bus upgrade " + FromMeshMCPServer,
|
|
||||||
Resolved: "Resolved: the bus upgrade started, and sends go on"}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sortedUnique is a list sorted, each once.
|
|
||||||
func sortedUnique(xs []string) []string {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var out []string
|
|
||||||
for _, x := range xs {
|
|
||||||
if !seen[x] {
|
|
||||||
seen[x] = true
|
|
||||||
out = append(out, x)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// clipWords keeps a plain sentence within a bound, at a word.
|
|
||||||
func clipWords(s string, n int) string {
|
|
||||||
if len(s) <= n {
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
cut := strings.LastIndex(s[:n-1], " ")
|
|
||||||
if cut <= 0 {
|
|
||||||
cut = n - 1
|
|
||||||
}
|
|
||||||
return s[:cut] + "…"
|
|
||||||
}
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
|
||||||
b := aBusOnAnchor()
|
|
||||||
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
|
|
||||||
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
|
|
||||||
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
|
|
||||||
Updated: updated}
|
|
||||||
}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
bus busPending
|
|
||||||
plans []inventory.Plan
|
|
||||||
first map[string]time.Time
|
|
||||||
want []string
|
|
||||||
since time.Time
|
|
||||||
}{
|
|
||||||
{"a walk refused for the bus", b,
|
|
||||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
|
|
||||||
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
|
|
||||||
{"refused earlier than its last save, as this controller saw it", b,
|
|
||||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
|
|
||||||
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
|
|
||||||
{"a walk refused for another reason", b,
|
|
||||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
|
|
||||||
nil, time.Time{}},
|
|
||||||
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
|
|
||||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
|
||||||
{"the bus already runs the build held: its step started", func() busPending {
|
|
||||||
o := aBusOnAnchor()
|
|
||||||
o.from = map[string]string{"anchor": o.to}
|
|
||||||
return o
|
|
||||||
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
|
||||||
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
|
|
||||||
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
|
|
||||||
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
|
|
||||||
return []inventory.Plan{p}
|
|
||||||
}(), nil, nil, time.Time{}},
|
|
||||||
} {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
|
|
||||||
var ids []string
|
|
||||||
for _, w := range got.waits {
|
|
||||||
ids = append(ids, w.plan)
|
|
||||||
}
|
|
||||||
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
|
|
||||||
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
|
|
||||||
}
|
|
||||||
if len(c.want) > 0 {
|
|
||||||
if !got.waits[0].since.Equal(c.since) {
|
|
||||||
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
|
|
||||||
}
|
|
||||||
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
|
|
||||||
t.Errorf("behind %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a
|
|
||||||
// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that
|
|
||||||
// ends it — and the wait is not read as a walk running late.
|
|
||||||
|
|
||||||
// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held.
|
|
||||||
func aBusOnAnchor() busPending {
|
|
||||||
return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"},
|
|
||||||
to: "32307bd1bbbb", same: map[string]bool{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it.
|
|
||||||
func refusedNote(b busPending, tier int) string {
|
|
||||||
held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) +
|
|
||||||
"), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)"
|
|
||||||
return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again"
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises
|
|
||||||
// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to,
|
|
||||||
// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the
|
|
||||||
// condition clears once the bus's machine runs the new build.
|
|
||||||
func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) {
|
|
||||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
|
||||||
b := aBusOnAnchor()
|
|
||||||
held := func(entered time.Duration) *signalFacts {
|
|
||||||
f := calm(now)
|
|
||||||
f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1,
|
|
||||||
entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}}
|
|
||||||
f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"},
|
|
||||||
waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122",
|
|
||||||
modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}}
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
f := held(time.Minute)
|
|
||||||
got := watchBusWaits(f)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("a send held for the bus a minute ago raised %+v", got)
|
|
||||||
}
|
|
||||||
o := got[0]
|
|
||||||
if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator {
|
|
||||||
t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus",
|
|
||||||
`"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} {
|
|
||||||
if !strings.Contains(o.Summary, want) {
|
|
||||||
t.Errorf("its summary does not say %q: %s", want, o.Summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" {
|
|
||||||
t.Errorf("its words do not say what waits and what the operator does: %+v", o)
|
|
||||||
}
|
|
||||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
|
||||||
Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok {
|
|
||||||
t.Errorf("its words are not plain: %s", why)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Past S3's bound: still the bus's wait, never a stalled walk.
|
|
||||||
late := held(45 * time.Minute)
|
|
||||||
if s3 := watchPlans(late); len(s3) != 0 {
|
|
||||||
t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3)
|
|
||||||
}
|
|
||||||
// A walk held for something else past its bound is still stalled.
|
|
||||||
other := held(45 * time.Minute)
|
|
||||||
other.plans[0].bus = false
|
|
||||||
if s3 := watchPlans(other); len(s3) != 1 {
|
|
||||||
t.Fatalf("a walk held for something else past its bound raised %+v", s3)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Through the keeper: open at the first tick, cleared when the step started.
|
|
||||||
store := conditions.NewInMemory()
|
|
||||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{},
|
|
||||||
Now: func() time.Time { return now }})
|
|
||||||
defer k.Close(context.Background())
|
|
||||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
||||||
w.see(t.Context(), held(time.Minute))
|
|
||||||
open, err := k.Open(t.Context())
|
|
||||||
if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" {
|
|
||||||
t.Fatalf("after the first tick, open: %+v (%v)", open, err)
|
|
||||||
}
|
|
||||||
started := held(time.Minute)
|
|
||||||
started.bus = busFacts{module: "nats", to: b.to}
|
|
||||||
started.plans[0].bus = false
|
|
||||||
w.see(t.Context(), started)
|
|
||||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
||||||
t.Fatalf("the step started, and open: %+v", open)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -29,7 +29,13 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
|||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
names = append(names, n.Name)
|
names = append(names, n.Name)
|
||||||
}
|
}
|
||||||
if err := broker.Raise(r, names); err != nil {
|
// Once every node runs the node-engine that reads no declaration, the declarations' stream and the nodes'
|
||||||
|
// consumers on it are not asserted again (novox/hq ADR 0539 clause 11; composed_start.go retires them).
|
||||||
|
raise := func() error { return broker.Raise(r, names) }
|
||||||
|
if declarationsRetiredNow(ctx, inv) {
|
||||||
|
raise = func() error { return broker.RaiseAfterDeclarations(r) }
|
||||||
|
}
|
||||||
|
if err := raise(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
|
// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
|
||||||
|
|||||||
@@ -7,14 +7,9 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||||
@@ -46,25 +41,33 @@ func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
|
|||||||
return broker.Consumer{}
|
return broker.Consumer{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
|
// aLateHolder is a module holding a build seat (node-build-go) on anchor, assigned after the controller started, and
|
||||||
// started, and the worker its seat's queue should have for it.
|
// the worker its seat's queue should have for it: a build seat is pulled by kind, so the worker is the one for the
|
||||||
|
// seat's first accepted kind (the build agent's seat, which had one worker for all, is gone with the builder).
|
||||||
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
|
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
// With an account to read, as a holder has: a module with none is no bus user, so its seat's workers are not derived.
|
||||||
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
|
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
|
||||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
|
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/late-builder/broker"}},
|
||||||
|
Claims: []catalogue.Claim{{Name: "node-build-go", Scope: catalogue.ScopeNode}}})
|
||||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
|
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
_, workers, err := seatTrafficObjects(t.Context(), open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
for _, s := range inventory.MeshSeats() {
|
for _, s := range inventory.MeshSeats() {
|
||||||
if s.Name == "node-build-agent" {
|
if s.Name != "node-build-go" || len(s.Accepts) == 0 {
|
||||||
c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
|
continue
|
||||||
if !needed {
|
|
||||||
t.Fatal("the build agent's seat needs no worker")
|
|
||||||
}
|
}
|
||||||
return c
|
for _, w := range workers {
|
||||||
|
if w.Name == broker.WorkerName(s.Name, s.Accepts[0]) {
|
||||||
|
return w
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
t.Fatal("the mesh declares no build agent's seat")
|
}
|
||||||
|
t.Fatal("the build seat's holder is derived no worker")
|
||||||
return broker.Consumer{}
|
return broker.Consumer{}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,43 +160,3 @@ func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
|||||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
|
||||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
|
||||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
|
||||||
url := testbus.URL(t)
|
|
||||||
open := aMesh(t)
|
|
||||||
js, err := broker.Dial(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
|
||||||
_ = js.Context().DeleteStream(s)
|
|
||||||
}
|
|
||||||
// The controller starting, before either was assigned.
|
|
||||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
consumer := aCarriedConsumer(t, open)
|
|
||||||
worker := aLateHolder(t, open)
|
|
||||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
|
||||||
for _, c := range []broker.Consumer{consumer, worker} {
|
|
||||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
|
||||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
server := link.ConnectNats(js, nil, nil)
|
|
||||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, c := range []broker.Consumer{consumer, worker} {
|
|
||||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
|
||||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
|
||||||
t.Fatal("a send that asserted everything raised a condition")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,103 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
|
||||||
//
|
|
||||||
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
|
||||||
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
|
||||||
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
|
||||||
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
|
||||||
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
|
||||||
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
|
||||||
// its gate starts as before.
|
|
||||||
|
|
||||||
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
|
||||||
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
|
||||||
var busyGateSlack = 5 * time.Minute
|
|
||||||
|
|
||||||
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
|
|
||||||
// controller's clock from when it first read the wait.
|
|
||||||
const busyEngineBound = 15 * time.Minute
|
|
||||||
|
|
||||||
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
|
|
||||||
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
|
|
||||||
// machine nor a wrong word from it holds a walk longer.
|
|
||||||
func busyGateEnds(b *inventory.GateBusy) time.Time {
|
|
||||||
until := b.Seen.Add(busyEngineBound)
|
|
||||||
if !b.Until.IsZero() && b.Until.Before(until) {
|
|
||||||
until = b.Until
|
|
||||||
}
|
|
||||||
return until.Add(busyGateSlack)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
|
||||||
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
|
||||||
// gate's Since to its end. Pure but for the gate it is handed.
|
|
||||||
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
|
||||||
if g.Busy != nil && g.Busy.Ended != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for _, j := range pairs {
|
|
||||||
b, waits := busyOn(j, f)
|
|
||||||
if !waits {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if g.Busy == nil {
|
|
||||||
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
|
|
||||||
Seen: now}
|
|
||||||
}
|
|
||||||
if g.Busy.Seen.IsZero() {
|
|
||||||
g.Busy.Seen = now
|
|
||||||
}
|
|
||||||
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
|
||||||
if now.After(busyGateEnds(g.Busy)) {
|
|
||||||
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
|
||||||
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
|
||||||
end := now
|
|
||||||
g.Busy.Ended, g.Busy.PastBound = &end, true
|
|
||||||
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
|
||||||
g.Read(now, false, g.Busy.Words())
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if g.Busy == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
// Replaced: the judging starts at the report that said so.
|
|
||||||
end := now
|
|
||||||
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
|
||||||
end = r.At.UTC()
|
|
||||||
}
|
|
||||||
g.Busy.Ended = &end
|
|
||||||
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
|
||||||
// running because it said busy.
|
|
||||||
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
|
||||||
r, said := f.reports[j.node]
|
|
||||||
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
|
||||||
return inventory.BusyWait{}, false
|
|
||||||
}
|
|
||||||
return inventory.WaitingFor(r.Busy, j.module)
|
|
||||||
}
|
|
||||||
|
|
||||||
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
|
||||||
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
|
||||||
func busyStill(j judged, f gateFacts) (string, bool) {
|
|
||||||
b, waits := busyOn(j, f)
|
|
||||||
if !waits {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
|
||||||
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
|
||||||
}
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
|
||||||
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
|
||||||
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
|
||||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
|
||||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
|
||||||
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
|
||||||
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
|
||||||
reported := sent.Add(20 * time.Second)
|
|
||||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
|
||||||
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
|
||||||
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
|
||||||
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
|
||||||
// As judgeMoves hands it: the report held against the gate's own send.
|
|
||||||
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
|
||||||
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
now := reported.Add(time.Minute)
|
|
||||||
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
|
||||||
t.Fatal("a gate whose module's container waits as busy judged it")
|
|
||||||
}
|
|
||||||
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
|
||||||
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
|
||||||
}
|
|
||||||
for _, said := range []string{gateLine(g), g.Last} {
|
|
||||||
if !strings.Contains(said, "waits for a busy container on novox") {
|
|
||||||
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(g.Last, "check of #280") {
|
|
||||||
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
|
||||||
now = reported.Add(10 * time.Minute)
|
|
||||||
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
|
||||||
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The machine reports it replaced: the judging starts at that report.
|
|
||||||
replacedAt := reported.Add(12 * time.Minute)
|
|
||||||
now = replacedAt.Add(5 * time.Second)
|
|
||||||
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
|
||||||
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
|
||||||
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
|
||||||
t.Fatal("the gate still waits after the container was replaced")
|
|
||||||
}
|
|
||||||
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
|
||||||
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
|
||||||
}
|
|
||||||
if strings.Contains(gateLine(g), "waits for a busy container") {
|
|
||||||
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
|
||||||
}
|
|
||||||
// And a later report listing it waiting again does not hold a gate that started.
|
|
||||||
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
|
||||||
t.Error("a gate that started waits again")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
|
||||||
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
|
||||||
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
|
||||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
|
||||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
|
||||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
|
||||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
|
||||||
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
|
||||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &sent, Current: true, Busy: waiting}}}
|
|
||||||
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
|
||||||
if busyWait(g, pairs, f, now) {
|
|
||||||
t.Fatal("a gate waited past the engine's bound and the slack")
|
|
||||||
}
|
|
||||||
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
|
||||||
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
|
||||||
}
|
|
||||||
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
|
||||||
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
|
||||||
// report not about the gate's send says nothing of it.
|
|
||||||
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
|
||||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
|
||||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
|
||||||
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
|
||||||
pairs := []judged{{module: "build-agent", node: "ace"}}
|
|
||||||
at := sent.Add(time.Minute)
|
|
||||||
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
|
||||||
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
|
||||||
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
|
||||||
}
|
|
||||||
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
|
||||||
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
|
||||||
if busyWait(g, pairs, elsewhere, at) {
|
|
||||||
t.Fatal("a wait reported about an earlier declaration held the gate")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
|
|
||||||
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
|
|
||||||
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
|
|
||||||
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
|
||||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
|
|
||||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
|
||||||
ahead := seen.Add(time.Hour)
|
|
||||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
|
||||||
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
|
||||||
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
|
|
||||||
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
|
|
||||||
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
|
|
||||||
}
|
|
||||||
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
|
|
||||||
t.Fatal("the gate stopped waiting within the bound on its own clock")
|
|
||||||
}
|
|
||||||
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
|
|
||||||
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
|
|
||||||
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
|
|
||||||
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
|
|
||||||
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
|
|
||||||
{"id": "agent-env", "type": "file"}}}}
|
|
||||||
lines := busyNotSent(modules, false)
|
|
||||||
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
|
|
||||||
!strings.Contains(lines[0], "replaced at once") {
|
|
||||||
t.Fatalf("the plan does not say the busy not sent: %v", lines)
|
|
||||||
}
|
|
||||||
if lines := busyNotSent(modules, true); len(lines) != 0 {
|
|
||||||
t.Fatalf("a busy sent is said as not sent: %v", lines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -24,28 +24,4 @@ func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
|
||||||
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
|
||||||
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
verb string
|
|
||||||
args map[string]any
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
|
||||||
{"push", map[string]any{"why": "w"}, true},
|
|
||||||
{"command", map[string]any{"command": "builds"}, false},
|
|
||||||
{"status", map[string]any{}, false},
|
|
||||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
|
||||||
} {
|
|
||||||
argv, err := argvFor(c.verb, c.args)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
|
||||||
}
|
|
||||||
if got := answersFirst(argv); got != c.want {
|
|
||||||
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||||
|
|||||||
@@ -1,643 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"path"
|
|
||||||
"regexp"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **The controller catches up on merges it did not hear** (novox/hq ADR 0457, issue 1336). A merge the forge announced while the
|
|
||||||
// controller was being replaced was never heard, and nothing asked for it again: the trunk held a merge no walk answered. At its
|
|
||||||
// start, and each time it hears a merge, the controller reads the last catchUpWindow merge commits of a trunk from the forge, and
|
|
||||||
// hears each one it holds no record of as if the forge had announced it then, on the warrant the router holds for it.
|
|
||||||
// It lasts until release 6 of design 47 deletes delivery plans and batches (ADR 0429 §7, §8).
|
|
||||||
|
|
||||||
// catchUpWindow is how many of a trunk's newest merge commits are read; an older merge is a person's to decide.
|
|
||||||
const catchUpWindow = 50
|
|
||||||
|
|
||||||
// kindMergeNotHeard is the condition of a merge the controller did not hear and holds no warrant to plan.
|
|
||||||
const kindMergeNotHeard = "merge-not-heard"
|
|
||||||
|
|
||||||
// trunkMerge is a merge commit of a trunk as the forge lists it.
|
|
||||||
type trunkMerge struct {
|
|
||||||
Commit string
|
|
||||||
// Head is the merge commit's second parent: the pull request's head that was merged.
|
|
||||||
Head string
|
|
||||||
Subject string
|
|
||||||
At time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// trunkCommit is a commit of a trunk as the forge lists it, newest first: a merge commit has a second parent.
|
|
||||||
type trunkCommit struct {
|
|
||||||
Commit, SecondParent, Subject string
|
|
||||||
At time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// trunkCommitsPage is one page (fifty) of a trunk's commits, newest first; empty past the last. A variable so a test needs no
|
|
||||||
// forge.
|
|
||||||
var trunkCommitsPage = func(ctx context.Context, owner, repo, branch string, page int) ([]trunkCommit, error) {
|
|
||||||
var found struct {
|
|
||||||
Result []struct {
|
|
||||||
SHA string `json:"sha"`
|
|
||||||
Parents []struct {
|
|
||||||
SHA string `json:"sha"`
|
|
||||||
} `json:"parents"`
|
|
||||||
Commit struct {
|
|
||||||
Message string `json:"message"`
|
|
||||||
Committer struct {
|
|
||||||
Date string `json:"date"`
|
|
||||||
} `json:"committer"`
|
|
||||||
} `json:"commit"`
|
|
||||||
} `json:"result"`
|
|
||||||
}
|
|
||||||
apiPath := fmt.Sprintf("/repos/%s/%s/commits?sha=%s&limit=50&page=%d&stat=false&verification=false&files=false",
|
|
||||||
owner, repo, url.QueryEscape(branch), page)
|
|
||||||
missing, err := askForgeApi(ctx, apiPath, &found)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if missing {
|
|
||||||
return nil, fmt.Errorf("the forge holds no branch %s of %s/%s", branch, owner, repo)
|
|
||||||
}
|
|
||||||
var out []trunkCommit
|
|
||||||
for _, c := range found.Result {
|
|
||||||
at, _ := time.Parse(time.RFC3339, c.Commit.Committer.Date)
|
|
||||||
tc := trunkCommit{Commit: c.SHA, Subject: firstLine(c.Commit.Message), At: at.UTC()}
|
|
||||||
if len(c.Parents) >= 2 {
|
|
||||||
tc.SecondParent = c.Parents[1].SHA
|
|
||||||
}
|
|
||||||
out = append(out, tc)
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readTrunkMerges is the last catchUpWindow merge commits of a repository's trunk, oldest first: pages are read until that many
|
|
||||||
// merges are held or the trunk ends.
|
|
||||||
func readTrunkMerges(ctx context.Context, owner, repo, branch string) ([]trunkMerge, error) {
|
|
||||||
var merges []trunkMerge
|
|
||||||
for page := 1; page <= 8 && len(merges) < catchUpWindow; page++ {
|
|
||||||
commits, err := trunkCommitsPage(ctx, owner, repo, branch, page)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(commits) == 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
for _, c := range commits {
|
|
||||||
// A merge of the trunk is a pull request's merge into it. A commit with a second parent that is none is a merge made
|
|
||||||
// inside some branch ("Merge remote-tracking branch 'origin/main' into fix/..."), which the trunk's history lists too and
|
|
||||||
// which no Approve ever covered: read as a gap it would raise a condition for nothing a person decides.
|
|
||||||
if c.SecondParent != "" && mergesIntoTrunk(c.Subject, branch) {
|
|
||||||
merges = append(merges, trunkMerge{Commit: c.Commit, Head: c.SecondParent, Subject: c.Subject, At: c.At})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return lastMerges(merges), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sinceTheRecord are the merges that can be gaps (novox/hq ADR 0463 §2): those made since the controller's record of merges begins on
|
|
||||||
// the mesh (first, the earliest merge it holds of any repository and branch, less mergeGrace as owedLate judges). A merge before it is
|
|
||||||
// history on every trunk, and a trunk with no merge kept of its own still has the merges made since as gaps. With no merge kept
|
|
||||||
// anywhere there is no record to have a gap in.
|
|
||||||
func sinceTheRecord(merges []trunkMerge, kept bool, first time.Time) []trunkMerge {
|
|
||||||
if !kept {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var out []trunkMerge
|
|
||||||
for _, m := range merges {
|
|
||||||
if !m.At.Before(first.Add(-mergeGrace)) {
|
|
||||||
out = append(out, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastMerges is the newest catchUpWindow of merges, given newest first, oldest first.
|
|
||||||
func lastMerges(newestFirst []trunkMerge) []trunkMerge {
|
|
||||||
if len(newestFirst) > catchUpWindow {
|
|
||||||
newestFirst = newestFirst[:catchUpWindow]
|
|
||||||
}
|
|
||||||
out := slices.Clone(newestFirst)
|
|
||||||
slices.Reverse(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
var mergeSubject = regexp.MustCompile(`^Merge pull request '(.*)' \(#(\d+)\) from (\S+) into (\S+)$`)
|
|
||||||
|
|
||||||
// pullOfSubject is the pull request a merge commit's subject names: its title and number.
|
|
||||||
func pullOfSubject(subject string) (title string, number int, ok bool) {
|
|
||||||
m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject))
|
|
||||||
if m == nil {
|
|
||||||
return "", 0, false
|
|
||||||
}
|
|
||||||
n, err := strconv.Atoi(m[2])
|
|
||||||
if err != nil || n <= 0 {
|
|
||||||
return "", 0, false
|
|
||||||
}
|
|
||||||
return m[1], n, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// mergesIntoTrunk says a commit's subject is a pull request's merge into this trunk.
|
|
||||||
func mergesIntoTrunk(subject, branch string) bool {
|
|
||||||
m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject))
|
|
||||||
return m != nil && m[4] == branch
|
|
||||||
}
|
|
||||||
|
|
||||||
// headBranchOfSubject is the branch a merge commit's subject says was merged.
|
|
||||||
func headBranchOfSubject(subject string) string {
|
|
||||||
if m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)); m != nil {
|
|
||||||
return m[3]
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// readMergeEvent is the merge as the forge would have announced it: the commit, the pull request, the files it changed and the
|
|
||||||
// directories among them that hold a module.json. A variable so a test needs no forge.
|
|
||||||
var readMergeEvent = func(ctx context.Context, owner, repo, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
|
||||||
m := link.SourceMoved{Owner: owner, Repo: repo, Base: branch, Head: headBranchOfSubject(tm.Subject), HeadSHA: tm.Head,
|
|
||||||
Commit: tm.Commit, MergedAt: tm.At.UTC().Format(time.RFC3339)}
|
|
||||||
if title, number, ok := pullOfSubject(tm.Subject); ok {
|
|
||||||
m.Number, m.Title = number, title
|
|
||||||
}
|
|
||||||
var found struct {
|
|
||||||
Result struct {
|
|
||||||
Files []struct {
|
|
||||||
Filename string `json:"filename"`
|
|
||||||
Status string `json:"status"`
|
|
||||||
} `json:"files"`
|
|
||||||
} `json:"result"`
|
|
||||||
}
|
|
||||||
missing, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(tm.Commit), &found)
|
|
||||||
if err != nil {
|
|
||||||
return m, err
|
|
||||||
}
|
|
||||||
if missing || len(found.Result.Files) == 0 {
|
|
||||||
return m, fmt.Errorf("the forge named no files for the merge %.8s of %s/%s", tm.Commit, owner, repo)
|
|
||||||
}
|
|
||||||
for _, f := range found.Result.Files {
|
|
||||||
m.Paths = append(m.Paths, f.Filename)
|
|
||||||
if f.Status == "removed" || f.Status == "deleted" {
|
|
||||||
m.Removed = append(m.Removed, f.Filename)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The forge lists a merge's files by the page; a list as long as a page may be a beginning.
|
|
||||||
m.PathsTruncated = len(m.Paths) >= 300
|
|
||||||
dirs := map[string]bool{}
|
|
||||||
for _, p := range m.Paths {
|
|
||||||
for d := path.Dir(p); ; d = path.Dir(d) {
|
|
||||||
dirs[d] = true
|
|
||||||
if d == "." || d == "/" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(dirs) > 64 {
|
|
||||||
return m, nil // too many to look at: the old rule stands, a directory is a module when its manifest changed
|
|
||||||
}
|
|
||||||
var holding []string
|
|
||||||
for d := range dirs {
|
|
||||||
file := "module.json"
|
|
||||||
if d != "." {
|
|
||||||
file = d + "/module.json"
|
|
||||||
}
|
|
||||||
_, exists, err := manifestCached(ctx, owner, repo, file, tm.Commit)
|
|
||||||
if err != nil {
|
|
||||||
return m, err
|
|
||||||
}
|
|
||||||
if exists {
|
|
||||||
holding = append(holding, d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(holding)
|
|
||||||
m.ModuleDirs, m.ModuleDirsSaid = holding, true
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mergeAskPrefix is what the id of a merge ask made by the retired module mesh-delivery begins with: the pull request's number
|
|
||||||
// and the head's first twelve digits (`merge-<number>-<head>-<try>`). The teller's ids name neither (see findMergeWarrant).
|
|
||||||
func mergeAskPrefix(number int, head string) string {
|
|
||||||
if len(head) > 12 {
|
|
||||||
head = head[:12]
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("merge-%d-%s-", number, head)
|
|
||||||
}
|
|
||||||
|
|
||||||
// askersOfMerges are the askers a merge's warrant is read under: the teller alone (ADR 0437 §6), since R4e stopped taking
|
|
||||||
// mesh-delivery's (ADR 0450 §8; issue 1318).
|
|
||||||
func askersOfMerges() []string { return []string{sdkdelivery.TellerModule} }
|
|
||||||
|
|
||||||
// caughtUpWarrant judges a chosen merge ask of the router as the warrant of a merge: the ask is the operator's, the warrant
|
|
||||||
// answers that very ask as it was published, and it is the operator's Approve proved by the channel it came through (ADR 0437
|
|
||||||
// §6, as movedOnApprove judges an announced one). The heads the Approve was bound to are checked by the caller, where it can
|
|
||||||
// name them.
|
|
||||||
func caughtUpWarrant(asker string, ask *asks.Ask, w *asks.Warrant) (string, error) {
|
|
||||||
if err := askerAccepted(asker); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case ask == nil || w == nil:
|
|
||||||
return "", errors.New("the router holds no closed record of the ask")
|
|
||||||
case ask.Who != asks.Operator:
|
|
||||||
return "", fmt.Errorf("the ask %s is for %q to answer, not the operator", ask.ID, ask.Who)
|
|
||||||
}
|
|
||||||
option, err := w.For(asker, *ask)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if option.ID != optionApprove || w.Option != optionApprove {
|
|
||||||
return "", fmt.Errorf("the operator chose %q on the ask %s, not an Approve", w.Option, w.Ask)
|
|
||||||
}
|
|
||||||
if w.By == nil || w.By.Who != asks.Operator || w.By.Verified == "" {
|
|
||||||
return "", fmt.Errorf("the ask %s was not answered by the operator on a channel that proves who answered", w.Ask)
|
|
||||||
}
|
|
||||||
return w.Says() + " (ask " + w.Ask + ")", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// routerAskIDs are the ids of the asks an asker has in the router's record. A variable so a test needs no bus.
|
|
||||||
var routerAskIDs = func(ctx context.Context, inv catalogueReader, asker string) ([]string, error) {
|
|
||||||
serving := servingBus.Load()
|
|
||||||
if serving == nil {
|
|
||||||
return nil, errors.New("only the serving controller reads the router's record")
|
|
||||||
}
|
|
||||||
bucket, err := asksRecords(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if bucket == "" {
|
|
||||||
return nil, errors.New("no module declares the operator channel's records")
|
|
||||||
}
|
|
||||||
api, err := jetstream.New(serving.Conn())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
kv, err := api.KeyValue(ctx, bucket)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return askIDsOf(ctx, kv, asker)
|
|
||||||
}
|
|
||||||
|
|
||||||
// askIDsOf are the ids of the asks an asker has in the router's bucket.
|
|
||||||
func askIDsOf(ctx context.Context, kv jetstream.KeyValue, asker string) ([]string, error) {
|
|
||||||
lister, err := kv.ListKeysFiltered(ctx, asker+".>")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var ids []string
|
|
||||||
for key := range lister.Keys() {
|
|
||||||
ids = append(ids, strings.TrimPrefix(key, asker+"."))
|
|
||||||
}
|
|
||||||
// The lister stops its own watch when it has sent the last key, so a stop of ours finds it stopped: nats answers that with
|
|
||||||
// "invalid subscription", which is no failure of the read (issue 1336). Any other error stays one.
|
|
||||||
if err := lister.Stop(); err != nil && !errors.Is(err, nats.ErrBadSubscription) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return ids, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// closedRecord is a router record of an ask that ended: it never changes, so it is read once.
|
|
||||||
type closedRecord struct {
|
|
||||||
ask *asks.Ask
|
|
||||||
w *asks.Warrant
|
|
||||||
}
|
|
||||||
|
|
||||||
var closedRecords sync.Map // asker + "." + id -> closedRecord
|
|
||||||
|
|
||||||
// closedRecordOf is the router's record of an ask, kept once it is closed. A nil ask is an ask still open, ended without a
|
|
||||||
// choice, or not there.
|
|
||||||
func closedRecordOf(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
|
||||||
if v, ok := closedRecords.Load(asker + "." + id); ok {
|
|
||||||
r := v.(closedRecord)
|
|
||||||
return r.ask, r.w, nil
|
|
||||||
}
|
|
||||||
ask, w, err := routerAskOf(ctx, inv, asker, id)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if ask != nil && w != nil {
|
|
||||||
closedRecords.Store(asker+"."+id, closedRecord{ask, w})
|
|
||||||
}
|
|
||||||
return ask, w, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryIDOf is the id the gate gives a delivery, and so the key of its head in the heads an Approve binds: the repository as
|
|
||||||
// owner/name, lower-cased, `@` and the first twelve characters of the head.
|
|
||||||
func deliveryIDOf(owner, repo, head string) string {
|
|
||||||
if len(head) > 12 {
|
|
||||||
head = head[:12]
|
|
||||||
}
|
|
||||||
return strings.ToLower(owner+"/"+repo) + "@" + head
|
|
||||||
}
|
|
||||||
|
|
||||||
// coversHead says an ask the operator approved is for this merge: its Approve is bound to the heads of the pull request alone
|
|
||||||
// (mergeAct over {delivery id: head}, which is how a pull request not in a group is asked), or, for a group's ask, whose other
|
|
||||||
// members' heads the router's record does not carry in full, the ask as published names this pull request at this head in its
|
|
||||||
// details ("<repo>#<number> ... at <head>:"), and the digest the warrant answers covers those words.
|
|
||||||
func coversHead(ask *asks.Ask, owner, repo string, number int, head string) (how string, ok bool) {
|
|
||||||
option, found := ask.Option(optionApprove)
|
|
||||||
if !found {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
heads := map[string]string{deliveryIDOf(owner, repo, head): head}
|
|
||||||
if option.Performs(mergeAct(ask.ID, optionApprove, heads)) == nil {
|
|
||||||
return "bound to the head of this pull request", true
|
|
||||||
}
|
|
||||||
short := head
|
|
||||||
if len(short) > 12 {
|
|
||||||
short = short[:12]
|
|
||||||
}
|
|
||||||
line := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(repo) + `#` + strconv.Itoa(number) + ` .* at ` + regexp.QuoteMeta(short) + `:`)
|
|
||||||
if line.MatchString(ask.Details) {
|
|
||||||
return "a group's ask naming this pull request at this head", true
|
|
||||||
}
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
// findMergeWarrant finds the warrant of a merge, with no read the controller does not make today: the router's chosen merge
|
|
||||||
// ask for it. Under the teller (releases 3e on) the id is `merge-<group>-<heads hash>-<tail>`, which names neither the pull
|
|
||||||
// request nor the head, so the teller's closed asks are read and the one the operator approved whose heads cover the merge's
|
|
||||||
// second parent is the warrant (coversHead). Under mesh-delivery, which made the earlier merges, the id names the pull request
|
|
||||||
// and the head and is found by them. Why it holds; or found false with why not, for a person to read; or an error when the
|
|
||||||
// router could not be read, which is no answer. A variable so a test needs no bus.
|
|
||||||
var findMergeWarrant = func(ctx context.Context, inv catalogueReader, owner, repo string, number int, head string) (why string, found bool, reason string, err error) {
|
|
||||||
reason = "the router holds no chosen merge ask of " + strings.Join(askersOfMerges(), " or ") + " that covers the head " + head[:min(12, len(head))]
|
|
||||||
for _, asker := range askersOfMerges() {
|
|
||||||
ids, err := routerAskIDs(ctx, inv, asker)
|
|
||||||
if err != nil {
|
|
||||||
return "", false, "", err
|
|
||||||
}
|
|
||||||
for _, id := range ids {
|
|
||||||
named := number > 0 && strings.HasPrefix(id, mergeAskPrefix(number, head))
|
|
||||||
if asker == sdkdelivery.TellerModule && !strings.HasPrefix(id, sdkdelivery.AskKindMerge+"-") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ask, w, err := closedRecordOf(ctx, inv, asker, id)
|
|
||||||
if err != nil {
|
|
||||||
return "", false, "", err
|
|
||||||
}
|
|
||||||
if ask == nil || w == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
warrant, err := caughtUpWarrant(asker, ask, w)
|
|
||||||
if err != nil {
|
|
||||||
if named {
|
|
||||||
reason = fmt.Sprintf("%s's ask %s holds no operator's Approve: %v", asker, id, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
how, covers := coversHead(ask, owner, repo, number, head)
|
|
||||||
if !covers && !named {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !covers {
|
|
||||||
how = "the ask's id names this pull request and head"
|
|
||||||
}
|
|
||||||
return warrant + ", " + how, true, "", nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", false, reason, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// caughtTrunk is one trunk the catalogue builds from.
|
|
||||||
type caughtTrunk struct{ Owner, Repo, Branch string }
|
|
||||||
|
|
||||||
func (t caughtTrunk) repository() string { return t.Owner + "/" + t.Repo }
|
|
||||||
|
|
||||||
// trunksOfCatalogue are the trunks of the repositories on the mesh's forge the catalogue builds a module from.
|
|
||||||
func trunksOfCatalogue(forms *sourceForms, entries []inventory.Entry) []caughtTrunk {
|
|
||||||
var out []caughtTrunk
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Provided || e.Source.Repository == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
owner, name, onForge := forms.onTheForge(e.Source.Repository, e.Source.Seat)
|
|
||||||
if !onForge {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
branch := followedBranch(e.Source.Ref)
|
|
||||||
if branch == "" {
|
|
||||||
branch = "main"
|
|
||||||
}
|
|
||||||
t := caughtTrunk{owner, name, branch}
|
|
||||||
if !slices.Contains(out, t) {
|
|
||||||
out = append(out, t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// catchUpBeside runs a catch-up beside the hearing of the merge that asked for it, so the bus's acknowledgement of that merge
|
|
||||||
// waits for no forge. A variable so a test runs it where it stands.
|
|
||||||
var catchUpBeside = func(run func()) { go run() }
|
|
||||||
|
|
||||||
// catchingUp makes one catch-up at a time.
|
|
||||||
var catchingUp sync.Mutex
|
|
||||||
|
|
||||||
// settledMerges are the merges this process found nothing to plan for: heard, and no record kept, because nothing reads them.
|
|
||||||
// They are not read from the forge again.
|
|
||||||
var settledMerges sync.Map
|
|
||||||
|
|
||||||
// catchUp hears the merges of the catalogue's trunks (only one, when only names it) that the controller holds no record of.
|
|
||||||
// It says whether a trunk was left for later: the forge, the router or the store did not answer.
|
|
||||||
func (f following) catchUp(ctx context.Context, only *caughtTrunk) (later bool) {
|
|
||||||
catchingUp.Lock()
|
|
||||||
defer catchingUp.Unlock()
|
|
||||||
inv := f.open.inventory
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: the catalogue cannot be read, so no merge is caught up now: %v\n", err)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
kept, first, err := inv.EarliestKept(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: what the controller kept cannot be read, so no merge is caught up now: %v\n", err)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if kept {
|
|
||||||
fmt.Printf("catch-up: the controller's record of merges begins %s; a merge made before it, less %s, is history on every trunk\n",
|
|
||||||
first.UTC().Format(time.RFC3339), mergeGrace)
|
|
||||||
} else {
|
|
||||||
fmt.Println("catch-up: the controller holds no merge of any trunk yet, so none can have been missed")
|
|
||||||
}
|
|
||||||
trunks := trunksOfCatalogue(newSourceForms(ctx, inv), entries)
|
|
||||||
if only != nil {
|
|
||||||
trunks = slices.DeleteFunc(trunks, func(t caughtTrunk) bool {
|
|
||||||
return !strings.EqualFold(t.repository(), only.repository()) || t.Branch != only.Branch
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for _, t := range trunks {
|
|
||||||
if f.catchUpTrunk(ctx, t, kept, first) {
|
|
||||||
later = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return later
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f following) catchUpTrunk(ctx context.Context, t caughtTrunk, kept bool, first time.Time) (later bool) {
|
|
||||||
inv := f.open.inventory
|
|
||||||
repository := t.repository()
|
|
||||||
merges, err := readTrunkMerges(ctx, t.Owner, t.Repo, t.Branch)
|
|
||||||
if err != nil {
|
|
||||||
// Not nothing to catch up: left for the next start or the next heard merge (ADR 0457 clause 3).
|
|
||||||
fmt.Printf("catch-up: the forge did not answer for the last merges of %s %s, so they are not caught up now; the next start "+
|
|
||||||
"or the next merge heard there does it: %v\n", repository, t.Branch, err)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// **A merge made before the controller's record of merges begins is history, never a gap** (novox/hq ADR 0463 §2).
|
|
||||||
read := len(merges)
|
|
||||||
merges = sinceTheRecord(merges, kept, first)
|
|
||||||
fmt.Printf("catch-up: %s %s: %d of the last %d pull request merges were made since the record began\n", repository, t.Branch,
|
|
||||||
len(merges), read)
|
|
||||||
unwarranted := map[string]bool{}
|
|
||||||
for _, tm := range merges {
|
|
||||||
id := fmt.Sprintf("%s_%.8s", repository, tm.Commit)
|
|
||||||
if _, done := settledMerges.Load(repository + "@" + tm.Commit); done {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err != nil {
|
|
||||||
fmt.Printf("catch-up: the record of %s cannot be read, so %s is not caught up now: %v\n", repository, tm.Commit[:8], err)
|
|
||||||
return true
|
|
||||||
} else if known {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
m, err := readMergeEvent(ctx, t.Owner, t.Repo, t.Branch, tm)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: the merge %.8s of %s cannot be read from the forge, so it is not caught up now: %v\n", tm.Commit, repository, err)
|
|
||||||
later = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
entries, err := inv.Catalogued(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
read, err := readForPlanning(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
facts, err := factsOfMerge(ctx, inv, m, time.Now().UTC(), entries, read)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: %s %.8s cannot be judged now: %v\n", repository, tm.Commit, err)
|
|
||||||
later = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// The very test hearMerge makes: a merge it would plan, or take as a first build, is a gap; any other it would say
|
|
||||||
// nothing reads, and is no gap for a person to decide.
|
|
||||||
if !facts.plans() && !facts.addsModuleToBuilds {
|
|
||||||
settledMerges.Store(repository+"@"+tm.Commit, true)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
why, found, reason, err := findMergeWarrant(ctx, inv, t.Owner, t.Repo, m.Number, tm.Head)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: the router cannot be read, so %s %.8s is not caught up now: %v\n", repository, tm.Commit, err)
|
|
||||||
later = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
unwarranted[id] = true
|
|
||||||
f.raiseMergeNotHeard(ctx, repository, t.Branch, tm, m, reason)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
now := time.Now().UTC()
|
|
||||||
said := fmt.Sprintf("caught up: merged %s, not heard, planned %s", tm.At.Format(time.RFC3339), now.Format(time.RFC3339))
|
|
||||||
fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard; %s; on the warrant %s\n", repository, t.Branch, tm.Commit, said, why)
|
|
||||||
m.Title = strings.TrimSpace(m.Title + " (" + said + ")")
|
|
||||||
if err := f.hearMerge(ctx, m, now); err != nil {
|
|
||||||
fmt.Printf("catch-up: %s %.8s could not be heard now: %v\n", repository, tm.Commit, err)
|
|
||||||
later = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err == nil && !known {
|
|
||||||
settledMerges.Store(repository+"@"+tm.Commit, true) // heard, and nothing reads it
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Cleared only when every gap was judged: one left for later may still be open.
|
|
||||||
if !later {
|
|
||||||
f.clearMergesHeard(ctx, repository, unwarranted)
|
|
||||||
}
|
|
||||||
return later
|
|
||||||
}
|
|
||||||
|
|
||||||
// raiseMergeNotHeard keeps open, and observes again, the condition of a merge with no warrant to plan it. One for each
|
|
||||||
// repository and commit, however many catch-ups find it.
|
|
||||||
func (f following) raiseMergeNotHeard(ctx context.Context, repository, branch string, tm trunkMerge, m link.SourceMoved, reason string) {
|
|
||||||
fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard, and is not planned: %s\n", repository, branch, tm.Commit, reason)
|
|
||||||
if conditionsFrom == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, tm.Commit), Kind: kindMergeNotHeard,
|
|
||||||
Severity: conditions.Warning, Source: "catch-up", Resolver: conditions.ResolverOperator,
|
|
||||||
Summary: fmt.Sprintf("the merge %.8s of %s into %s was not heard by the controller, and no operator's Approve of it is on "+
|
|
||||||
"record to plan it on: a person decides", tm.Commit, repository, branch),
|
|
||||||
Said: fmt.Sprintf("%s %.8s (pull request %d, head %.8s, merged %s): %s", repository, tm.Commit, m.Number, tm.Head,
|
|
||||||
tm.At.Format(time.RFC3339), reason),
|
|
||||||
Headline: fmt.Sprintf("A merge into %s was never delivered", repository),
|
|
||||||
Explanation: fmt.Sprintf("A merge into %s was made while the controller did not hear it, and no Approve of yours for it is on "+
|
|
||||||
"record that the controller can plan it on. It stays open while the merge is among the latest %d merges of that repository. "+
|
|
||||||
"The details say what was looked for.", repository, catchUpWindow),
|
|
||||||
Needs: "decide whether that merge should be delivered, and say so if it should.",
|
|
||||||
Resolved: "the merge is planned, or is no longer among the latest merges of its repository"}
|
|
||||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
|
||||||
fmt.Printf("catch-up: the condition of %s %.8s could not be kept: %v\n", repository, tm.Commit, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// clearMergesHeard clears the open conditions of a repository's merges not heard that this catch-up did not find again: the
|
|
||||||
// gap was planned, or left the window.
|
|
||||||
func (f following) clearMergesHeard(ctx context.Context, repository string, still map[string]bool) {
|
|
||||||
if conditionsFrom == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
open, err := conditionsFrom.Open(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("catch-up: the open conditions cannot be read, so none is cleared now: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, c := range open {
|
|
||||||
if c.Kind != kindMergeNotHeard || c.Subject.Scope != conditions.ScopeMerge ||
|
|
||||||
!strings.HasPrefix(c.Subject.ID, repository+"_") || still[c.Subject.ID] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err := conditionsFrom.Clear(ctx, c.Key, "the merge is planned, or left the last merges of its trunk"); err != nil {
|
|
||||||
fmt.Printf("catch-up: the condition %s could not be cleared: %v\n", c.Key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// catchUpAtStart catches up every trunk once the controller serves, and again every minute while a trunk was left for later.
|
|
||||||
func catchUpAtStart(ctx context.Context, open *stores) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(30 * time.Second):
|
|
||||||
}
|
|
||||||
for try := 0; try < 20; try++ {
|
|
||||||
if !(following{open: open}).catchUp(ctx, nil) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(time.Minute):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Println("catch-up: a trunk is still left for later after twenty tries; the next merge heard there does it")
|
|
||||||
}
|
|
||||||
@@ -1,352 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The checks of ADR 0457 (issue 1336): what the controller does about merges it did not hear.
|
|
||||||
|
|
||||||
// aTrunk is a forge's trunk as the catch-up reads it: its merges oldest first, the module each moves, and which of them the
|
|
||||||
// router holds an operator's Approve for.
|
|
||||||
type aTrunk struct {
|
|
||||||
merges []trunkMerge
|
|
||||||
module map[string]string
|
|
||||||
approved map[string]bool
|
|
||||||
read []string // the trunks asked of the forge, as owner/repo
|
|
||||||
events int // merges read in full from the forge
|
|
||||||
silent bool // the forge does not answer
|
|
||||||
}
|
|
||||||
|
|
||||||
func aTrunkOf(t *testing.T, repo string, modules ...string) *aTrunk {
|
|
||||||
t.Helper()
|
|
||||||
tr := &aTrunk{module: map[string]string{}, approved: map[string]bool{}}
|
|
||||||
for i, m := range modules {
|
|
||||||
c := fmt.Sprintf("%s-c%02d", repo, i+1)
|
|
||||||
tr.merges = append(tr.merges, trunkMerge{Commit: c, Head: fmt.Sprintf("%shead%02d", repo, i+1),
|
|
||||||
Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from pr/%s into main", i+1, 100+i, c),
|
|
||||||
At: t0.Add(time.Duration(i) * time.Minute)})
|
|
||||||
tr.module[c] = m
|
|
||||||
tr.approved[c] = true
|
|
||||||
}
|
|
||||||
was, wasEvent, wasWarrant := trunkCommitsPage, readMergeEvent, findMergeWarrant
|
|
||||||
trunkCommitsPage = func(_ context.Context, owner, name, _ string, page int) ([]trunkCommit, error) {
|
|
||||||
tr.read = append(tr.read, owner+"/"+name)
|
|
||||||
if tr.silent {
|
|
||||||
return nil, errors.New("the forge did not answer")
|
|
||||||
}
|
|
||||||
if page > 1 || owner+"/"+name != "novox/"+repo {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
var newest []trunkCommit
|
|
||||||
for i := len(tr.merges) - 1; i >= 0; i-- {
|
|
||||||
m := tr.merges[i]
|
|
||||||
newest = append(newest, trunkCommit{Commit: m.Commit, SecondParent: m.Head, Subject: m.Subject, At: m.At})
|
|
||||||
}
|
|
||||||
return newest, nil
|
|
||||||
}
|
|
||||||
readMergeEvent = func(_ context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
|
||||||
tr.events++
|
|
||||||
m := catalogueMerge(tm.Commit, tr.module[tm.Commit], tm.At)
|
|
||||||
m.Owner, m.Repo, m.Base, m.HeadSHA = owner, name, branch, tm.Head
|
|
||||||
_, m.Number, _ = pullOfSubject(tm.Subject)
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
findMergeWarrant = func(_ context.Context, _ catalogueReader, _, _ string, number int, head string) (string, bool, string, error) {
|
|
||||||
for _, m := range tr.merges {
|
|
||||||
if strings.Contains(m.Subject, fmt.Sprintf("(#%d)", number)) && m.Head == head && tr.approved[m.Commit] {
|
|
||||||
return "the operator chose Approve (ask merge-" + fmt.Sprint(number) + ")", true, "", nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", false, "the router holds no chosen merge ask for the pull request and head", nil
|
|
||||||
}
|
|
||||||
wasBeside := catchUpBeside
|
|
||||||
catchUpBeside = func(run func()) { run() }
|
|
||||||
t.Cleanup(func() {
|
|
||||||
trunkCommitsPage, readMergeEvent, findMergeWarrant, catchUpBeside = was, wasEvent, wasWarrant, wasBeside
|
|
||||||
settledMerges.Clear()
|
|
||||||
})
|
|
||||||
return tr
|
|
||||||
}
|
|
||||||
|
|
||||||
// aKeptMergeBefore is a merge the controller heard before the merges of a test: where its record begins, so what a trunk holds from
|
|
||||||
// then on can be a gap (novox/hq ADR 0463 §2).
|
|
||||||
func aKeptMergeBefore(t *testing.T, open *stores) {
|
|
||||||
t.Helper()
|
|
||||||
hear(t, open, catalogueMerge("mesh-catalog-seed", "app", t0.Add(-time.Hour)), t0.Add(-time.Hour))
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeKnown(t *testing.T, open *stores, commit string) bool {
|
|
||||||
t.Helper()
|
|
||||||
_, known, err := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", commit)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return known
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 1: the trunk holds merges 1, 2 and 3 and the controller holds records of 1 and 3 only; at its start it plans 2, said
|
|
||||||
// as caught up; a second start plans nothing new.
|
|
||||||
func TestAGapBeforeANewerHeardMergeIsPlannedAtTheControllersStart(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
|
|
||||||
hear(t, open, catalogueMerge("mesh-catalog-c01", "app", t0), t0.Add(time.Second))
|
|
||||||
hear(t, open, catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)), t0.Add(2*time.Minute+time.Second))
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c02") {
|
|
||||||
t.Fatal("the merge to be caught up is known before the catch-up")
|
|
||||||
}
|
|
||||||
if (following{open: open}).catchUp(t.Context(), nil) {
|
|
||||||
t.Fatal("the catch-up says a trunk was left for later")
|
|
||||||
}
|
|
||||||
if !mergeKnown(t, open, "mesh-catalog-c02") {
|
|
||||||
t.Fatal("the merge the controller did not hear was not planned")
|
|
||||||
}
|
|
||||||
kept, _, _ := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", "mesh-catalog-c02")
|
|
||||||
if said := announcedOf(kept).Title; !strings.Contains(said, "caught up: merged ") || !strings.Contains(said, "not heard, planned ") {
|
|
||||||
t.Fatalf("the merge does not say it was caught up: %q", said)
|
|
||||||
}
|
|
||||||
cutAt(t, open, t0.Add(10*time.Minute))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) == 0 {
|
|
||||||
t.Fatal("no walk answers the caught-up merge")
|
|
||||||
}
|
|
||||||
if _, in := ws[0].Modules["notes"]; !in {
|
|
||||||
t.Fatalf("the walk does not build notes, which the caught-up merge moved: %v", ws[0].Modules)
|
|
||||||
}
|
|
||||||
before := tr.events
|
|
||||||
if (following{open: open}).catchUp(t.Context(), nil) || tr.events != before {
|
|
||||||
t.Fatalf("a second start read %d merge(s) in full again", tr.events-before)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 1: a heard merge of a repository whose trunk holds earlier merges the controller has no record of plans them all, the
|
|
||||||
// earlier first, and the forge is read for that repository's trunk only.
|
|
||||||
func TestAHeardMergeCatchesUpTheEarlierOnesOfItsOwnTrunkOnly(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
|
|
||||||
var order []string
|
|
||||||
inner := readMergeEvent
|
|
||||||
readMergeEvent = func(ctx context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
|
||||||
order = append(order, tm.Commit)
|
|
||||||
return inner(ctx, owner, name, branch, tm)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { readMergeEvent = inner })
|
|
||||||
tr.read = nil
|
|
||||||
heard := catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute))
|
|
||||||
if err := (following{open: open}).SourceMoved(t.Context(), heard); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, c := range []string{"mesh-catalog-c01", "mesh-catalog-c02", "mesh-catalog-c03"} {
|
|
||||||
if !mergeKnown(t, open, c) {
|
|
||||||
t.Fatalf("%s is not planned: the heard merge and the earlier ones of its trunk are all wanted", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(order) != 2 || order[0] != "mesh-catalog-c01" || order[1] != "mesh-catalog-c02" {
|
|
||||||
t.Fatalf("the earlier merges were heard in the order %v, not the trunk's", order)
|
|
||||||
}
|
|
||||||
if len(tr.read) == 0 {
|
|
||||||
t.Fatal("the forge was not read for the trunk of the merge heard")
|
|
||||||
}
|
|
||||||
for _, r := range tr.read {
|
|
||||||
if r != "novox/mesh-catalog" {
|
|
||||||
t.Fatalf("the forge was read for %s, not only for the trunk of the merge heard: %v", r, tr.read)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 1: a trunk of fifty-one merges whose oldest is unheard plans nothing: the oldest is outside the window.
|
|
||||||
func TestAMergeOlderThanTheLastFiftyIsNotRead(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
modules := make([]string, 51)
|
|
||||||
for i := range modules {
|
|
||||||
modules[i] = "app"
|
|
||||||
}
|
|
||||||
aTrunkOf(t, "mesh-catalog", modules...)
|
|
||||||
// Fifty are known; the oldest is not.
|
|
||||||
for i := 2; i <= 51; i++ {
|
|
||||||
c := fmt.Sprintf("mesh-catalog-c%02d", i)
|
|
||||||
hear(t, open, catalogueMerge(c, "app", t0.Add(time.Duration(i)*time.Minute)), t0.Add(time.Duration(i)*time.Minute+time.Second))
|
|
||||||
}
|
|
||||||
// Paged as the forge pages: fifty a page, and a trunk of another repository is empty (a short page, or none).
|
|
||||||
pages := trunkCommitsPage
|
|
||||||
trunkCommitsPage = func(ctx context.Context, owner, name, branch string, page int) ([]trunkCommit, error) {
|
|
||||||
all, err := pages(ctx, owner, name, branch, 1)
|
|
||||||
if err != nil || page < 1 || (page-1)*50 >= len(all) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return all[(page-1)*50 : min(page*50, len(all))], nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = pages })
|
|
||||||
(following{open: open}).catchUp(t.Context(), nil)
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("the oldest of fifty-one merges was read and planned, outside the window")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 2: a gap with no chosen Approve in the router is not planned and raises one condition naming its repository and
|
|
||||||
// commit; a second start leaves that one condition open, observed twice; planning the gap clears it.
|
|
||||||
func TestAGapWithNoApproveRaisesOneConditionKeptOpenAndClearedWhenPlanned(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
aKeptMergeBefore(t, open)
|
|
||||||
keeper, _ := withConditionsInMemory(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "notes")
|
|
||||||
tr.approved["mesh-catalog-c01"] = false
|
|
||||||
catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
|
|
||||||
catch()
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge with no Approve on record was planned")
|
|
||||||
}
|
|
||||||
raised := func() []conditions.Condition {
|
|
||||||
t.Helper()
|
|
||||||
all, err := keeper.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var out []conditions.Condition
|
|
||||||
for _, c := range all {
|
|
||||||
if c.Kind == kindMergeNotHeard {
|
|
||||||
out = append(out, c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
one := raised()
|
|
||||||
if len(one) != 1 || !strings.Contains(one[0].Subject.ID, "novox/mesh-catalog") || !strings.Contains(one[0].Subject.ID, "mesh-cata") {
|
|
||||||
t.Fatalf("the gap raised %+v, not one condition naming its repository and commit", one)
|
|
||||||
}
|
|
||||||
catch()
|
|
||||||
if again := raised(); len(again) != 1 || again[0].Observations < 2 {
|
|
||||||
t.Fatalf("a second catch-up left %+v: one condition, observed twice, was wanted", again)
|
|
||||||
}
|
|
||||||
tr.approved["mesh-catalog-c01"] = true
|
|
||||||
catch()
|
|
||||||
if !mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("the gap was not planned once its Approve was on record")
|
|
||||||
}
|
|
||||||
if left := raised(); len(left) != 0 {
|
|
||||||
t.Fatalf("planning the gap left its condition open: %+v", left)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clause 3: a forge that does not answer leaves the catch-up undone and says so, and the next start does it.
|
|
||||||
func TestAForgeThatDoesNotAnswerLeavesTheCatchUpForTheNextStart(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
aKeptMergeBefore(t, open)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "notes")
|
|
||||||
tr.silent = true
|
|
||||||
if later := (following{open: open}).catchUp(t.Context(), nil); !later {
|
|
||||||
t.Fatal("a forge that did not answer counted as nothing to catch up")
|
|
||||||
}
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge was planned without the forge's word")
|
|
||||||
}
|
|
||||||
tr.silent = false
|
|
||||||
if later := (following{open: open}).catchUp(t.Context(), nil); later || !mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatalf("the next start did not catch up (left for later: %v)", later)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Finding 2 of the review: a gap that adds a module is a gap, judged as hearMerge judges it. With the operator's Approve on record it
|
|
||||||
// is heard (hearMerge asks the new module in its walk); without, it raises the condition and is not lost in silence.
|
|
||||||
func TestAGapThatAddsAModuleIsHeardOnItsWarrantAndRaisedWithout(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
aKeptMergeBefore(t, open)
|
|
||||||
keeper, _ := withConditionsInMemory(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "newmodule")
|
|
||||||
tr.approved["mesh-catalog-c01"] = false
|
|
||||||
catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
|
|
||||||
catch()
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge adding a module was heard with no Approve on record")
|
|
||||||
}
|
|
||||||
all, err := keeper.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var raised int
|
|
||||||
for _, c := range all {
|
|
||||||
if c.Kind == kindMergeNotHeard {
|
|
||||||
raised++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if raised != 1 {
|
|
||||||
t.Fatalf("a merge adding a module with no Approve raised %d condition(s), not one: it was lost in silence", raised)
|
|
||||||
}
|
|
||||||
tr.approved["mesh-catalog-c01"] = true
|
|
||||||
catch()
|
|
||||||
if !mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge adding a module was not heard once its Approve was on record")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A trunk with merges from before the controller began keeping merges, and merges since: only the later ones are gaps.
|
|
||||||
func TestAMergeFromBeforeTheRecordBeganIsHistoryAndNoCondition(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
keeper, _ := withConditionsInMemory(t)
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "notes", "notes", "notes")
|
|
||||||
tr.approved["mesh-catalog-c01"], tr.approved["mesh-catalog-c02"], tr.approved["mesh-catalog-c03"] = false, false, false
|
|
||||||
// The record begins at the second merge: the first is two days older.
|
|
||||||
tr.merges[0].At = t0.Add(-48 * time.Hour)
|
|
||||||
hear(t, open, catalogueMerge("mesh-catalog-c02", "notes", t0.Add(time.Minute)), t0.Add(time.Minute))
|
|
||||||
delete(tr.approved, "mesh-catalog-c02")
|
|
||||||
(following{open: open}).catchUp(t.Context(), nil)
|
|
||||||
// The first merge (two days before the record) was not even read from the forge: only the third, which is unknown, was.
|
|
||||||
if tr.events != 1 {
|
|
||||||
t.Fatalf("%d merge(s) were read from the forge: the one from before the record is history and is not read", tr.events)
|
|
||||||
}
|
|
||||||
all, err := keeper.Open(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var raised int
|
|
||||||
for _, c := range all {
|
|
||||||
if c.Kind == kindMergeNotHeard {
|
|
||||||
raised++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if raised != 1 {
|
|
||||||
t.Fatalf("%d condition(s) raised: only the merge since the record began (the third) is a gap", raised)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq ADR 0463 §2: the record begins at the earliest merge kept on ANY trunk. A trunk the controller kept no merge of has the
|
|
||||||
// merges made since as gaps (mesh-delivery #1 on novox), and the merges made before as history.
|
|
||||||
func TestATrunkWithNoKeptMergeHasItsLaterMergesAsGapsAndItsEarlierOnesAsHistory(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asksRecorded(t)
|
|
||||||
// The record begins on another trunk, at t0 less an hour.
|
|
||||||
hear(t, open, repoMerge("one", "c-one", t0.Add(-time.Hour)), t0.Add(-time.Hour))
|
|
||||||
tr := aTrunkOf(t, "mesh-catalog", "notes", "notes")
|
|
||||||
tr.merges[0].At = t0.Add(-72 * time.Hour) // before the record: history
|
|
||||||
(following{open: open}).catchUp(t.Context(), nil)
|
|
||||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
|
||||||
t.Fatal("a merge from before the record began was heard")
|
|
||||||
}
|
|
||||||
if !mergeKnown(t, open, "mesh-catalog-c02") {
|
|
||||||
t.Fatal("a merge since the record began, on a trunk with no merge kept, was not caught up")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,318 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The pieces of the catch-up that need no store (novox/hq ADR 0457, issue 1336).
|
|
||||||
|
|
||||||
func TestAMergeCommitsSubjectNamesItsPullRequest(t *testing.T) {
|
|
||||||
title, number, ok := pullOfSubject("Merge pull request 'Fix keycloak's sign-in (issue 1332)' (#378) from fix/1332-sign-in into main")
|
|
||||||
if !ok || number != 378 || title != "Fix keycloak's sign-in (issue 1332)" {
|
|
||||||
t.Fatalf("read %q %d %v", title, number, ok)
|
|
||||||
}
|
|
||||||
if b := headBranchOfSubject("Merge pull request 'x' (#378) from fix/1332-sign-in into main"); b != "fix/1332-sign-in" {
|
|
||||||
t.Fatalf("the head branch read as %q", b)
|
|
||||||
}
|
|
||||||
if _, _, ok := pullOfSubject("Merge branch 'main' of somewhere"); ok {
|
|
||||||
t.Fatal("a merge that names no pull request read as one")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOnlyTheLastFiftyMergesOfATrunkAreRead(t *testing.T) {
|
|
||||||
pages := map[int][]trunkCommit{}
|
|
||||||
var all []trunkCommit // newest first: 51 merges, a plain commit between each pair
|
|
||||||
for i := 51; i >= 1; i-- {
|
|
||||||
all = append(all, trunkCommit{Commit: fmt.Sprintf("merge-%02d", i), SecondParent: fmt.Sprintf("head-%02d", i),
|
|
||||||
Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from fix/x into main", i, i)},
|
|
||||||
trunkCommit{Commit: fmt.Sprintf("plain-%02d", i)})
|
|
||||||
}
|
|
||||||
for i := 0; i < len(all); i += 50 {
|
|
||||||
pages[i/50+1] = all[i:min(i+50, len(all))]
|
|
||||||
}
|
|
||||||
was := trunkCommitsPage
|
|
||||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { return pages[page], nil }
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = was })
|
|
||||||
got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got) != catchUpWindow || got[0].Commit != "merge-02" || got[len(got)-1].Commit != "merge-51" {
|
|
||||||
t.Fatalf("read %d merges from %s to %s: the oldest of 51 is outside the window", len(got), got[0].Commit, got[len(got)-1].Commit)
|
|
||||||
}
|
|
||||||
if got[0].Head != "head-02" {
|
|
||||||
t.Fatalf("a merge's head is its second parent, not %q", got[0].Head)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) {
|
|
||||||
was := trunkCommitsPage
|
|
||||||
trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) {
|
|
||||||
return nil, errors.New("the forge did not answer")
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = was })
|
|
||||||
if got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main"); err == nil {
|
|
||||||
t.Fatalf("a forge that did not answer read as %d merges: nothing to catch up", len(got))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) {
|
|
||||||
ok := approvedMerge(t)
|
|
||||||
if _, err := caughtUpWarrant(sdkdelivery.TellerModule, &ok.ask, &ok.warrant); err != nil {
|
|
||||||
t.Errorf("the operator's Approve under the teller was refused: %v", err)
|
|
||||||
}
|
|
||||||
// 1318 (ADR 0450 §8): mesh-delivery's warrant of before the gate is no longer taken.
|
|
||||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &ok.warrant); err == nil {
|
|
||||||
t.Error("a warrant under mesh-delivery was accepted")
|
|
||||||
}
|
|
||||||
if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil {
|
|
||||||
t.Error("a warrant of an asker the controller takes none from was accepted")
|
|
||||||
}
|
|
||||||
declined := ok.warrant
|
|
||||||
declined.Option, declined.Label = "decline", "Decline"
|
|
||||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &declined); err == nil {
|
|
||||||
t.Error("a Decline was accepted as a warrant")
|
|
||||||
}
|
|
||||||
unproved := ok.warrant
|
|
||||||
by := *unproved.By
|
|
||||||
by.Verified = ""
|
|
||||||
unproved.By = &by
|
|
||||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &unproved); err == nil {
|
|
||||||
t.Error("an answer whose channel proves nobody was accepted")
|
|
||||||
}
|
|
||||||
if _, err := caughtUpWarrant(theDelivery, nil, nil); err == nil {
|
|
||||||
t.Error("a router that holds nothing was accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// aTellersAsk is the ask the delivery gate composes for a pull request, as the router holds it once the operator chose Approve
|
|
||||||
// (the gate's composeMergeAsk: its id from sdkdelivery.AskingID of the group and the heads, its Approve bound to mergeAct over the
|
|
||||||
// heads, the pull request and its head's first twelve digits in the details), with the warrant the router wrote.
|
|
||||||
func aTellersAsk(t *testing.T, group string, members ...sdkdelivery.Head) (asks.Ask, asks.Warrant) {
|
|
||||||
t.Helper()
|
|
||||||
key, err := sdkdelivery.MergeAskKey(group, members)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
id := sdkdelivery.AskingID(key, 1)
|
|
||||||
heads := map[string]string{}
|
|
||||||
var details []string
|
|
||||||
for _, m := range members {
|
|
||||||
owner, repo, _ := strings.Cut(m.Repository, "/")
|
|
||||||
heads[deliveryIDOf(owner, repo, m.Head)] = m.Head
|
|
||||||
details = append(details, fmt.Sprintf("%s#%d %q at %s: issues: none named; review: approved; reaches: no delivery plan yet", repo, m.Number, "title", m.Head[:12]))
|
|
||||||
}
|
|
||||||
bind := func(option string) string {
|
|
||||||
d, err := asks.ActDigest(mergeAct(id, option, heads))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
at := time.Date(2026, 10, 20, 12, 2, 1, 0, time.UTC)
|
|
||||||
ask := asks.Ask{ID: id, Headline: "Merge into main", Explanation: "passed its checks.", Who: asks.Operator, Expires: at.Add(24 * time.Hour),
|
|
||||||
OnExpiry: "nothing is merged; the ask is made again", Details: strings.Join(details, "\n"),
|
|
||||||
Options: []asks.Option{
|
|
||||||
{ID: optionApprove, Label: "Approve", Does: "merges", Level: asks.Approve, Binds: bind(optionApprove)},
|
|
||||||
{ID: "decline", Label: "Decline", Does: "nothing is merged", Level: asks.Approve, Binds: bind("decline")}}}
|
|
||||||
w := asks.Warrant{Ask: id, Asker: sdkdelivery.TellerModule, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve",
|
|
||||||
Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"},
|
|
||||||
At: at.Add(time.Minute), AskDigest: ask.Digest()}
|
|
||||||
return ask, w
|
|
||||||
}
|
|
||||||
|
|
||||||
// withRouter hands findMergeWarrant these asks as the router's records, by asker.
|
|
||||||
func withRouter(t *testing.T, records map[string]map[string]struct {
|
|
||||||
ask asks.Ask
|
|
||||||
w asks.Warrant
|
|
||||||
}) {
|
|
||||||
t.Helper()
|
|
||||||
wasIDs, wasAsk := routerAskIDs, routerAskOf
|
|
||||||
closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true })
|
|
||||||
routerAskIDs = func(_ context.Context, _ catalogueReader, asker string) ([]string, error) {
|
|
||||||
var ids []string
|
|
||||||
for id := range records[asker] {
|
|
||||||
ids = append(ids, id)
|
|
||||||
}
|
|
||||||
return ids, nil
|
|
||||||
}
|
|
||||||
routerAskOf = func(_ context.Context, _ catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
|
||||||
r, ok := records[asker][id]
|
|
||||||
if !ok {
|
|
||||||
return nil, nil, nil
|
|
||||||
}
|
|
||||||
return &r.ask, &r.w, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() {
|
|
||||||
routerAskIDs, routerAskOf = wasIDs, wasAsk
|
|
||||||
closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
type aRouterRecord = struct {
|
|
||||||
ask asks.Ask
|
|
||||||
w asks.Warrant
|
|
||||||
}
|
|
||||||
|
|
||||||
// The teller's asks, as the router held them on 2026-10-20: the id is the router's own key, rebuilt here from the group and the
|
|
||||||
// head the gate composed it for (mesh-controller#405 at 8a330afd, mesh-catalog#381 at 1c702422, mesh-host#124 at 52600ec7). It names
|
|
||||||
// neither the pull request nor the head, so the warrant is found by the heads the Approve is bound to.
|
|
||||||
func TestATellersWarrantIsFoundByTheHeadItsApproveIsBoundTo(t *testing.T) {
|
|
||||||
real := []struct {
|
|
||||||
group, key, owner, repo string
|
|
||||||
number int
|
|
||||||
head string
|
|
||||||
}{
|
|
||||||
{"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-ba59b03220f8-989afe0a3bcf", "novox", "mesh-controller", 405, "8a330afd44ba852873339cf1210e90989dca46e1"},
|
|
||||||
{"fix/1351-remove-todays-delivery-module", "merge-fix_1351_remove_todays_delivery_module-64a353-d93b97b9697b", "novox", "mesh-catalog", 381, "1c7024224dd77c5b603299378883e0993b2e2fdd"},
|
|
||||||
{"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-6a59ba48da14-424e053ebc9c", "novox", "mesh-host", 124, "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"},
|
|
||||||
}
|
|
||||||
teller := map[string]aRouterRecord{}
|
|
||||||
for _, r := range real {
|
|
||||||
ask, w := aTellersAsk(t, r.group, sdkdelivery.Head{Repository: r.owner + "/" + r.repo, Number: r.number, Head: r.head})
|
|
||||||
if ask.ID != r.key {
|
|
||||||
t.Fatalf("the id of %s#%d is %q, and the router held it as %q: the fixture is not the router's", r.repo, r.number, ask.ID, r.key)
|
|
||||||
}
|
|
||||||
teller[ask.ID] = aRouterRecord{ask, w}
|
|
||||||
}
|
|
||||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: teller})
|
|
||||||
for _, r := range real {
|
|
||||||
why, found, reason, err := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, r.head)
|
|
||||||
if err != nil || !found || !strings.Contains(why, r.key) || !strings.Contains(why, "bound to the head of this pull request") {
|
|
||||||
t.Errorf("%s#%d: the warrant was not found: %q %v %q %v", r.repo, r.number, why, found, reason, err)
|
|
||||||
}
|
|
||||||
// Another head of the same pull request is another merge: the Approve was not given for it.
|
|
||||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, "ffffffffffffffffffffffffffffffffffffffff"); found {
|
|
||||||
t.Errorf("%s#%d: another head found the warrant of this one", r.repo, r.number)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// One pull request's Approve is none for another repository's merge at the same head digits.
|
|
||||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-tools", 405, real[0].head); found {
|
|
||||||
t.Error("a merge of another repository found mesh-controller#405's warrant")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A group's ask binds several heads, and the router's record carries only twelve digits of the other members': the pull request is
|
|
||||||
// found in the ask as it was published, which the warrant's digest covers.
|
|
||||||
func TestAGroupsWarrantIsFoundByTheAsksWordsForThePullRequestAndItsHead(t *testing.T) {
|
|
||||||
a := sdkdelivery.Head{Repository: "novox/mesh-host", Number: 124, Head: "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"}
|
|
||||||
b := sdkdelivery.Head{Repository: "novox/mesh-controller", Number: 405, Head: "8a330afd44ba852873339cf1210e90989dca46e1"}
|
|
||||||
ask, w := aTellersAsk(t, "feat/r4b-sdk-users", a, b)
|
|
||||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, w}}})
|
|
||||||
for _, h := range []sdkdelivery.Head{a, b} {
|
|
||||||
owner, repo, _ := strings.Cut(h.Repository, "/")
|
|
||||||
why, found, reason, err := findMergeWarrant(t.Context(), nil, owner, repo, h.Number, h.Head)
|
|
||||||
if err != nil || !found || !strings.Contains(why, "a group's ask naming this pull request at this head") {
|
|
||||||
t.Errorf("%s#%d: not found in the group's ask: %q %v %q %v", repo, h.Number, why, found, reason, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 381, a.Head); found {
|
|
||||||
t.Error("a pull request the group's ask does not name found its warrant")
|
|
||||||
}
|
|
||||||
// A Decline is no warrant.
|
|
||||||
declined := w
|
|
||||||
declined.Option, declined.Label = "decline", "Decline"
|
|
||||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, declined}}})
|
|
||||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-host", 124, a.Head); found {
|
|
||||||
t.Error("a Decline found a warrant")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A trunk shorter than a page, or with none, is read without a fault: the pages end when the forge's do.
|
|
||||||
func TestAShortTrunkIsReadWholeAndAnEmptyOneIsNoMerges(t *testing.T) {
|
|
||||||
was := trunkCommitsPage
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = was })
|
|
||||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) {
|
|
||||||
if page > 1 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return []trunkCommit{{Commit: "m2", SecondParent: "h2", Subject: "Merge pull request 'two' (#2) from fix/b into main"},
|
|
||||||
{Commit: "p"}, {Commit: "m1", SecondParent: "h1", Subject: "Merge pull request 'one' (#1) from fix/a into main"}}, nil
|
|
||||||
}
|
|
||||||
got, err := readTrunkMerges(t.Context(), "novox", "x", "main")
|
|
||||||
if err != nil || len(got) != 2 || got[0].Commit != "m1" || got[1].Commit != "m2" {
|
|
||||||
t.Fatalf("a short trunk read as %+v %v", got, err)
|
|
||||||
}
|
|
||||||
trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { return nil, nil }
|
|
||||||
if got, err := readTrunkMerges(t.Context(), "novox", "x", "main"); err != nil || len(got) != 0 {
|
|
||||||
t.Fatalf("an empty trunk read as %+v %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ADR 0253: what the operator reads of a merge not heard is plain: no hash, no identifier, nothing to do "by hand".
|
|
||||||
func TestTheConditionOfAMergeNotHeardIsInPlainWords(t *testing.T) {
|
|
||||||
keeper, _ := withConditionsInMemory(t)
|
|
||||||
tm := trunkMerge{Commit: "c0880c4d1f2e3a4b5c6d7e8f9a0b1c2d3e4f5a6b", Head: "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372"}
|
|
||||||
(following{}).raiseMergeNotHeard(t.Context(), "novox/mesh-catalog", "main", tm, link.SourceMoved{Number: 378},
|
|
||||||
"the router holds no chosen merge ask of mesh-delivery-teller or mesh-delivery that covers the head b9104454f9a5")
|
|
||||||
open, err := keeper.Open(t.Context())
|
|
||||||
if err != nil || len(open) != 1 {
|
|
||||||
t.Fatalf("opened %d conditions, %v", len(open), err)
|
|
||||||
}
|
|
||||||
c := open[0]
|
|
||||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs,
|
|
||||||
Resolved: c.Resolved}); !ok {
|
|
||||||
t.Fatalf("the condition's words are not plain: %s\n%q\n%q\n%q", why, c.Headline, c.Explanation, c.Needs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The trunk's history lists the merges made inside a pull request's branch too. On novox (2026-10-20) 15 of the 15 non-pull-request
|
|
||||||
// "gaps" of mesh-catalog and mesh-controller were such merges; only a pull request's merge into this trunk is a merge to catch up.
|
|
||||||
func TestAMergeMadeInsideABranchIsNoMergeOfTheTrunk(t *testing.T) {
|
|
||||||
was := trunkCommitsPage
|
|
||||||
t.Cleanup(func() { trunkCommitsPage = was })
|
|
||||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) {
|
|
||||||
if page > 1 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return []trunkCommit{
|
|
||||||
{Commit: "c0880c4d", SecondParent: "b9104454", Subject: "Merge pull request 'Read the grant file' (#378) from fix/1332 into main"},
|
|
||||||
{Commit: "6394a62a", SecondParent: "x1", Subject: "Merge remote-tracking branch 'origin/main' into fix/1081-withdraw"},
|
|
||||||
{Commit: "896f1d28", SecondParent: "x2", Subject: "Merge feat/agents-write-git-ssh into feat/agents-write-git-ssh-f (issue 955 F)"},
|
|
||||||
{Commit: "4d03755c", SecondParent: "x3", Subject: "Bring the branch up to main after #354 (issue 1047)"},
|
|
||||||
{Commit: "other", SecondParent: "x4", Subject: "Merge pull request 'Elsewhere' (#5) from fix/y into nox-mesh"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main")
|
|
||||||
if err != nil || len(got) != 1 || got[0].Commit != "c0880c4d" {
|
|
||||||
t.Fatalf("read %+v %v: only keycloak's pull request merge into main is a merge to catch up", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq ADR 0463 §2: the record of merges begins at the earliest merge the controller holds on any trunk. A merge before it is
|
|
||||||
// history on every trunk; a trunk with no merge kept of its own still has the merges made since as gaps (mesh-delivery #1 on novox).
|
|
||||||
func TestOnlyTheMergesSinceTheRecordBeganAreGapsOnEveryTrunk(t *testing.T) {
|
|
||||||
first := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) // the first merge it heard, on some trunk of the mesh
|
|
||||||
trunk := func(prefix string) []trunkMerge {
|
|
||||||
return []trunkMerge{
|
|
||||||
{Commit: prefix + "-old-1", At: first.Add(-30 * 24 * time.Hour)},
|
|
||||||
{Commit: prefix + "-old-2", At: first.Add(-2 * time.Hour)},
|
|
||||||
{Commit: prefix + "-within-grace", At: first.Add(-mergeGrace / 2)},
|
|
||||||
{Commit: prefix + "-since", At: first.Add(24 * time.Hour)},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Two trunks, one of which the controller has kept no merge of: both read by the one starting point.
|
|
||||||
for _, prefix := range []string{"kept-trunk", "trunk-with-no-kept-merge"} {
|
|
||||||
var commits []string
|
|
||||||
for _, m := range sinceTheRecord(trunk(prefix), true, first) {
|
|
||||||
commits = append(commits, m.Commit)
|
|
||||||
}
|
|
||||||
if want := prefix + "-within-grace " + prefix + "-since"; strings.Join(commits, " ") != want {
|
|
||||||
t.Errorf("%s: the gaps are %v, want %q: earlier ones are history on every trunk, later ones are gaps", prefix, commits, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if none := sinceTheRecord(trunk("x"), false, time.Time{}); len(none) != 0 {
|
|
||||||
t.Errorf("with no merge kept anywhere there are %d gap(s), but no record to have one in", len(none))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -54,7 +54,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
|||||||
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
||||||
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
||||||
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
||||||
go copySentBuildsSoon(ctx, inv)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,39 +61,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
|||||||
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
||||||
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
||||||
|
|
||||||
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
|
|
||||||
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
|
|
||||||
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
|
|
||||||
// that names a commit and empties each column it has copied.
|
|
||||||
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
|
|
||||||
failed := false
|
|
||||||
for {
|
|
||||||
_, err := inv.CopySentBuilds(ctx)
|
|
||||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
|
|
||||||
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
|
||||||
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
|
|
||||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
|
||||||
o.Summary += err.Error()
|
|
||||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
|
||||||
failed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if keeper := conditionsFrom; keeper != nil && failed {
|
|
||||||
_, _ = keeper.Clear(ctx, o.Key(), "copied")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(chainMoveRetry):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// chainMoveRetry is how long the copy waits before it tries again.
|
// chainMoveRetry is how long the copy waits before it tries again.
|
||||||
const chainMoveRetry = 30 * time.Second
|
const chainMoveRetry = 30 * time.Second
|
||||||
|
|
||||||
|
|||||||
@@ -1,245 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/chain"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/lease"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there,
|
|
||||||
// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live
|
|
||||||
// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's
|
|
||||||
// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send
|
|
||||||
// is recorded as the controller records one it made, and the gate's facts are the store's.
|
|
||||||
func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var sent [][]string
|
|
||||||
wasSend := sendRollout
|
|
||||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
||||||
sent = append(sent, append([]string(nil), names...))
|
|
||||||
current, err := open.inventory.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, node := range names {
|
|
||||||
digest := fmt.Sprintf("d-%s-%d", node, len(sent))
|
|
||||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest,
|
|
||||||
map[string]string{"notes": current["notes"].Commit}); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { sendRollout = wasSend })
|
|
||||||
wasGather := gatherGateFacts
|
|
||||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
|
||||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
|
||||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
|
||||||
reports, err := open.inventory.LastReports(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return f, err
|
|
||||||
}
|
|
||||||
for _, r := range reports {
|
|
||||||
f.reports[r.Node] = r
|
|
||||||
}
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
|
||||||
|
|
||||||
// A merge of notes is heard, and its window closes: the batch is cut into the walk.
|
|
||||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
|
||||||
cutAt(t, open, t0.Add(time.Second+90*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
|
||||||
}
|
|
||||||
rollout := ws[0].ID
|
|
||||||
|
|
||||||
// The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the
|
|
||||||
// outcome in as it does.
|
|
||||||
asking, err := inv.PlanByID(ctx, rollout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ {
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
buildID := asking.Modules["notes"].Build
|
|
||||||
if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) {
|
|
||||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
|
||||||
}
|
|
||||||
digest := "sha256:" + strings.Repeat("a", 64)
|
|
||||||
result := outcome(buildID, "modules/notes", "c-notes", "notes", "")
|
|
||||||
result.ReuseKey = "rk-notes"
|
|
||||||
result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}}
|
|
||||||
if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The walk sends the tier to its first machine.
|
|
||||||
for try := 0; try < 3 && len(sent) == 0; try++ {
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
}
|
|
||||||
if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) {
|
|
||||||
t.Fatalf("sent %v", sent)
|
|
||||||
}
|
|
||||||
|
|
||||||
// One read of the assignment.
|
|
||||||
on, err := inv.AssignmentsOn(ctx, "anchor")
|
|
||||||
a, ok := assignmentOf(on, "notes")
|
|
||||||
if err != nil || !ok {
|
|
||||||
t.Fatalf("the assignments of anchor: %+v %v", on, err)
|
|
||||||
}
|
|
||||||
// Beside it a module pulled in by the assignment, which no send of this walk put anywhere.
|
|
||||||
if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") {
|
|
||||||
t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard)
|
|
||||||
}
|
|
||||||
if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout || a.Artifact != digest ||
|
|
||||||
a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() {
|
|
||||||
t.Fatalf("the assignment names %+v, not commit c-notes, build %s, artifact %s, rollout %s", a, buildID, digest, rollout)
|
|
||||||
}
|
|
||||||
// One read of the rollout.
|
|
||||||
p, err := inv.PlanByID(ctx, rollout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}}
|
|
||||||
if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 ||
|
|
||||||
p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil {
|
|
||||||
t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange)
|
|
||||||
}
|
|
||||||
// And one read of the build names the same commit, with what it published and its reuse key.
|
|
||||||
built, found, err := inv.BuildByID(ctx, buildID)
|
|
||||||
if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" {
|
|
||||||
t.Fatalf("the build: %+v %v %v", built, found, err)
|
|
||||||
}
|
|
||||||
lately, err := inv.Builds(ctx, "notes", 1)
|
|
||||||
if err != nil || len(lately) != 1 || lately[0].ReuseKey != "rk-notes" || lately[0].Published == nil ||
|
|
||||||
lately[0].Published.Artifact != digest {
|
|
||||||
t.Fatalf("what the build published: %+v %v", lately, err)
|
|
||||||
}
|
|
||||||
// The verbs say the links.
|
|
||||||
if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") ||
|
|
||||||
!strings.Contains(lines[1], buildID) {
|
|
||||||
t.Fatalf("`plans` says %v", lines)
|
|
||||||
}
|
|
||||||
if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) {
|
|
||||||
t.Fatalf("`node show` says %v", lines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks
|
|
||||||
// nothing (issue 1095).
|
|
||||||
func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
|
||||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
node := nodeID(t, open, "anchor")
|
|
||||||
inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}})
|
|
||||||
if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Sent again by hand, with no rollout: the assignment still names the rollout that put the build there.
|
|
||||||
if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
on, _ := inv.AssignmentsOn(ctx, "anchor")
|
|
||||||
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
|
|
||||||
t.Fatalf("the assignment names %+v", on)
|
|
||||||
}
|
|
||||||
// A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit.
|
|
||||||
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() {
|
|
||||||
t.Fatalf("a send of unknown builds left %+v", on)
|
|
||||||
}
|
|
||||||
// A module sent a commit no build is kept of is named by its commit alone.
|
|
||||||
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" ||
|
|
||||||
runOf(on, "notes").Rollout != "" {
|
|
||||||
t.Fatalf("the assignment after a push of c2 by hand names %+v", on)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it.
|
|
||||||
func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) {
|
|
||||||
for _, a := range on {
|
|
||||||
if a.Module == module {
|
|
||||||
return a, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return chain.Assignment{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func runOf(on []chain.Assignment, module string) chain.Assignment {
|
|
||||||
a, _ := assignmentOf(on, module)
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
|
|
||||||
// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered
|
|
||||||
// through the `assign` verb for a module whose build is in flight, one read of `<node>.<module>` says state pending with the
|
|
||||||
// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied,
|
|
||||||
// and the table of pending assignments is read by nobody. No key is written by hand.
|
|
||||||
func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) {
|
|
||||||
open := aCatalogueMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
asksWithPaths(t)
|
|
||||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
|
||||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
|
||||||
t.Fatalf("an assignment while its build runs was refused: %v", err)
|
|
||||||
}
|
|
||||||
waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
|
||||||
if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() ||
|
|
||||||
waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" ||
|
|
||||||
!strings.HasPrefix(waiting.Pending.ID, "p-") {
|
|
||||||
t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err)
|
|
||||||
}
|
|
||||||
if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") {
|
|
||||||
t.Fatalf("a module only waiting is assigned: %v", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
|
||||||
if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil ||
|
|
||||||
made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID {
|
|
||||||
t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err)
|
|
||||||
}
|
|
||||||
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
|
||||||
t.Fatal("the build registered sensors and it was not assigned")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,121 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/chain"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a
|
|
||||||
// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of
|
|
||||||
// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the
|
|
||||||
// build before and after.
|
|
||||||
|
|
||||||
// rolloutModules is what a plan sends of each module: the commit and the build of it.
|
|
||||||
func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule {
|
|
||||||
out := make(map[string]inventory.RolloutModule, len(p.Modules))
|
|
||||||
for m, s := range p.Modules {
|
|
||||||
if s != nil && s.Commit != "" {
|
|
||||||
out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout.
|
|
||||||
func sendingAs(ctx context.Context, p *inventory.Plan) context.Context {
|
|
||||||
return inventory.WithRollout(ctx, p.ID, rolloutModules(p))
|
|
||||||
}
|
|
||||||
|
|
||||||
// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit,
|
|
||||||
// else the commit the machine was last sent for it. Read before the send, which writes the new ones.
|
|
||||||
func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref {
|
|
||||||
was := map[string]chain.Ref{}
|
|
||||||
if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known {
|
|
||||||
for m, commit := range sent {
|
|
||||||
was[m] = chain.Ref{Commit: commit}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ran, err := inv.AssignmentRuns(ctx, node); err == nil {
|
|
||||||
for m, r := range ran {
|
|
||||||
was[m] = r
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return was
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and
|
|
||||||
// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the
|
|
||||||
// module is replaced, so a send made again says it once.
|
|
||||||
func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) {
|
|
||||||
for _, m := range modules {
|
|
||||||
s := p.Modules[m]
|
|
||||||
if s == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}}
|
|
||||||
if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 {
|
|
||||||
change.Before = p.Changed[at].Before
|
|
||||||
p.Changed[at] = change
|
|
||||||
} else {
|
|
||||||
p.Changed = append(p.Changed, change)
|
|
||||||
}
|
|
||||||
if s.Build != "" && !slices.Contains(p.Builds, s.Build) {
|
|
||||||
p.Builds = append(p.Builds, s.Build)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) {
|
|
||||||
sent := at.UTC()
|
|
||||||
p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none.
|
|
||||||
func refSaid(r chain.Ref) string {
|
|
||||||
switch {
|
|
||||||
case r.Build == "" && r.Commit == "":
|
|
||||||
return "nothing"
|
|
||||||
case r.Build == "":
|
|
||||||
return "commit " + short(r.Commit)
|
|
||||||
}
|
|
||||||
return r.Build + " (" + short(r.Commit) + ")"
|
|
||||||
}
|
|
||||||
|
|
||||||
// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after.
|
|
||||||
func changedLines(p inventory.Plan) []string {
|
|
||||||
if len(p.Changed) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := []string{" changed:"}
|
|
||||||
for _, c := range p.Changed {
|
|
||||||
out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the
|
|
||||||
// rollout that put it there and since when.
|
|
||||||
func runLines(assigned []chain.Assignment) []string {
|
|
||||||
var out []string
|
|
||||||
for _, a := range assigned {
|
|
||||||
if a.Build == "" && a.Commit == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit}))
|
|
||||||
if a.Rollout != "" {
|
|
||||||
line += ", sent by " + a.Rollout
|
|
||||||
}
|
|
||||||
if !a.Since.IsZero() {
|
|
||||||
line += " at " + a.Since.Local().Format("2006-01-02 15:04")
|
|
||||||
}
|
|
||||||
out = append(out, line)
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return append([]string{" what they run, as the chain names it:"}, out...)
|
|
||||||
}
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/chain"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one
|
|
||||||
// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the
|
|
||||||
// rollout carried once each, and one step for a machine and a tier.
|
|
||||||
func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
|
||||||
p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{
|
|
||||||
"notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}}
|
|
||||||
was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}}
|
|
||||||
|
|
||||||
noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at)
|
|
||||||
noteChanges(&p, "laptop", []string{"notes"}, nil, at)
|
|
||||||
// The send noted again, after what was sent had become what the machine runs: the first before stands.
|
|
||||||
noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute))
|
|
||||||
|
|
||||||
want := []chain.Change{
|
|
||||||
{Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}},
|
|
||||||
{Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}},
|
|
||||||
{Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}},
|
|
||||||
}
|
|
||||||
if len(p.Changed) != len(want) {
|
|
||||||
t.Fatalf("changed %+v", p.Changed)
|
|
||||||
}
|
|
||||||
for k := range want {
|
|
||||||
if p.Changed[k] != want[k] {
|
|
||||||
t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" {
|
|
||||||
t.Errorf("builds %v", p.Builds)
|
|
||||||
}
|
|
||||||
if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" {
|
|
||||||
t.Errorf("steps %+v", p.Steps)
|
|
||||||
}
|
|
||||||
lines := changedLines(p)
|
|
||||||
if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") ||
|
|
||||||
!strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") {
|
|
||||||
t.Errorf("changed lines %q", lines)
|
|
||||||
}
|
|
||||||
if got := changedLines(inventory.Plan{}); got != nil {
|
|
||||||
t.Errorf("a plan that changed nothing says %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) {
|
|
||||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
|
||||||
lines := runLines([]chain.Assignment{
|
|
||||||
{Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at},
|
|
||||||
{Node: "anchor", Module: "web", Commit: "c3"},
|
|
||||||
{Node: "anchor", Module: "never-sent"},
|
|
||||||
})
|
|
||||||
if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") ||
|
|
||||||
!strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") {
|
|
||||||
t.Fatalf("lines %q", lines)
|
|
||||||
}
|
|
||||||
if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil {
|
|
||||||
t.Fatal("a module never sent is said to run something")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
|
|
||||||
// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
|
|
||||||
// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
|
|
||||||
// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
|
|
||||||
func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
|
|
||||||
open := windowed(t)
|
|
||||||
everyHeadWaits(t)
|
|
||||||
asked := asksRecorded(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
|
||||||
cutAt(t, open, t0.Add(91*time.Second))
|
|
||||||
ws, _ := walks(t, open)
|
|
||||||
if len(ws) != 1 {
|
|
||||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
|
||||||
}
|
|
||||||
rollout := ws[0].ID
|
|
||||||
asking, err := inv.PlanByID(ctx, rollout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ {
|
|
||||||
advancePlans(ctx, open)
|
|
||||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
buildID := asking.Modules["notes"].Build
|
|
||||||
if buildID == "" || len(*asked) != 1 {
|
|
||||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
|
||||||
}
|
|
||||||
digest := "sha256:" + strings.Repeat("b", 64)
|
|
||||||
result := outcome(buildID, "modules/notes", "c-notes", "notes", "")
|
|
||||||
result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}}
|
|
||||||
if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// What the live path made is in the chain.
|
|
||||||
if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" {
|
|
||||||
t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err)
|
|
||||||
}
|
|
||||||
if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout {
|
|
||||||
t.Fatalf("the rollout is not read from the chain: %+v %v", p, err)
|
|
||||||
}
|
|
||||||
if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
} else if _, ok := assignmentOf(on, "notes"); !ok {
|
|
||||||
t.Fatalf("the assignment is not read from the chain: %+v", on)
|
|
||||||
}
|
|
||||||
|
|
||||||
// And the catalog of the store holds none of them.
|
|
||||||
have := map[string]bool{}
|
|
||||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
|
||||||
have[strings.Trim(row[0], `"`)] = true
|
|
||||||
}
|
|
||||||
for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} {
|
|
||||||
if have[table] {
|
|
||||||
t.Errorf("the table %s is in the controller's store", table)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, table := range []string{"build_gate", "assignment_generation"} {
|
|
||||||
if !have[table] {
|
|
||||||
t.Errorf("the table %s, which stays, is gone", table)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
|
|
||||||
t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,221 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"path"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
|
|
||||||
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
|
|
||||||
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
|
|
||||||
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
|
|
||||||
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
|
|
||||||
// planner, so the two can be compared.
|
|
||||||
|
|
||||||
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
|
|
||||||
type policyOf func(module string) (inventory.Upgrade, bool)
|
|
||||||
|
|
||||||
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
|
|
||||||
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
|
|
||||||
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
|
|
||||||
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
|
|
||||||
byName := map[string]inventory.Entry{}
|
|
||||||
for _, e := range entries {
|
|
||||||
byName[e.Manifest.Module] = e
|
|
||||||
}
|
|
||||||
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
|
|
||||||
holder := ""
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
|
|
||||||
holder = e.On[0]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var granting []string
|
|
||||||
machines := map[string]*link.MachinePlan{}
|
|
||||||
machine := func(name string) *link.MachinePlan {
|
|
||||||
if machines[name] == nil {
|
|
||||||
machines[name] = &link.MachinePlan{Machine: name}
|
|
||||||
}
|
|
||||||
return machines[name]
|
|
||||||
}
|
|
||||||
for _, tier := range r.Plan.Tiers {
|
|
||||||
for _, name := range tier {
|
|
||||||
e, held := byName[name]
|
|
||||||
if !held {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
u, known := policy(name)
|
|
||||||
waits := known && !u.RollOut
|
|
||||||
for _, on := range e.On {
|
|
||||||
if waits {
|
|
||||||
machine(on).Waits = append(machine(on).Waits, name)
|
|
||||||
} else {
|
|
||||||
machine(on).Receives = append(machine(on).Receives, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
|
|
||||||
!slices.Contains(granting, name) {
|
|
||||||
granting = append(granting, name)
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
|
||||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
|
||||||
// until a person takes the step, so merging it is a promise to take it.
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+
|
|
||||||
"nothing else is sent to %s either — unless the new build turns out the same as the one running there "+
|
|
||||||
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
|
|
||||||
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
|
|
||||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
|
||||||
case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData):
|
|
||||||
// Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377).
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+
|
|
||||||
"its move recreates none of its containers and moves none of its data, which the walk judges and then sends",
|
|
||||||
name, orNone(u.From)))
|
|
||||||
case waits && len(e.On) > 0:
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
|
||||||
orNone(u.From)))
|
|
||||||
}
|
|
||||||
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
|
|
||||||
var offers []string
|
|
||||||
for _, o := range e.Manifest.Provides {
|
|
||||||
offers = append(offers, o.Name)
|
|
||||||
}
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
|
|
||||||
name, strings.Join(offers, ", ")))
|
|
||||||
}
|
|
||||||
if e.Manifest.Data != nil && len(e.On) > 0 {
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(granting) > 0 {
|
|
||||||
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
|
|
||||||
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
|
|
||||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
|
|
||||||
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
|
|
||||||
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
|
|
||||||
}
|
|
||||||
var names []string
|
|
||||||
for name := range machines {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
for _, name := range names {
|
|
||||||
p.Machines = append(p.Machines, *machines[name])
|
|
||||||
}
|
|
||||||
p.Summary = summaryOf(p)
|
|
||||||
p.Class = changeClassOf(p, entries)
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
|
|
||||||
const classNone = "none"
|
|
||||||
|
|
||||||
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
|
|
||||||
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
|
|
||||||
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
|
|
||||||
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
|
|
||||||
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
|
|
||||||
// resolver claim its manifest makes.
|
|
||||||
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
|
|
||||||
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
|
|
||||||
return classNone
|
|
||||||
}
|
|
||||||
modules := append(append([]string{}, p.Moved...), p.Dependents...)
|
|
||||||
for _, dir := range p.New {
|
|
||||||
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
|
|
||||||
modules = append(modules, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return classOfModules(modules, entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
|
||||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
|
||||||
const grantsStepWord = "grants step"
|
|
||||||
|
|
||||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
|
||||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
|
||||||
|
|
||||||
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
|
|
||||||
func summaryOf(p link.ChangePlan) string {
|
|
||||||
if len(p.Moved) == 0 && len(p.New) == 0 {
|
|
||||||
return "builds nothing: the change touches no module of the mesh's graph"
|
|
||||||
}
|
|
||||||
var parts []string
|
|
||||||
what := strings.Join(p.Moved, ", ")
|
|
||||||
if len(p.Dependents) > 0 {
|
|
||||||
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
|
|
||||||
}
|
|
||||||
if len(p.New) > 0 {
|
|
||||||
if what != "" {
|
|
||||||
what += ", "
|
|
||||||
}
|
|
||||||
what += "new: " + strings.Join(p.New, ", ")
|
|
||||||
}
|
|
||||||
var to []string
|
|
||||||
for _, m := range p.Machines {
|
|
||||||
if len(m.Receives) > 0 {
|
|
||||||
to = append(to, m.Machine)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(to) > 0 {
|
|
||||||
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
|
|
||||||
} else {
|
|
||||||
parts = append(parts, "builds "+what+", sent nowhere")
|
|
||||||
}
|
|
||||||
bus := "no bus step"
|
|
||||||
for _, s := range p.Steps {
|
|
||||||
switch {
|
|
||||||
case strings.HasPrefix(s, busUpgradeNeeded):
|
|
||||||
bus = busUpgradeNeeded
|
|
||||||
case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step":
|
|
||||||
bus = "a bus step"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
parts = append(parts, bus)
|
|
||||||
waiting := 0
|
|
||||||
for _, m := range p.Machines {
|
|
||||||
waiting += len(m.Waits)
|
|
||||||
}
|
|
||||||
if waiting > 0 {
|
|
||||||
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
|
|
||||||
}
|
|
||||||
return strings.Join(parts, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// planText is a change plan as a person reads it, for the pull request's comment.
|
|
||||||
func planText(p link.ChangePlan) string {
|
|
||||||
var b strings.Builder
|
|
||||||
fmt.Fprintf(&b, "change plan of %s at %.8s into %s: %s\n", p.Repository, p.Head, p.Base, p.Summary)
|
|
||||||
for i, tier := range p.Tiers {
|
|
||||||
fmt.Fprintf(&b, " tier %d: %s\n", i, strings.Join(tier, ", "))
|
|
||||||
}
|
|
||||||
for _, m := range p.Machines {
|
|
||||||
line := " " + m.Machine + ": "
|
|
||||||
if len(m.Receives) > 0 {
|
|
||||||
line += "receives " + strings.Join(m.Receives, ", ")
|
|
||||||
}
|
|
||||||
if len(m.Waits) > 0 {
|
|
||||||
if len(m.Receives) > 0 {
|
|
||||||
line += "; "
|
|
||||||
}
|
|
||||||
line += "waits for a person: " + strings.Join(m.Waits, ", ")
|
|
||||||
}
|
|
||||||
b.WriteString(line + "\n")
|
|
||||||
}
|
|
||||||
for _, s := range p.Steps {
|
|
||||||
b.WriteString(" - " + s + "\n")
|
|
||||||
}
|
|
||||||
if len(p.Unread) > 0 {
|
|
||||||
fmt.Fprintf(&b, " read by no module's build: %s\n", strings.Join(p.Unread, ", "))
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
@@ -1,78 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
|
|
||||||
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
|
|
||||||
// person, a provider whose consumers follow it.
|
|
||||||
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
|
||||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
|
||||||
entry := func(name, path string, on ...string) inventory.Entry {
|
|
||||||
e := fromRepo(name, catalogue_, path)
|
|
||||||
e.Source.BuiltFrom = "old"
|
|
||||||
e.On = on
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
nats := entry("nats", "modules/nats", "anchor")
|
|
||||||
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
||||||
gitea := entry("gitea", "modules/gitea", "anchor")
|
|
||||||
held := entry("photos", "modules/photos", "anchor", "laptop")
|
|
||||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
|
||||||
tools.On = []string{"anchor", "laptop"}
|
|
||||||
entries := []inventory.Entry{nats, gitea, held, tools}
|
|
||||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
|
||||||
policy := func(module string) (inventory.Upgrade, bool) {
|
|
||||||
if module == "photos" {
|
|
||||||
return inventory.Upgrade{RollOut: false, From: "a person"}, true
|
|
||||||
}
|
|
||||||
return inventory.Upgrade{RollOut: true}, true
|
|
||||||
}
|
|
||||||
plan := func(paths ...string) link.ChangePlan {
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
|
||||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
|
||||||
}
|
|
||||||
|
|
||||||
p := plan("modules/gitea/index.ts")
|
|
||||||
if p.Summary != "builds gitea → anchor; no bus step" {
|
|
||||||
t.Errorf("one module's change reads %q", p.Summary)
|
|
||||||
}
|
|
||||||
|
|
||||||
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
|
|
||||||
if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
|
||||||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
|
|
||||||
t.Errorf("the bus and a held module read %q", p.Summary)
|
|
||||||
}
|
|
||||||
got := map[string]string{}
|
|
||||||
for _, m := range p.Machines {
|
|
||||||
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
|
|
||||||
}
|
|
||||||
// The bus first, what stands on it after: the build plan's order, per machine.
|
|
||||||
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
|
|
||||||
t.Errorf("the deploy plan reads %v", got)
|
|
||||||
}
|
|
||||||
text := strings.Join(p.Steps, "\n")
|
|
||||||
// Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a
|
|
||||||
// person runs the bus's step, and the verb that does.
|
|
||||||
for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor",
|
|
||||||
"mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person",
|
|
||||||
"nats provides mesh-bus"} {
|
|
||||||
if !strings.Contains(text, want) {
|
|
||||||
t.Errorf("the steps do not say %q:\n%s", want, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
p = plan("merge-check.sh")
|
|
||||||
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
|
|
||||||
t.Errorf("a root file's plan reads %+v", p)
|
|
||||||
}
|
|
||||||
if !strings.Contains(planText(p), "read by no module's build: merge-check.sh") {
|
|
||||||
t.Errorf("the plan's text does not say why nothing is built:\n%s", planText(p))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact
|
|
||||||
// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the
|
|
||||||
// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which
|
|
||||||
// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store,
|
|
||||||
// as the controller's own collection does. The build seat's holders let go of anything a week old on their own,
|
|
||||||
// so a word lost here leaves a week's builds, never more.
|
|
||||||
|
|
||||||
// checkBuildsIn is the scratch namespace as the controller reaches the artifact store.
|
|
||||||
func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) {
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if address == "" {
|
|
||||||
return nil, errors.New("the artifact store is not on the mesh's network")
|
|
||||||
}
|
|
||||||
return builder.Registry{Address: address}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgetChecks lets go of a pull request's check builds and answers what it did.
|
|
||||||
var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) {
|
|
||||||
scratch, err := checkBuildsIn(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
n, err := scratch.Forget(ctx, pull)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge
|
|
||||||
// it answers: the walk took what it reuses. Said, never fatal.
|
|
||||||
func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) {
|
|
||||||
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, pull := range pullsOfWalk(&p) {
|
|
||||||
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fmt.Printf("%s: %s\n", p.ID, said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAWalkNamesThePullRequestsOfItsMergesOnce(t *testing.T) {
|
|
||||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
|
||||||
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
|
||||||
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
|
||||||
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(pullsOfWalk(p), want) {
|
|
||||||
t.Fatalf("the walk names %+v", pullsOfWalk(p))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
|
||||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
|
||||||
g := aGateMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := g.open.inventory
|
|
||||||
gateEvery = 0
|
|
||||||
|
|
||||||
digest := "sha256:" + strings.Repeat("c", 64)
|
|
||||||
reused := func(id, reference string) link.BuildResult {
|
|
||||||
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
|
|
||||||
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
|
|
||||||
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
|
|
||||||
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
|
|
||||||
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
|
|
||||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
|
|
||||||
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
|
|
||||||
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
|
|
||||||
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
|
|
||||||
}
|
|
||||||
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), "scratch") {
|
|
||||||
t.Fatalf("a build in the scratch namespace was registered: %v", err)
|
|
||||||
}
|
|
||||||
id := link.NewBuildID(time.Now().Add(time.Second))
|
|
||||||
result := reused(id, "store.internal:5000/app/server@"+digest)
|
|
||||||
if _, _, err := takeIn(ctx, inv, result); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
current, err := inv.CurrentBuilds(ctx)
|
|
||||||
if err != nil || current["app"].Commit != "c3" {
|
|
||||||
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
|
|
||||||
}
|
|
||||||
shelf, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
raw, _ := json.Marshal(shelf["app"].Resources)
|
|
||||||
if !strings.Contains(string(raw), "app/server@"+digest) {
|
|
||||||
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
|
|
||||||
asked := time.Now().UTC()
|
|
||||||
p := g.plan(t)
|
|
||||||
p.Commit = "c3"
|
|
||||||
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
|
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
|
|
||||||
p = g.plan(t)
|
|
||||||
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
|
|
||||||
t.Fatalf("the walk keeps %+v", m)
|
|
||||||
}
|
|
||||||
for i := 0; i < 6; i++ {
|
|
||||||
advancePlans(ctx, g.open)
|
|
||||||
}
|
|
||||||
p = g.plan(t)
|
|
||||||
gate := p.Modules["app"].Gate
|
|
||||||
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
|
|
||||||
t.Fatalf("the gate judged %+v, not the reused build", gate)
|
|
||||||
}
|
|
||||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
|
|
||||||
t.Fatalf("the first machine was sent %v, not the reused build", sent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in.
|
|
||||||
|
|
||||||
// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the
|
|
||||||
// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one
|
|
||||||
// that reads the repository through a context not at all. A change that moves no module builds nothing.
|
|
||||||
func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
|
|
||||||
entries := []inventory.Entry{
|
|
||||||
{Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}},
|
|
||||||
{Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}},
|
|
||||||
{Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}},
|
|
||||||
}
|
|
||||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"}
|
|
||||||
scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}}
|
|
||||||
clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil }
|
|
||||||
|
|
||||||
got := checkBuildsOf(p, scope, entries, clone)
|
|
||||||
want := []link.CheckBuild{
|
|
||||||
{Module: "app", Path: "modules/app", Merged: true, Tier: 0},
|
|
||||||
{Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"},
|
|
||||||
{Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1},
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want)
|
|
||||||
}
|
|
||||||
if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil {
|
|
||||||
t.Fatalf("a change that moves no module asks for builds: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
|
|
||||||
// run never does, and neither does an ask that is not a walk's.
|
|
||||||
|
|
||||||
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
|
|
||||||
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
|
|
||||||
var forgot []builder.PullRef
|
|
||||||
was := forgetChecks
|
|
||||||
forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) {
|
|
||||||
forgot = append(forgot, pull)
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { forgetChecks = was })
|
|
||||||
p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
|
||||||
{Repository: "novox/shop", Number: 7}}}}
|
|
||||||
|
|
||||||
p.WalkSaid = inventory.WalkStarted
|
|
||||||
forgetChecksOfWalk(t.Context(), nil, p)
|
|
||||||
p.WalkSaid = ""
|
|
||||||
forgetChecksOfWalk(t.Context(), nil, p)
|
|
||||||
if len(forgot) != 0 {
|
|
||||||
t.Fatalf("a walk not ended let go of %v, which it may still take", forgot)
|
|
||||||
}
|
|
||||||
p.WalkSaid = inventory.WalkEnded
|
|
||||||
forgetChecksOfWalk(t.Context(), nil, p)
|
|
||||||
if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) {
|
|
||||||
t.Fatalf("the walk ended let go of %v", forgot)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is.
|
|
||||||
func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) {
|
|
||||||
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" {
|
|
||||||
t.Fatalf("a module's own build reads as a scratch build: %s", got)
|
|
||||||
}
|
|
||||||
scratch := "store.internal:5000/scratch/app/server@sha256:aa"
|
|
||||||
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch {
|
|
||||||
t.Fatalf("a scratch build was not found: %q", got)
|
|
||||||
}
|
|
||||||
if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" {
|
|
||||||
t.Fatal("a scratch archive was not found")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The walk says per module what it did: "reused from …" or "built: <why>" — and a build seat from before says
|
|
||||||
// neither, which is said.
|
|
||||||
func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
|
||||||
if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" {
|
|
||||||
t.Fatalf("said %q", got)
|
|
||||||
}
|
|
||||||
if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" {
|
|
||||||
t.Fatalf("said %q", got)
|
|
||||||
}
|
|
||||||
if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") {
|
|
||||||
t.Fatalf("said %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the
|
|
||||||
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
|
|
||||||
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
|
||||||
// is named in testdata/store-tests.
|
|
||||||
|
|
||||||
// A check is handed only the builds earlier checks of its own pull request at its own head made: another pull
|
|
||||||
// request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read.
|
|
||||||
func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) {
|
|
||||||
was := checkedBuildsOf
|
|
||||||
t.Cleanup(func() { checkedBuildsOf = was })
|
|
||||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
|
||||||
mine := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
|
||||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-0", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
|
||||||
another := link.CheckedBuild{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 8}, Check: "build-9",
|
|
||||||
Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
|
||||||
var asked [][]link.PullRef
|
|
||||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
|
||||||
asked = append(asked, pulls)
|
|
||||||
return []link.CheckedBuild{mine, earlier, another}, nil
|
|
||||||
}
|
|
||||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Commit: "bbbbbbbbbbbb0123456789"}
|
|
||||||
builds := []link.CheckBuild{{Module: "app"}}
|
|
||||||
if got := namedForTheCheck(t.Context(), p, builds); !reflect.DeepEqual(got, []link.CheckedBuild{mine}) ||
|
|
||||||
!reflect.DeepEqual(asked, [][]link.PullRef{{pull}}) {
|
|
||||||
t.Fatalf("the check is handed %+v, read for %v", got, asked)
|
|
||||||
}
|
|
||||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
|
||||||
return nil, errors.New("no bus")
|
|
||||||
}
|
|
||||||
if got := namedForTheCheck(t.Context(), p, builds); got != nil {
|
|
||||||
t.Fatalf("verdicts that could not be read named %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,203 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
|
|
||||||
// runs it** (novox/hq issue 286): the same code (builder.Check), the same ask the controller would make of
|
|
||||||
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
|
|
||||||
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
|
|
||||||
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
|
|
||||||
// the mesh runs.
|
|
||||||
//
|
|
||||||
// **The build seat is the reference.** A merge-check.sh that passed on an agent's machine failed on the
|
|
||||||
// seat for three reasons that were each the agent's environment, never the change: a newer Go whose gofmt
|
|
||||||
// lays a file out differently, a sibling checkout at the agent's feature branch where the seat had the
|
|
||||||
// commit the mesh runs, and a different user. Run here, a check sees what the seat will.
|
|
||||||
//
|
|
||||||
// check-here [--tree <checkout>] [--base main] [--registry <artifact store>] [--forge <url of the owner>]
|
|
||||||
// [--number <n>] [--user uid:gid] [--facts store|<file>] [--keep]
|
|
||||||
//
|
|
||||||
// The checkout's HEAD is what is checked, and it must be committed: the seat checks a commit, never a
|
|
||||||
// working tree.
|
|
||||||
func checkHereCommand(ctx context.Context, args []string) error {
|
|
||||||
set := flag.NewFlagSet("check-here", flag.ContinueOnError)
|
|
||||||
tree := set.String("tree", ".", "the change's checkout; its HEAD is checked")
|
|
||||||
base := set.String("base", "main", "the branch the change would merge into")
|
|
||||||
registry := set.String("registry", os.Getenv("MESH_REGISTRY"), "the artifact store holding the facts and the toolchains")
|
|
||||||
forge := set.String("forge", "", "where the repositories beside it are cloned from, as <forge>/<repository>.git; "+
|
|
||||||
"the checkout's origin without its own name when not given")
|
|
||||||
number := set.Int("number", 0, "the pull request's number, when there is one")
|
|
||||||
// The build seat's service runs as root, and its check containers run as the builder does.
|
|
||||||
user := set.String("user", "0:0", "the user the check's containers run as: the build seat's")
|
|
||||||
keep := set.Bool("keep", false, "keep the workspace afterwards")
|
|
||||||
factsFrom := set.String("facts", "store", "the facts snapshot: `store`, the one the artifact store holds — "+
|
|
||||||
"what the seat reads — or a file")
|
|
||||||
if _, err := parseAround(set, args); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if *registry == "" {
|
|
||||||
return errors.New("check-here reads the facts and the toolchains from the artifact store: --registry <host:port> or MESH_REGISTRY")
|
|
||||||
}
|
|
||||||
dir, err := filepath.Abs(*tree)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
git := func(args ...string) (string, error) {
|
|
||||||
cmd := exec.CommandContext(ctx, "git", args...)
|
|
||||||
cmd.Dir = dir
|
|
||||||
out, err := cmd.Output()
|
|
||||||
return strings.TrimSpace(string(out)), err
|
|
||||||
}
|
|
||||||
if dirty, err := git("status", "--porcelain", "--untracked-files=no"); err != nil {
|
|
||||||
return fmt.Errorf("%s is not a checkout: %w", dir, err)
|
|
||||||
} else if dirty != "" {
|
|
||||||
return errors.New("the checkout has changes not committed: the build seat checks a commit, so commit first")
|
|
||||||
}
|
|
||||||
head, err := git("rev-parse", "HEAD")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
origin, err := git("remote", "get-url", "origin")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the checkout has no origin to say which repository it is: %w", err)
|
|
||||||
}
|
|
||||||
owner, repo, prefix := ownerRepoOf(origin)
|
|
||||||
if *forge == "" {
|
|
||||||
*forge = prefix
|
|
||||||
}
|
|
||||||
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
|
|
||||||
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
|
|
||||||
}
|
|
||||||
// Without rename detection, so a file moved out of a build source is said under its old name as well:
|
|
||||||
// its going is a change to the build that held it (novox/hq ADR 0267).
|
|
||||||
changedText, err := git("diff", "--name-only", "--no-renames", "origin/"+*base+"...HEAD")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var paths, removed []string
|
|
||||||
for _, p := range strings.Split(changedText, "\n") {
|
|
||||||
if p = strings.TrimSpace(p); p == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
paths = append(paths, p)
|
|
||||||
if _, err := os.Stat(filepath.Join(dir, p)); os.IsNotExist(err) {
|
|
||||||
removed = append(removed, p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = os.Setenv("MESH_REGISTRY", *registry)
|
|
||||||
f, err := readFacts(ctx, *factsFrom)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the facts snapshot cannot be read: %w", err)
|
|
||||||
}
|
|
||||||
entries, read, edges, err := graphOfFacts(f)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
p := link.PullUpdated{Owner: owner, Repo: repo, Number: *number, Base: *base, Commit: head, Paths: paths,
|
|
||||||
Removed: removed, ModuleDirs: moduleDirsIn(dir, paths), ModuleDirsSaid: true}
|
|
||||||
scope := pullScope(p, entries, read, edges)
|
|
||||||
_, scriptErr := os.Stat(filepath.Join(dir, builder.CheckScript))
|
|
||||||
if !scope.gated() && !scope.Mesh {
|
|
||||||
fmt.Printf("%s: %s, and the repository is not the mesh's: the build seat runs nothing for it\n",
|
|
||||||
owner+"/"+repo, noModuleTouched)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !scope.gated() && scriptErr != nil {
|
|
||||||
fmt.Printf("%s: %s; %s\n", owner+"/"+repo, noModuleTouched, noMergeCheck)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
toolchains := map[string]string{}
|
|
||||||
for language, reference := range f.Versions.Toolchains {
|
|
||||||
toolchains[language] = catalogue.Rerouted(reference, *registry)
|
|
||||||
}
|
|
||||||
if len(toolchains) == 0 {
|
|
||||||
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
|
|
||||||
"issue 286, and the seat's toolchain cannot be known here")
|
|
||||||
}
|
|
||||||
beside := map[string]builder.Beside{}
|
|
||||||
for d, ref := range f.Beside {
|
|
||||||
from := d
|
|
||||||
if d == "mesh-controller-main" {
|
|
||||||
from = "mesh-controller"
|
|
||||||
}
|
|
||||||
beside[d] = builder.Beside{Repository: strings.TrimSuffix(*forge, "/") + "/" + from + ".git", Ref: ref}
|
|
||||||
}
|
|
||||||
id := fmt.Sprintf("check-here-%d", time.Now().UnixNano())
|
|
||||||
spec := builder.CheckSpec{ID: id, Repository: dir, Ref: head, Owner: owner, Repo: repo, Number: *number,
|
|
||||||
Paths: paths, Beside: beside, Modules: scope.Modules, New: scope.New, Manifests: scope.Manifests,
|
|
||||||
Base: *base, Judge: scope.Judge, Toolchain: toolchains["go"], Toolchains: toolchains, User: *user}
|
|
||||||
|
|
||||||
workspace, err := os.MkdirTemp("", "mesh-check-here-")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !*keep {
|
|
||||||
defer removeWorkspace(spec.Toolchain, workspace, *user)
|
|
||||||
}
|
|
||||||
fmt.Fprintf(os.Stderr, "checking %s/%s at %.8s as the build seat would, against the facts of %s, in %s\n",
|
|
||||||
owner, repo, head, f.Taken.Format(time.RFC3339), workspace)
|
|
||||||
v, err := builder.Check(ctx, builder.Command, spec, workspace, *registry, builder.GitCredential{},
|
|
||||||
func(step, message string) {
|
|
||||||
if step != "output" {
|
|
||||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the check could not run — on the seat an error, never a pass: %w", err)
|
|
||||||
}
|
|
||||||
fmt.Println(v.Report)
|
|
||||||
fmt.Println()
|
|
||||||
fmt.Printf("mesh/merge-gate: %s — %s\n", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary)
|
|
||||||
if v.Repo != nil {
|
|
||||||
fmt.Printf("mesh/repo-check: %s — %s\n", strings.ToUpper(v.Repo.Verdict), v.Repo.Summary)
|
|
||||||
}
|
|
||||||
for _, l := range []*builder.Layer{v.Gate, v.Repo} {
|
|
||||||
if l != nil && l.Verdict != "pass" && l.Verdict != "warning" {
|
|
||||||
return errors.New("the build seat would not pass this change")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ownerRepoOf reads owner, repository and the owner's URL from a remote: ssh://git@host:222/novox/mesh-host.git
|
|
||||||
// → novox, mesh-host, ssh://git@host:222/novox.
|
|
||||||
func ownerRepoOf(remote string) (string, string, string) {
|
|
||||||
trimmed := strings.TrimSuffix(strings.TrimSuffix(remote, "/"), ".git")
|
|
||||||
cut := strings.LastIndexAny(trimmed, "/:")
|
|
||||||
if cut < 0 {
|
|
||||||
return "", trimmed, ""
|
|
||||||
}
|
|
||||||
repo, prefix := trimmed[cut+1:], trimmed[:cut]
|
|
||||||
owner := prefix
|
|
||||||
if at := strings.LastIndexAny(prefix, "/:"); at >= 0 {
|
|
||||||
owner = prefix[at+1:]
|
|
||||||
}
|
|
||||||
return owner, repo, prefix
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeWorkspace removes what the check left, written as the seat's user: by a container of that user
|
|
||||||
// when it is not this one.
|
|
||||||
func removeWorkspace(image, workspace, user string) {
|
|
||||||
if image != "" && user != fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) {
|
|
||||||
// Everything in it — the check's HOME is the workspace, so the toolchain's own files are there too.
|
|
||||||
_ = exec.Command("docker", "run", "--rm", "--user", user, "--volume", workspace+":/workspace", image,
|
|
||||||
"sh", "-c", "rm -rf /workspace/* /workspace/.[!.]*").Run()
|
|
||||||
}
|
|
||||||
_ = os.RemoveAll(workspace)
|
|
||||||
}
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1212/AC1 and AC2: the check request names the pulls bucket of each module claiming the delivery seat, in the order
|
|
||||||
// of the claimants' names.
|
|
||||||
func TestACheckNamesThePullsBucketsOfTheSeatsClaimants(t *testing.T) {
|
|
||||||
for name, c := range map[string]struct {
|
|
||||||
entries []inventory.Entry
|
|
||||||
want []string
|
|
||||||
}{
|
|
||||||
"only the gate": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")}, []string{"mesh-delivery-gate_pulls"}},
|
|
||||||
"only today's": {[]inventory.Entry{claimingTheSeat("mesh-delivery"), namedOnly("gitea")}, []string{"mesh-delivery_pulls"}},
|
|
||||||
"both, name order": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls", "mesh-delivery-gate_pulls"}},
|
|
||||||
"none": {[]inventory.Entry{namedOnly("gitea")}, nil},
|
|
||||||
} {
|
|
||||||
if got := pullsBucketsOf(c.entries); !reflect.DeepEqual(got, c.want) {
|
|
||||||
t.Errorf("%s: %v, want %v", name, got, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1212/AC1 and AC2: the check a request carries names those buckets (the wiring, not only the helper).
|
|
||||||
func TestTheCheckRequestCarriesThePullsBuckets(t *testing.T) {
|
|
||||||
for name, c := range map[string]struct {
|
|
||||||
entries []inventory.Entry
|
|
||||||
want []string
|
|
||||||
}{
|
|
||||||
"only the gate": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}, []string{"mesh-delivery-gate_pulls"}},
|
|
||||||
"only today's": {[]inventory.Entry{claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls"}},
|
|
||||||
"both, name order": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls", "mesh-delivery-gate_pulls"}},
|
|
||||||
} {
|
|
||||||
got := checkOf(link.PullUpdated{Owner: "novox", Repo: "a", Number: 7}, checkScope{}, c.entries, nil, nil, nil, nil)
|
|
||||||
if !reflect.DeepEqual(got.Pulls, c.want) || got.Owner != "novox" || got.Number != 7 {
|
|
||||||
t.Errorf("%s: %+v, want pulls %v", name, got, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+18
-563
@@ -2,9 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
@@ -13,46 +10,18 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A pull request's merge check (novox/hq to-be 45 §9, ADR 0237 as amended 2026-10-06): the forge
|
// What a pull request reaches of the mesh's graph (novox/hq to-be 45 §9, issue 280). The controller no longer checks a
|
||||||
// announces every pull request's new head, the controller decides what is checked, asks the build seat to
|
// pull request or says a verdict on it (R4g, ADR 0503 §3, ADR 0501 §5): the checker judges the heads, and mesh-delivery
|
||||||
// check it, and says the verdict as `checked`, which the forge's holder sets as the pull request's
|
// asks the operator's Approve on its verdict. What stays here is the mapping of a pull request's changed paths onto the
|
||||||
// statuses. **Before merge, never after**: every check the mesh had ran after a merge, on a machine.
|
// mesh's module graph by the planner's own answer (reachOfMerge, touchedBy), which the delivery verbs (`delivery order`,
|
||||||
//
|
// the change plan) and the head's declared config (tellPullDeclares) read: **a changed file touches exactly the modules
|
||||||
// **The mesh's module graph decides, not the repository.** The controller holds the graph — every module,
|
// whose build reads it** — a module's own directory (the whole repository for one built from its root), or a repository
|
||||||
// the repository and directory it is built from — and maps the pull request's changed paths onto it by
|
// its recipe packages — and a directory the change adds a module.json in is a new module.
|
||||||
// the planner's own answer (reachOfMerge, touchedBy — the one place a changed file is mapped onto modules,
|
|
||||||
// for the merge handler, the release planner, the merge gate and this check; issue 280): **a changed file
|
|
||||||
// touches exactly the modules whose build reads it** — a module's own directory (the whole repository for
|
|
||||||
// one built from its root), or a repository its recipe packages. A file no build reads — a script at the
|
|
||||||
// root, a README — touches no module. A directory the change adds a module.json in, which the graph does
|
|
||||||
// not hold yet (said by the head, issue 278), is a new module and is checked too.
|
|
||||||
//
|
|
||||||
// - touches a module: the build seat runs **the gate** — `mesh/merge-gate`, the touched manifests, every
|
|
||||||
// machine composed with the change, the replays — and the repository's own merge-check.sh beside it;
|
|
||||||
// - touches none, in a repository that is the mesh's (it sources a module on some branch, or shares the
|
|
||||||
// core's owner): the gate is a pass that says so — a fact, not a missing check — and the repository's
|
|
||||||
// own merge-check.sh runs as `mesh/repo-check`, a warning when it has none;
|
|
||||||
// - touches none, anywhere else: the gate is a pass that says so, and nothing more is said.
|
|
||||||
//
|
|
||||||
// What is checked is decided here and run there (internal/builder/check.go).
|
|
||||||
|
|
||||||
// checkTimeout is how long one check may run on the build seat. Said here so the ask's watchdog (S6)
|
|
||||||
// and the builder agree on what late means.
|
|
||||||
const checkTimeout = 45 * time.Minute
|
|
||||||
|
|
||||||
// noModuleTouched is the gate's word for a change that touches nothing of the graph.
|
|
||||||
const noModuleTouched = "the change touches no module of the mesh's graph"
|
|
||||||
|
|
||||||
// noMergeCheck is the repository layer's word for a repository of the mesh with no merge-check.sh.
|
|
||||||
const noMergeCheck = "the repository declares no merge-check.sh: none of its own tests run before it merges"
|
|
||||||
|
|
||||||
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
|
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
|
||||||
// each beside it — and whose owner is the mesh's own.
|
// each beside it — and whose owner is the mesh's own.
|
||||||
@@ -69,15 +38,11 @@ type checkScope struct {
|
|||||||
New []string
|
New []string
|
||||||
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
|
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
|
||||||
Manifests []string
|
Manifests []string
|
||||||
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
|
// Unread the changed files no module's build reads.
|
||||||
// module's build reads.
|
|
||||||
Width, Tiers int
|
|
||||||
Unread []string
|
Unread []string
|
||||||
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
|
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
|
||||||
// core's, or the change adds a module to it.
|
// core's, or the change adds a module to it.
|
||||||
Mesh bool
|
Mesh bool
|
||||||
// Judge is who judges the gate (link.JudgeSelf, or the running controller).
|
|
||||||
Judge string
|
|
||||||
// From is a module built from the repository, for how the mesh clones it; nil when none is.
|
// From is a module built from the repository, for how the mesh clones it; nil when none is.
|
||||||
From *inventory.Entry
|
From *inventory.Entry
|
||||||
// Reach is the planner's whole answer, which the change plan is made from.
|
// Reach is the planner's whole answer, which the change plan is made from.
|
||||||
@@ -95,14 +60,9 @@ func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]
|
|||||||
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
|
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
|
||||||
ModuleDirsSaid: p.ModuleDirsSaid}
|
ModuleDirsSaid: p.ModuleDirsSaid}
|
||||||
r := reachOfMerge(m, entries, read, edges)
|
r := reachOfMerge(m, entries, read, edges)
|
||||||
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
|
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread, Reach: r}
|
||||||
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
|
|
||||||
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
|
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
|
||||||
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
|
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
|
||||||
switch e.Manifest.Module {
|
|
||||||
case "mesh-controller":
|
|
||||||
s.Judge = link.JudgeSelf
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for _, d := range r.Added {
|
for _, d := range r.Added {
|
||||||
s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile))
|
s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile))
|
||||||
@@ -138,35 +98,29 @@ func sourceOwner(repository string) string {
|
|||||||
return strings.ToLower(parts[len(parts)-2])
|
return strings.ToLower(parts[len(parts)-2])
|
||||||
}
|
}
|
||||||
|
|
||||||
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
|
// PullUpdated hears a pull request's new head. It asks no check and says no verdict (pullUpdated).
|
||||||
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||||
_, err := f.pullUpdated(ctx, p, os.Stdout)
|
return f.pullUpdated(ctx, p, os.Stdout)
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// pullUpdated is PullUpdated saying what it did to w. It returns the id of the check it asked of the build seat, or
|
// pullUpdated is PullUpdated saying what it did to w. It asks nothing of anybody and says no verdict (R4g; ADR 0503 §3, §5,
|
||||||
// "" when it said a verdict itself and asked nothing. A command that answers JSON on standard output (`delivery
|
// ADR 0527): the checker judges every repository whose trunk requires `mesh/check`, hq included. All it still does is tell
|
||||||
// check`) gives it standard error, so its output stays the document alone (novox/hq issue 1311).
|
// mesh-config what the head declares (ADR 0502), whatever judges it.
|
||||||
func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Writer) (string, error) {
|
func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Writer) error {
|
||||||
inv := f.open.inventory
|
inv := f.open.inventory
|
||||||
entries, err := inv.Catalogued(ctx)
|
entries, err := inv.Catalogued(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
read, err := readForPlanning(ctx, inv)
|
read, err := readForPlanning(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
edges, err := inv.Dependencies(ctx)
|
edges, err := inv.Dependencies(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
scope := pullScope(p, entries, read, edges)
|
scope := pullScope(p, entries, read, edges)
|
||||||
// **The merge gate's checks stop** (novox/hq ADR 0503 §3, §5; issue 1299): the checker judges every repository whose
|
|
||||||
// trunk requires `mesh/check`. A repository whose trunk requires `mesh/repo-check` (hq), whatever else it requires
|
|
||||||
// (ADR 0513 §4), has its own merge-check.sh asked of build-agent, and said with that layer alone; any other is asked
|
|
||||||
// nothing and said nothing. What the forge's protection requires is read, never listed (lead decision L11).
|
|
||||||
// Mesh-config is still told what the head declares (ADR 0502), whatever judges it.
|
|
||||||
this := pullAtOf(p)
|
this := pullAtOf(p)
|
||||||
if this.Truncated {
|
if this.Truncated {
|
||||||
this.Paths, this.Truncated = truncatedPaths(entries, p.Owner, p.Repo, scope.Modules), false
|
this.Paths, this.Truncated = truncatedPaths(entries, p.Owner, p.Repo, scope.Modules), false
|
||||||
@@ -174,310 +128,7 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri
|
|||||||
if scope.gated() || holdsModules(entries, p.Owner, p.Repo) {
|
if scope.gated() || holdsModules(entries, p.Owner, p.Repo) {
|
||||||
tellPullDeclares(ctx, w, []pullAt{this})
|
tellPullDeclares(ctx, w, []pullAt{this})
|
||||||
}
|
}
|
||||||
required, err := requiredStatusesOf(ctx, p.Owner, p.Repo, p.Base)
|
|
||||||
switch {
|
|
||||||
case err != nil:
|
|
||||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): what %s's protection requires cannot be read, so nothing is asked and nothing is "+
|
|
||||||
"said: %v\n", p.Owner, p.Repo, p.Number, p.Commit, p.Base, err)
|
|
||||||
return "", nil
|
|
||||||
case !asksRepoCheck(required):
|
|
||||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s does not require %s, and %s; nothing is asked\n", p.Owner, p.Repo, p.Number,
|
|
||||||
p.Commit, p.Base, sdkdelivery.StatusContextRepoCheck, checkerWords(required))
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
return f.repoCheckAlone(ctx, w, p, scope, entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
// asksRepoCheck says what a trunk's protection has the controller ask: its repository's own merge-check.sh whenever the
|
|
||||||
// trunk requires `mesh/repo-check`, whatever else it requires (ADR 0513 §4: a trunk requiring both has the checker judge
|
|
||||||
// `mesh/check` and the repository check asked beside it), and nothing otherwise.
|
|
||||||
func asksRepoCheck(required []string) bool {
|
|
||||||
return slices.Contains(required, sdkdelivery.StatusContextRepoCheck)
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkerWords says who judges a trunk that requires no `mesh/repo-check`.
|
|
||||||
func checkerWords(required []string) string {
|
|
||||||
if slices.Contains(required, sdkdelivery.StatusContextCheck) {
|
|
||||||
return "the checker judges its " + sdkdelivery.StatusContextCheck
|
|
||||||
}
|
|
||||||
return "does not require " + sdkdelivery.StatusContextCheck + " either, so nothing is said"
|
|
||||||
}
|
|
||||||
|
|
||||||
// repoCheckAlone asks build-agent for a repository's own merge-check.sh alone, for a pull request of a repository whose
|
|
||||||
// trunk requires `mesh/repo-check`, whether or not it also requires `mesh/check` (ADR 0503 §5, ADR 0513 §4): no gate, no module built, its verdict said with the
|
|
||||||
// repo-check layer alone.
|
|
||||||
func (f following) repoCheckAlone(ctx context.Context, w io.Writer, p link.PullUpdated, scope checkScope,
|
|
||||||
entries []inventory.Entry) (string, error) {
|
|
||||||
inv := f.open.inventory
|
|
||||||
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
|
||||||
u, err := inv.UpgradeOf(ctx, module)
|
|
||||||
return u, err == nil
|
|
||||||
})
|
|
||||||
fmt.Fprint(w, planText(plan))
|
|
||||||
direct := directVerdict(p, plan)
|
|
||||||
switch {
|
|
||||||
case !scope.Mesh:
|
|
||||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: "the repository is not the mesh's: nothing run"}
|
|
||||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): the repository is not the mesh's: said, nothing run\n", p.Owner, p.Repo, p.Number,
|
|
||||||
p.Commit)
|
|
||||||
sayChecked(ctx, w, direct)
|
|
||||||
return "", nil
|
|
||||||
case p.MergeCheckSaid && !p.MergeCheck:
|
|
||||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
|
|
||||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, noMergeCheck)
|
|
||||||
sayChecked(ctx, w, direct)
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
request, err := checkRequestFor(ctx, f.open, p, checkScope{Mesh: true, From: scope.From}, entries)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
request.Check.Plan = &plan
|
|
||||||
seat := buildSeatHeld(ctx)
|
|
||||||
ask, err := askOverOn(seat)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer ask.Close()
|
|
||||||
if err := ask.Ask(ctx, request); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): asked %s to run its own merge-check.sh alone, as %s\n", p.Owner, p.Repo, p.Number,
|
|
||||||
p.Commit, seat, request.ID)
|
|
||||||
return request.ID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// requiredStatusesOf is what the forge's protection of a repository's branch requires before a merge, as the forge
|
|
||||||
// picks the rule it applies (the one named for the branch, else the first glob covering it). A variable so a test needs
|
|
||||||
// no forge.
|
|
||||||
var requiredStatusesOf = func(ctx context.Context, owner, repo, branch string) ([]string, error) {
|
|
||||||
var rules struct {
|
|
||||||
Rules []protectionRule `json:"rules"`
|
|
||||||
}
|
|
||||||
js, err := aBus()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
raw, _ := json.Marshal(map[string]any{"owner": owner, "repo": repo})
|
|
||||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_branch_protection_get", raw, 30*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if answer.Error != "" {
|
|
||||||
return nil, fmt.Errorf("gitea.gitea_branch_protection_get: %s", answer.Error)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(answer.Result, &rules); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
r := appliedRule(rules.Rules, branch)
|
|
||||||
if r == nil {
|
|
||||||
// empty-on-error: not an error; no rule covers the branch, so it requires nothing, which the caller says
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return r.RequiredStatuses, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1).
|
|
||||||
const composedWords = "judged by its group's composed check"
|
|
||||||
|
|
||||||
// gateComposed is a gate layer's verdict when no gate was run for the member alone: never a pass. Mesh-delivery reads it
|
|
||||||
// as the composed check's to judge, and the forge's holder sets no mesh/merge-gate from it.
|
|
||||||
const gateComposed = "composed"
|
|
||||||
|
|
||||||
// composedGate says a group member's own verdict has no gate of its own: its gate layer, and the verdict that is
|
|
||||||
// the gate's, read "composed" in place of a pass.
|
|
||||||
func composedGate(c *link.Checked) {
|
|
||||||
c.Verdict, c.Summary = gateComposed, composedWords
|
|
||||||
c.Gate = &link.CheckLayer{Verdict: gateComposed, Summary: composedWords}
|
|
||||||
}
|
|
||||||
|
|
||||||
// directVerdict is the verdict of a change no build judges — it touches no module, and either the repository is not
|
|
||||||
// the mesh's or the head holds no merge-check.sh. **It names the trunk commit it holds for** (novox/hq ADR 0323 §3):
|
|
||||||
// the base branch's head the forge announced with the head, so the merge tool can weigh it against the trunk's head
|
|
||||||
// and does not refuse it for ever (a verdict naming no trunk is never the trunk's head's). Nothing was built to
|
|
||||||
// judge: the trunk commit is the one the forge said the change was announced against.
|
|
||||||
func directVerdict(p link.PullUpdated, plan link.ChangePlan) link.Checked {
|
|
||||||
trunk := p.BaseSHA
|
|
||||||
if len(trunk) > 12 {
|
|
||||||
trunk = trunk[:12]
|
|
||||||
}
|
|
||||||
if trunk != "" {
|
|
||||||
plan.Trunk = trunk
|
|
||||||
}
|
|
||||||
return link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
|
||||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
|
||||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan, Trunk: trunk}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
|
|
||||||
// and what is read beside it.
|
|
||||||
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
|
|
||||||
entries []inventory.Entry) (link.BuildRequest, error) {
|
|
||||||
shelf := map[string]catalogue.Manifest{}
|
|
||||||
for _, e := range entries {
|
|
||||||
shelf[e.Manifest.Module] = e.Manifest
|
|
||||||
}
|
|
||||||
world, err := restOfTheMesh(ctx, open, shelf, "")
|
|
||||||
if err != nil {
|
|
||||||
return link.BuildRequest{}, err
|
|
||||||
}
|
|
||||||
clone := func(s inventory.Source) (string, error) {
|
|
||||||
if s.Seat == "" {
|
|
||||||
return s.Repository, nil
|
|
||||||
}
|
|
||||||
return clonedFromSeat(world, s.Seat, s.Repository)
|
|
||||||
}
|
|
||||||
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
|
|
||||||
source := inventory.Source{Seat: gitSeat, Repository: p.Owner + "/" + p.Repo}
|
|
||||||
if scope.From != nil {
|
|
||||||
source = scope.From.Source
|
|
||||||
}
|
|
||||||
repository, err := clone(source)
|
|
||||||
if err != nil {
|
|
||||||
return link.BuildRequest{}, err
|
|
||||||
}
|
|
||||||
current, err := open.inventory.CurrentBuilds(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return link.BuildRequest{}, err
|
|
||||||
}
|
|
||||||
beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone)
|
|
||||||
if err != nil {
|
|
||||||
return link.BuildRequest{}, err
|
|
||||||
}
|
|
||||||
builds := checkBuildsOf(p, scope, entries, clone)
|
|
||||||
return link.BuildRequest{
|
|
||||||
ID: link.NewBuildID(time.Now()),
|
|
||||||
Repository: repository,
|
|
||||||
Ref: p.Commit,
|
|
||||||
Held: heldBy(ctx),
|
|
||||||
Seats: seatBases(ctx),
|
|
||||||
Source: sourceOnSeat(source),
|
|
||||||
Check: checkOf(p, scope, entries, beside, builds, namedForTheCheck(ctx, p, builds), runningBy(ctx)),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkOf is the check a request carries: the pull request, what the change touches, and the buckets of the pull
|
|
||||||
// requests' keys of the modules claiming the delivery seat (novox/hq ADR 0437 §3, issue 1212).
|
|
||||||
func checkOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry, beside map[string]link.CheckedOut,
|
|
||||||
builds []link.CheckBuild, named []link.CheckedBuild, running map[string]string) *link.CheckRequest {
|
|
||||||
return &link.CheckRequest{Pulls: pullsBucketsOf(entries), Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
|
||||||
Trunk: p.BaseSHA, Ready: p.Ready, Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents,
|
|
||||||
New: scope.New, Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: named, Running: running}
|
|
||||||
}
|
|
||||||
|
|
||||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
|
||||||
// by the module the mesh builds from it, at besideRefs. running is the commit the mesh runs of a module.
|
|
||||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
|
||||||
clone func(inventory.Source) (string, error)) (map[string]link.CheckedOut, error) {
|
|
||||||
beside := map[string]link.CheckedOut{}
|
|
||||||
for _, e := range entries {
|
|
||||||
dir, core := coreModules[e.Manifest.Module]
|
|
||||||
if !core || e.Provided || e.Source.Repository == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
url, err := clone(e.Source)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
refs := besideRefs(dir, running(e.Manifest.Module))
|
|
||||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
|
||||||
if dir == "mesh-controller" {
|
|
||||||
// The controller's main stays its main: it judges only a check the running controller predates.
|
|
||||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
|
||||||
if e.Source.Seat != "" {
|
|
||||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
|
||||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
|
||||||
sibling)}); err == nil {
|
|
||||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return beside, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// trunkOf is the branch a module's source follows: main when it records none, or records a commit.
|
|
||||||
func trunkOf(s inventory.Source) string {
|
|
||||||
if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" {
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
return "main"
|
|
||||||
}
|
|
||||||
|
|
||||||
// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the
|
|
||||||
// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key
|
|
||||||
// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be
|
|
||||||
// read, and the check builds, as before.
|
|
||||||
func namedForTheCheck(ctx context.Context, p link.PullUpdated, builds []link.CheckBuild) []link.CheckedBuild {
|
|
||||||
if len(builds) == 0 || p.Commit == "" {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
|
||||||
pull := link.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}
|
|
||||||
read, err := checkedBuildsOf(ctx, []link.PullRef{pull})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf(" the earlier checks' builds of %s/%s#%d could not be read, so the check builds: %v\n",
|
|
||||||
p.Owner, p.Repo, p.Number, err)
|
|
||||||
// empty-on-error: said above; a check handed no named build builds every module, as before issue 703
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var named []link.CheckedBuild
|
|
||||||
for _, b := range read {
|
|
||||||
if b.Pull == pull && link.SameHead(b.Head, p.Commit) {
|
|
||||||
named = append(named, b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return named
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
|
|
||||||
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
|
|
||||||
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
|
|
||||||
// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context,
|
|
||||||
// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that
|
|
||||||
// moves no module builds nothing.
|
|
||||||
func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry,
|
|
||||||
clone func(inventory.Source) (string, error)) []link.CheckBuild {
|
|
||||||
if len(scope.Reach.Plan.Tiers) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
|
|
||||||
byName := map[string]inventory.Entry{}
|
|
||||||
for _, e := range entries {
|
|
||||||
byName[e.Manifest.Module] = e
|
|
||||||
}
|
|
||||||
skip := map[string]string{}
|
|
||||||
for _, e := range scope.Reach.Packaging {
|
|
||||||
skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " +
|
|
||||||
"cannot hand the merge"
|
|
||||||
}
|
|
||||||
for _, e := range scope.Reach.Deleted {
|
|
||||||
skip[e.Manifest.Module] = "the merge deletes it"
|
|
||||||
}
|
|
||||||
var out []link.CheckBuild
|
|
||||||
for tier, names := range scope.Reach.Plan.Tiers {
|
|
||||||
for _, name := range names {
|
|
||||||
e, known := byName[name]
|
|
||||||
b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]}
|
|
||||||
switch {
|
|
||||||
case !known:
|
|
||||||
b.Skip = "the catalogue does not hold it"
|
|
||||||
case b.Skip != "":
|
|
||||||
case sourceIs(e.Source, m):
|
|
||||||
b.Merged, b.Path = true, e.Source.Path
|
|
||||||
default:
|
|
||||||
url, err := clone(e.Source)
|
|
||||||
if err != nil {
|
|
||||||
b.Skip = "its repository cannot be cloned: " + err.Error()
|
|
||||||
break
|
|
||||||
}
|
|
||||||
b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref)
|
|
||||||
}
|
|
||||||
out = append(out, b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||||
@@ -533,201 +184,5 @@ func sdkPinned() string {
|
|||||||
return "main"
|
return "main"
|
||||||
}
|
}
|
||||||
|
|
||||||
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
|
|
||||||
func siblingOf(repository, name string) string {
|
|
||||||
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
|
|
||||||
return repository[:cut+1] + name
|
|
||||||
}
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceOnSeat is a source's seat form, nil for one on no seat.
|
|
||||||
func sourceOnSeat(s inventory.Source) *link.SourceOnSeat {
|
|
||||||
if s.Seat == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &link.SourceOnSeat{Seat: s.Seat, Repository: s.Repository}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkEvents is where the serving controller says a check's verdict; nil in a command.
|
// checkEvents is where the serving controller says a check's verdict; nil in a command.
|
||||||
var checkEvents link.Bus
|
var checkEvents link.Bus
|
||||||
|
|
||||||
// maxCheckReport is how much of a check's report travels in its verdict: enough for the failures and
|
|
||||||
// the machines, never a log.
|
|
||||||
const maxCheckReport = 60 << 10
|
|
||||||
|
|
||||||
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
|
|
||||||
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
|
|
||||||
func checked(ctx context.Context, result link.BuildResult) {
|
|
||||||
// **A check stopped by its key says nothing** (novox/hq ADR 0406 §4, issue 974, review 2): it stopped because the head or
|
|
||||||
// the trunk commit moved on, and the newer request will judge. Said as an error it would name the live head — red
|
|
||||||
// statuses on the forge, a delivery refused — and written to the key it would stand in the newer request's way.
|
|
||||||
if result.Failed == link.SupersededByKey {
|
|
||||||
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
sayChecked(ctx, os.Stdout, checkedOf(result))
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
|
|
||||||
// could not run.
|
|
||||||
func checkedOf(result link.BuildResult) link.Checked {
|
|
||||||
c := link.Checked{ID: result.ID, On: result.On, Commit: result.Ref}
|
|
||||||
if result.Checked != nil {
|
|
||||||
c.Owner, c.Repo, c.Number = result.Checked.Owner, result.Checked.Repo, result.Checked.Number
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case result.Check != nil:
|
|
||||||
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
|
|
||||||
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
|
|
||||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
|
||||||
// repositories read beside it.
|
|
||||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
|
||||||
c.Built = result.Check.Built
|
|
||||||
c.Validator = result.Check.Validator
|
|
||||||
c.Siblings, c.SiblingsUnknown, c.JudgeFrom = result.Check.Siblings, result.Check.SiblingsUnknown, result.Check.JudgeFrom
|
|
||||||
if c.Gate == nil {
|
|
||||||
// A build seat from before the layers: its verdict is the gate's.
|
|
||||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
|
||||||
}
|
|
||||||
case result.Failed != "":
|
|
||||||
// The check could not run: an error, never read as a pass — on both layers it was asked for.
|
|
||||||
c.Verdict, c.Summary = "error", "the check could not run: "+firstLine(result.Failed)
|
|
||||||
default:
|
|
||||||
c.Verdict, c.Summary = "error", "the build seat answered the check with no verdict"
|
|
||||||
}
|
|
||||||
if c.Verdict == "" {
|
|
||||||
c.Verdict = "error"
|
|
||||||
}
|
|
||||||
if result.Check == nil {
|
|
||||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
|
||||||
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
|
||||||
}
|
|
||||||
// **A group member's own check has no gate of its own** (novox/hq ADR 0403 clause 1): what ran was the repository's
|
|
||||||
// check alone, and its gate layer is "composed", never a pass. A failure or an error stays what it was.
|
|
||||||
if result.Checked != nil && result.Checked.Composed && result.Check != nil && c.Gate != nil &&
|
|
||||||
(c.Gate.Verdict == "pass" || c.Gate.Verdict == "warning") {
|
|
||||||
composedGate(&c)
|
|
||||||
}
|
|
||||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
|
|
||||||
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
|
|
||||||
}
|
|
||||||
if result.Checked != nil && c.Gate != nil && len(c.Gate.Dependents) == 0 {
|
|
||||||
c.Gate.Dependents = result.Checked.Dependents
|
|
||||||
}
|
|
||||||
if result.Checked != nil {
|
|
||||||
c.Plan = result.Checked.Plan
|
|
||||||
if c.Plan != nil && c.Trunk != "" {
|
|
||||||
// The kept delivery plan records the trunk commit with its other inputs (ADR 0323 §3).
|
|
||||||
plan := *c.Plan
|
|
||||||
plan.Trunk = c.Trunk
|
|
||||||
c.Plan = &plan
|
|
||||||
}
|
|
||||||
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
|
|
||||||
if g := result.Checked; g.Group != "" {
|
|
||||||
c.Group = g.Group
|
|
||||||
// Every member is named with its trunk commit (novox/hq issue 1106): the seat's word, else the one the forge
|
|
||||||
// announced with the head, so the forge's merge tool never meets a member whose verdict names none.
|
|
||||||
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
|
|
||||||
Commit: result.Ref, Trunk: firstTrunk(c.Trunk, g.Trunk)})
|
|
||||||
for _, m := range g.Members {
|
|
||||||
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
|
|
||||||
Commit: m.Ref, Trunk: firstTrunk(memberTrunk(result.Check, m), m.Trunk)})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, l := range []*link.CheckLayer{c.Gate, c.RepoCheck} {
|
|
||||||
if l != nil && l.Verdict == "" {
|
|
||||||
l.Verdict = "error"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **A check run again says so** (novox/hq issue 594). The summaries are the lines the pull request
|
|
||||||
// shows, clipped by the forge to a few words, so they get only a short marker after the verdict's own
|
|
||||||
// words; the report gets the whole sentence. Layers copied, so the build's own outcome is left as it came.
|
|
||||||
if mark := link.RunMarker(result.Delivery); mark != "" {
|
|
||||||
c.Summary += " " + mark
|
|
||||||
for _, l := range []**link.CheckLayer{&c.Gate, &c.RepoCheck} {
|
|
||||||
if *l != nil {
|
|
||||||
layer := **l
|
|
||||||
layer.Summary += " " + mark
|
|
||||||
*l = &layer
|
|
||||||
}
|
|
||||||
}
|
|
||||||
c.Report = link.Restarted(result.Delivery) + "\n\n" + c.Report
|
|
||||||
}
|
|
||||||
if len(c.Report) > maxCheckReport {
|
|
||||||
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
// firstTrunk is the first trunk commit named, as twelve characters; empty when none is.
|
|
||||||
func firstTrunk(named ...string) string {
|
|
||||||
for _, n := range named {
|
|
||||||
if n != "" {
|
|
||||||
return twelve(n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// memberTrunk is the trunk commit the build seat merged one of a group's heads into (novox/hq ADR 0323 §6), empty
|
|
||||||
// when it said none.
|
|
||||||
func memberTrunk(outcome *link.CheckOutcome, m link.GroupMember) string {
|
|
||||||
if outcome == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
for _, j := range outcome.Members {
|
|
||||||
if strings.EqualFold(j.Owner, m.Owner) && strings.EqualFold(j.Repo, m.Repo) && strings.HasPrefix(m.Ref, j.Commit) {
|
|
||||||
return j.Trunk
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func prefixedAll(prefix string, items []string) []string {
|
|
||||||
out := make([]string, 0, len(items))
|
|
||||||
for _, i := range items {
|
|
||||||
out = append(out, prefix+i)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
|
|
||||||
func sayChecked(ctx context.Context, w io.Writer, c link.Checked) {
|
|
||||||
// **No merge-gate layer from R4e, for any repository** (ADR 0503 §3, §5; issue 1299): the forge's holder writes
|
|
||||||
// `mesh/repo-check` from the repo-check layer alone, and the checker owes `mesh/check`.
|
|
||||||
c.Gate = nil
|
|
||||||
repo := "none"
|
|
||||||
if c.RepoCheck != nil {
|
|
||||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
|
||||||
}
|
|
||||||
judged := ""
|
|
||||||
if c.Trunk != "" {
|
|
||||||
judged = " merged into trunk " + c.Trunk
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, "%s: %s/%s#%d at %.8s%s checked on %s: repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
|
||||||
c.Commit, judged, orSomewhere(c.On), repo)
|
|
||||||
if checkEvents == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
body, err := json.Marshal(c)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
||||||
defer stop()
|
|
||||||
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
|
|
||||||
fmt.Fprintf(w, "%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// pullsBucketsOf are the buckets of the pull requests' keys a check names for the build agent: one per module
|
|
||||||
// claiming the delivery seat, in the order of their names (novox/hq ADR 0437 §3, issue 1212).
|
|
||||||
func pullsBucketsOf(entries []inventory.Entry) []string {
|
|
||||||
var out []string
|
|
||||||
for _, m := range claimantsOf(entries, catalogue.DeliverySeat) {
|
|
||||||
out = append(out, broker.BucketName(m, "pulls"))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,56 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) {
|
|
||||||
js, err := broker.Dial(testbus.URL(t))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(js.Close)
|
|
||||||
if err := broker.AssertMeshStreams(js); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
legacy, err := js.Conn().JetStream()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
was := checkEvents
|
|
||||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
|
||||||
t.Cleanup(func() { checkEvents = was })
|
|
||||||
heard := make(chan *nats.Msg, 4)
|
|
||||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer func() { _ = sub.Unsubscribe() }()
|
|
||||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3}
|
|
||||||
checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey})
|
|
||||||
checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
|
||||||
select {
|
|
||||||
case m := <-heard:
|
|
||||||
var c link.Checked
|
|
||||||
_ = json.Unmarshal(m.Data, &c)
|
|
||||||
if c.ID != "broken" || c.Verdict != "error" {
|
|
||||||
t.Fatalf("the first verdict said is %+v; want only the broken check's error", c)
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("nothing was said")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case m := <-heard:
|
|
||||||
t.Fatalf("a second verdict was said: %s", m.Data)
|
|
||||||
case <-time.After(300 * time.Millisecond):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,109 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// C1 (ADR 0503 §3, §5; issue 1299, lead decision L11), with the store and a real build queue: a pull request of a
|
|
||||||
// repository whose trunk requires mesh/check asks build-agent nothing and says nothing; one whose trunk requires
|
|
||||||
// mesh/repo-check alone (hq) has its own merge-check.sh asked, and every verdict said names no gate layer; a protection
|
|
||||||
// that cannot be read asks and says nothing.
|
|
||||||
func TestThePullRequestChecksAreAskedOnlyForARepositoryTheCheckerDoesNotJudge(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
js, _ := aBuildQueue(t)
|
|
||||||
servingBus.Store(js)
|
|
||||||
t.Cleanup(func() { servingBus.Store(nil) })
|
|
||||||
said := &saidOnTheBus{}
|
|
||||||
checkEvents = said
|
|
||||||
t.Cleanup(func() { checkEvents = nil })
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
|
||||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// hq is cloned as a module built from it is, from its URL: no forge is held in this test.
|
|
||||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "hq-docs", Version: "1"},
|
|
||||||
inventory.Source{Repository: "http://forge.invalid/novox/hq.git", Ref: "main", Path: "docs", BuiltFrom: "c0",
|
|
||||||
Head: "c0"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
required := map[string][]string{"novox/mesh-controller": {"mesh/check"}, "novox/hq": {"mesh/repo-check"}}
|
|
||||||
was := requiredStatusesOf
|
|
||||||
requiredStatusesOf = func(_ context.Context, owner, repo, _ string) ([]string, error) {
|
|
||||||
if r, ok := required[owner+"/"+repo]; ok {
|
|
||||||
return r, nil
|
|
||||||
}
|
|
||||||
return nil, errors.New("the forge did not answer")
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { requiredStatusesOf = was })
|
|
||||||
queued := func() int {
|
|
||||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return len(q.Asks)
|
|
||||||
}
|
|
||||||
pull := func(repo string, number int, mergeCheck bool) link.PullUpdated {
|
|
||||||
p := link.PullUpdated{Owner: "novox", Repo: repo, Number: number, Base: "main", Commit: "abcdefabcdefabcdefabcdefabcdefabcdefabcd",
|
|
||||||
BaseSHA: "1111111111111111111111111111111111111111", Paths: []string{"main.go"}, ModuleDirsSaid: true}
|
|
||||||
p.MergeCheck, p.MergeCheckSaid = mergeCheck, true
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
f := following{open: open, oneShot: true}
|
|
||||||
|
|
||||||
if err := f.PullUpdated(ctx, pull("mesh-controller", 7, true)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if n := queued(); n != 0 || len(said.events) != 0 {
|
|
||||||
t.Fatalf("0503 §3: a repository the checker judges asked %d check(s) and said %v", n, said.events)
|
|
||||||
}
|
|
||||||
if err := f.PullUpdated(ctx, pull("unread", 3, true)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if n := queued(); n != 0 || len(said.events) != 0 {
|
|
||||||
t.Fatalf("L11: a protection that cannot be read asked %d check(s) and said %v", n, said.events)
|
|
||||||
}
|
|
||||||
if err := f.PullUpdated(ctx, pull("hq", 9, true)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(q.Asks) != 1 || q.Asks[0].Request.Check == nil || len(q.Asks[0].Request.Check.Modules) != 0 ||
|
|
||||||
q.Asks[0].Request.Check.Repo != "hq" {
|
|
||||||
t.Fatalf("0503 §5: hq's repository check is not the one ask: %+v", q.Asks)
|
|
||||||
}
|
|
||||||
// A head with no merge-check.sh is said at once, with the repo-check layer alone.
|
|
||||||
if err := f.PullUpdated(ctx, pull("hq", 10, false)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(said.events) != 1 {
|
|
||||||
t.Fatalf("hq's head with no merge-check.sh was said %d time(s)", len(said.events))
|
|
||||||
}
|
|
||||||
var c link.Checked
|
|
||||||
if err := json.Unmarshal(said.bodies[0], &c); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if c.Gate != nil || c.RepoCheck == nil || c.RepoCheck.Verdict != "warning" {
|
|
||||||
t.Fatalf("0503 §5: the verdict said names a gate layer, or no repo-check layer: %+v", c)
|
|
||||||
}
|
|
||||||
// A trunk requiring both (hq, ADR 0513 §4): the checker judges mesh/check, and the repository check is still asked.
|
|
||||||
required["novox/hq"] = []string{"mesh/check", "mesh/repo-check"}
|
|
||||||
if err := f.PullUpdated(ctx, pull("hq", 11, true)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if q, err := link.ReadQueue(ctx, js, link.TheBuildMachine); err != nil || len(q.Asks) != 2 ||
|
|
||||||
q.Asks[1].Request.Check == nil || q.Asks[1].Request.Check.Repo != "hq" {
|
|
||||||
t.Fatalf("0513 §4: a trunk requiring both had no repository check asked: %+v %v", q.Asks, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -76,8 +75,8 @@ func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) {
|
|||||||
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
|
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if s.Mesh != c.mesh || s.Judge != c.judge {
|
if s.Mesh != c.mesh {
|
||||||
t.Errorf("%s: the mesh's %v judged by %q, wanted %v by %q", c.what, s.Mesh, s.Judge, c.mesh, c.judge)
|
t.Errorf("%s: the mesh's %v, wanted %v", c.what, s.Mesh, c.mesh)
|
||||||
}
|
}
|
||||||
if s.gated() != (c.modules != "" || c.new != "") {
|
if s.gated() != (c.modules != "" || c.new != "") {
|
||||||
t.Errorf("%s: gated %v", c.what, s.gated())
|
t.Errorf("%s: gated %v", c.what, s.gated())
|
||||||
@@ -94,243 +93,3 @@ func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
|
|||||||
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
|
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Each layer is said; a check that could not run is an error on both, never a pass; a build seat from
|
|
||||||
// before the layers is read as the gate.
|
|
||||||
func TestAChecksLayersAreEachSaidAndAnErrorIsNeverAPass(t *testing.T) {
|
|
||||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3, Modules: []string{"gitea"}}
|
|
||||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
|
|
||||||
if c.Verdict != "error" || c.Gate == nil || c.Gate.Verdict != "error" || c.RepoCheck == nil || c.RepoCheck.Verdict != "error" {
|
|
||||||
t.Fatalf("a check that could not run said %+v", c)
|
|
||||||
}
|
|
||||||
if strings.Join(c.Gate.Modules, ",") != "gitea" {
|
|
||||||
t.Errorf("the gate names %v", c.Gate.Modules)
|
|
||||||
}
|
|
||||||
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "warning", Summary: "wide"}})
|
|
||||||
if c.Gate == nil || c.Gate.Verdict != "warning" || c.RepoCheck != nil {
|
|
||||||
t.Fatalf("an outcome without layers said %+v", c)
|
|
||||||
}
|
|
||||||
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "pass",
|
|
||||||
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "its merge-check.sh failed"}}})
|
|
||||||
if c.RepoCheck.Verdict != "fail" || c.Gate.Verdict != "pass" {
|
|
||||||
t.Fatalf("the layers said %+v / %+v", c.Gate, c.RepoCheck)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
|
|
||||||
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
|
|
||||||
// module in both, and what stands on a touched module is named as its dependents in both.
|
|
||||||
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
|
|
||||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
|
||||||
nats := fromRepo("nats", catalogue, "modules/nats")
|
|
||||||
nats.Source.BuiltFrom = "old"
|
|
||||||
gitea := fromRepo("gitea", catalogue, "modules/gitea")
|
|
||||||
gitea.Source.BuiltFrom = "old"
|
|
||||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
|
||||||
entries := []inventory.Entry{nats, gitea, tools}
|
|
||||||
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
|
|
||||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
|
||||||
read := map[string][]inventory.ReadRepository{}
|
|
||||||
|
|
||||||
for _, c := range []struct {
|
|
||||||
what string
|
|
||||||
paths []string
|
|
||||||
moved, dependents string
|
|
||||||
width int
|
|
||||||
unread string
|
|
||||||
}{
|
|
||||||
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
|
|
||||||
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
|
|
||||||
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
|
|
||||||
} {
|
|
||||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
|
||||||
s := pullScope(p, entries, read, edges)
|
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
|
||||||
r := reachOfMerge(m, entries, read, edges)
|
|
||||||
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
|
|
||||||
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
|
|
||||||
}
|
|
||||||
if got := strings.Join(s.Dependents, ","); got != c.dependents {
|
|
||||||
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
|
|
||||||
}
|
|
||||||
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
|
|
||||||
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
|
|
||||||
}
|
|
||||||
if got := strings.Join(s.Unread, ","); got != c.unread {
|
|
||||||
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A repository whose build another module's recipe packages: that module is reached through the
|
|
||||||
// planner's `read`, and what stands on it after it.
|
|
||||||
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
|
|
||||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
|
|
||||||
Paths: []string{"src/index.ts"}}, entries, read, edges)
|
|
||||||
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
|
|
||||||
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a build of a commit off its repository's
|
|
||||||
// default branch — a pull request's head, a branch built by hand, a rebuild or replay of one — is recorded
|
|
||||||
// and never registered, so nothing can send it; a check or a dry run never is either.
|
|
||||||
func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
|
|
||||||
on := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main", OnTrunk: true}
|
|
||||||
if err := publishable(on, ""); err != nil {
|
|
||||||
t.Errorf("a build on the trunk was refused: %v", err)
|
|
||||||
}
|
|
||||||
off := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main"}
|
|
||||||
if err := publishable(off, ""); !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "main") {
|
|
||||||
t.Errorf("a build off the trunk was let through: %v", err)
|
|
||||||
}
|
|
||||||
for _, r := range []link.BuildResult{
|
|
||||||
{ID: "c", Trunk: "main", OnTrunk: true, Check: &link.CheckOutcome{Verdict: "pass"}},
|
|
||||||
{ID: "d", Trunk: "main", OnTrunk: true, Checked: &link.CheckRequest{}},
|
|
||||||
{ID: "e", Trunk: "main", OnTrunk: true, DryRun: true},
|
|
||||||
} {
|
|
||||||
if publishable(r, "") == nil {
|
|
||||||
t.Errorf("%s, a check or a dry run, was publishable", r.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A module that follows a branch other than the default: that branch is its trunk, and the default
|
|
||||||
// is not.
|
|
||||||
follows := link.BuildResult{ID: "g", Commit: "abc", Trunk: "master", Branches: []string{"nox-mesh"}}
|
|
||||||
if err := publishable(follows, "nox-mesh"); err != nil {
|
|
||||||
t.Errorf("a commit on the branch a module follows was refused: %v", err)
|
|
||||||
}
|
|
||||||
if err := publishable(link.BuildResult{ID: "h", Commit: "abc", Trunk: "master", OnTrunk: true,
|
|
||||||
Branches: []string{"master"}}, "nox-mesh"); err == nil {
|
|
||||||
t.Error("a commit on the default but not on the branch the module follows was published")
|
|
||||||
}
|
|
||||||
if err := publishable(link.BuildResult{ID: "i", Commit: "abc", Trunk: "main", Branches: []string{"feat/x"}}, ""); err == nil {
|
|
||||||
t.Error("a feature branch's commit was published")
|
|
||||||
}
|
|
||||||
// A build seat older than the rule says nothing: let through and said, so the rule can reach the mesh.
|
|
||||||
if err := publishable(link.BuildResult{ID: "f", Commit: "abc"}, ""); err != nil {
|
|
||||||
t.Errorf("a build seat that said nothing was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A verdict names the trunk commit it judged** (novox/hq ADR 0323 §3, issue 341): the controller's `checked`
|
|
||||||
// carries the head and the trunk commit the build seat merged it into, the commits of the repositories read beside
|
|
||||||
// it, how long it took, and the kept delivery plan records the trunk commit with its other inputs. A group's
|
|
||||||
// composed check names, per member, the trunk commit that head was merged into (§6).
|
|
||||||
func TestAVerdictNamesTheTrunkCommitItJudged(t *testing.T) {
|
|
||||||
plan := &link.ChangePlan{Repository: "novox/mesh-host", Base: "main", Head: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
||||||
Summary: "builds mesh-host"}
|
|
||||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 66, Base: "main", Plan: plan}
|
|
||||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Checked: asked,
|
|
||||||
Check: &link.CheckOutcome{Verdict: "pass", Took: "4m12s", Head: "dfb1415aaaaa", Trunk: "d28ca95bbbbb",
|
|
||||||
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "pass"},
|
|
||||||
Beside: map[string]string{"mesh-catalog": "0123456789ab", "mesh-lab": "ba9876543210"}}})
|
|
||||||
if c.Trunk != "d28ca95bbbbb" || c.Commit != "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
|
||||||
t.Fatalf("the verdict names head %s and trunk %q", c.Commit, c.Trunk)
|
|
||||||
}
|
|
||||||
if c.Beside["mesh-catalog"] != "0123456789ab" || c.Beside["mesh-lab"] != "ba9876543210" {
|
|
||||||
t.Fatalf("the verdict names the repositories read beside it as %v", c.Beside)
|
|
||||||
}
|
|
||||||
if c.Took != "4m12s" {
|
|
||||||
t.Fatalf("the verdict says it took %q", c.Took)
|
|
||||||
}
|
|
||||||
if c.Plan == nil || c.Plan.Trunk != "d28ca95bbbbb" || c.Plan.Head != plan.Head {
|
|
||||||
t.Fatalf("the kept plan records %+v", c.Plan)
|
|
||||||
}
|
|
||||||
if plan.Trunk != "" {
|
|
||||||
t.Fatal("the plan the controller asked with was changed: the verdict's is a copy")
|
|
||||||
}
|
|
||||||
|
|
||||||
group := &link.CheckRequest{Owner: "novox", Repo: "mesh-controller", Number: 7, Base: "main", Group: "feat/x",
|
|
||||||
Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-host", Number: 8, Ref: "cccccccccccc0000", Base: "main"}}}
|
|
||||||
c = checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{
|
|
||||||
Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"},
|
|
||||||
Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "cccccccccccc", Trunk: "222222222222"}}}})
|
|
||||||
if len(c.Members) != 2 || c.Members[0].Trunk != "111111111111" || c.Members[1].Trunk != "222222222222" ||
|
|
||||||
c.Members[1].Commit != "cccccccccccc0000" {
|
|
||||||
t.Fatalf("the composed check names its members %+v", c.Members)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A group's composed verdict names every member's trunk commit** (novox/hq issue 1106): a member the build seat named no
|
|
||||||
// trunk for (an SDK member whose base the seat did not merge, or a seat from before the rule) is named by the trunk the forge
|
|
||||||
// announced with its head, and the primary the same, so the merge tool never reads "names no trunk commit" on a member
|
|
||||||
// whose trunk did not move.
|
|
||||||
func TestAGroupVerdictNamesEveryMembersTrunkTheSDKMemberToo(t *testing.T) {
|
|
||||||
group := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 7, Base: "main", Group: "feat/x",
|
|
||||||
Trunk: "dddddddddddd0000",
|
|
||||||
Members: []link.GroupMember{
|
|
||||||
{Owner: "novox", Repo: "mesh-sdk", Number: 8, Ref: "cccccccccccc0000", Base: "main", Trunk: "eeeeeeeeeeee0000"},
|
|
||||||
{Owner: "novox", Repo: "mesh-host", Number: 9, Ref: "bbbbbbbbbbbb0000", Base: "main", Trunk: "ffffffffffff0000"}}}
|
|
||||||
c := checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{
|
|
||||||
Verdict: "pass", Head: "aaaaaaaaaaaa", Gate: &link.CheckLayer{Verdict: "pass"},
|
|
||||||
Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "bbbbbbbbbbbb", Trunk: "222222222222"}}}})
|
|
||||||
want := []string{"dddddddddddd", "eeeeeeeeeeee", "222222222222"}
|
|
||||||
if len(c.Members) != 3 {
|
|
||||||
t.Fatalf("the composed check names its members %+v", c.Members)
|
|
||||||
}
|
|
||||||
for i, m := range c.Members {
|
|
||||||
if m.Trunk != want[i] {
|
|
||||||
t.Fatalf("member %d (%s/%s) is named with trunk %q, want %q (the seat's word first, else the announced one): %+v",
|
|
||||||
i, m.Owner, m.Repo, m.Trunk, want[i], c.Members)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The merge made for a check is never registered** (novox/hq ADR 0323 §2): a build of it — a commit on no branch
|
|
||||||
// of the forge — is refused, naming the trunk. The rule of ADR 0238 §1 stands unchanged; this holds it for the
|
|
||||||
// merge commit a check makes.
|
|
||||||
func TestABuildOfACheckMergeIsRefusedNamingTheTrunk(t *testing.T) {
|
|
||||||
merge := link.BuildResult{ID: "b", Commit: "4b5c6d7e8f90aabbccddeeff0011223344556677", Trunk: "main"}
|
|
||||||
err := publishable(merge, "")
|
|
||||||
if !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "not on main") {
|
|
||||||
t.Fatalf("a build of a check's merge was answered %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A verdict no build judges names its trunk commit too** (novox/hq ADR 0323 §3): a change touching no module of a
|
|
||||||
// repository with no merge-check.sh (novox/mesh-tools-go) is answered by the controller itself. Naming no trunk, the
|
|
||||||
// merge tool refused it, asked again, got the same verdict, and refused for ever.
|
|
||||||
func TestADirectVerdictNamesTheTrunkTheForgeAnnounced(t *testing.T) {
|
|
||||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-tools-go", Number: 3, Base: "main", Commit: "1a2b3c4d5e6f7a8b9c0d",
|
|
||||||
BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
|
|
||||||
c := directVerdict(p, link.ChangePlan{Repository: "novox/mesh-tools-go", Base: "main", Head: p.Commit})
|
|
||||||
if c.Trunk != "7a8b9c0d1e2f" || c.Plan == nil || c.Plan.Trunk != "7a8b9c0d1e2f" {
|
|
||||||
t.Fatalf("the direct verdict names trunk %q, its plan %+v", c.Trunk, c.Plan)
|
|
||||||
}
|
|
||||||
if c.Verdict != "pass" || c.Commit != p.Commit {
|
|
||||||
t.Fatalf("the direct verdict is %+v", c)
|
|
||||||
}
|
|
||||||
if c := directVerdict(link.PullUpdated{Owner: "novox", Repo: "x", Commit: "abc"}, link.ChangePlan{}); c.Trunk != "" {
|
|
||||||
t.Fatalf("a forge that said no base head gave the trunk %q", c.Trunk)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A head checked again through mesh-delivery (`delivery check --members`, a recheck) reaches the same direct verdict:
|
|
||||||
// it names the base head the forge last announced, kept on the member, or the merge tool would refuse it for ever.
|
|
||||||
func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
|
|
||||||
m := orderMember{ID: "novox/mesh-tools-go@1a2b3c4d5e6f", Repository: "novox/mesh-tools-go", Base: "main",
|
|
||||||
Head: "1a2b3c4d5e6f7a8b9c0d", Number: 3, BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
|
|
||||||
if c := directVerdict(m.pull(), link.ChangePlan{}); c.Trunk != "7a8b9c0d1e2f" {
|
|
||||||
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
|
|
||||||
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
|
|
||||||
|
|
||||||
// C2 (ADR 0513 §4, ADR 0503 §5; issue 1299): a trunk requiring mesh/repo-check has its repository check asked, whether or
|
|
||||||
// not it also requires mesh/check (hq requires both); a trunk requiring mesh/check alone, or neither, is asked nothing.
|
|
||||||
func TestARepositoryCheckIsAskedWheneverTheTrunkRequiresIt(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
required []string
|
|
||||||
asks bool
|
|
||||||
}{
|
|
||||||
{[]string{"mesh/check", "mesh/repo-check"}, true},
|
|
||||||
{[]string{"mesh/repo-check"}, true},
|
|
||||||
{[]string{"mesh/check"}, false},
|
|
||||||
{nil, false},
|
|
||||||
} {
|
|
||||||
if got := asksRepoCheck(c.required); got != c.asks {
|
|
||||||
t.Fatalf("a trunk requiring %v asks its repository check: %v, want %v", c.required, got, c.asks)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/artifacts"
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
@@ -215,27 +217,34 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
r := sweep(ctx, inv, artifacts.Store{Address: address}, references, kept, afterBuild)
|
r := sweep(ctx, inv, artifacts.Store{Address: address}, references, kept, afterBuild)
|
||||||
|
sayCollected(os.Stderr, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayCollected says what one sweep after a build did.
|
||||||
|
func sayCollected(w io.Writer, r sweepResult) {
|
||||||
if r.HoldersWritten > 0 {
|
if r.HoldersWritten > 0 {
|
||||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", r.HoldersWritten)
|
fmt.Fprintf(w, "the artifact store now holds %d more kept archive(s) by a manifest\n", r.HoldersWritten)
|
||||||
}
|
}
|
||||||
if r.Missing > 0 {
|
if r.Missing > 0 {
|
||||||
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
fmt.Fprintf(w, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||||
"`collection` lists them\n", r.Missing)
|
"`collection` lists them\n", r.Missing)
|
||||||
}
|
}
|
||||||
if len(r.LetGo) > 0 {
|
if len(r.LetGo) > 0 {
|
||||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n", len(r.LetGo))
|
// **Each one named** (novox/hq issue 1933): a count could not say which archive a machine's 404 was let go of.
|
||||||
|
fmt.Fprintf(w, "the artifact store let go of %d artifact(s) the mesh no longer keeps: %s\n", len(r.LetGo),
|
||||||
|
strings.Join(r.LetGo, ", "))
|
||||||
}
|
}
|
||||||
if r.Stopped != "" && !r.Bounded {
|
if r.Stopped != "" && !r.Bounded {
|
||||||
fmt.Fprintln(os.Stderr, r.Stopped)
|
fmt.Fprintln(w, r.Stopped)
|
||||||
}
|
}
|
||||||
if r.Held && r.Left > 0 {
|
if r.Held && r.Left > 0 {
|
||||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", r.Left)
|
fmt.Fprintf(w, "%d more to collect; the next build asks again\n", r.Left)
|
||||||
}
|
}
|
||||||
if r.Skipped > 0 {
|
if r.Skipped > 0 {
|
||||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", r.Skipped)
|
fmt.Fprintf(w, "%d artifact(s) the sweep will not address were skipped\n", r.Skipped)
|
||||||
}
|
}
|
||||||
if r.Indexes > 0 {
|
if r.Indexes > 0 {
|
||||||
fmt.Fprintf(os.Stderr, "%d eligible index(es) wait for a collect a person confirms, which takes their platforms too\n", r.Indexes)
|
fmt.Fprintf(w, "%d eligible index(es) wait for a collect a person confirms, which takes their platforms too\n", r.Indexes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -245,41 +245,8 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
|
|||||||
if out.Sequence, err = inv.Sequence(ctx, n.ID); err != nil {
|
if out.Sequence, err = inv.Sequence(ctx, n.ID); err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
if out.Epoch, err = inv.SentEpoch(ctx, n.ID); err != nil {
|
// Every node-engine of release 6 reads a module per resource and a build per module.
|
||||||
return out, err
|
out.ReadsModules = true
|
||||||
}
|
|
||||||
if out.Generation, err = inv.SentGeneration(ctx, n.ID); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.ReadsModules, err = inv.ReadsModules(ctx, n.ID); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
runs, err := inv.AssignmentRuns(ctx, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
if len(runs) > 0 {
|
|
||||||
out.Runs = make(map[string]compose.Run, len(runs))
|
|
||||||
for module, ref := range runs {
|
|
||||||
out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A recorded module kept at the build this machine runs, in the context a send composes under (recorded_kept.go): its
|
|
||||||
// manifest and the commit, applied to this node's own resolution only.
|
|
||||||
// What cannot be kept refuses this machine alone (keepRecorded's refusal, as planFor returned it).
|
|
||||||
kept, err := recordedKept(ctx, open, n.Name)
|
|
||||||
if err != nil {
|
|
||||||
*refused = err
|
|
||||||
} else if len(kept) > 0 {
|
|
||||||
out.KeptBuilds = kept
|
|
||||||
if out.Kept, err = keptManifests(ctx, open, n.Name, kept); err != nil {
|
|
||||||
*refused = err
|
|
||||||
out.KeptBuilds, out.Kept = nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stated, has, err := inv.HealthOf(ctx, n.Name)
|
stated, has, err := inv.HealthOf(ctx, n.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
@@ -291,18 +258,6 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// keptManifests is the manifest of each module kept at the build a machine runs: keepRecorded's reads, and its refusals in its
|
|
||||||
// words, without putting any of it into a shelf (that is the node's own Kept, applied by the composition).
|
|
||||||
func keptManifests(ctx context.Context, open *stores, node string, kept map[string]string) (map[string]catalogue.Manifest, error) {
|
|
||||||
// applyKept puts each manifest into the shelf it is given and refuses a build that cannot be read; the shelf here is a
|
|
||||||
// scratch one, whose entries are the kept manifests.
|
|
||||||
scratch := map[string]catalogue.Manifest{}
|
|
||||||
if err := applyKept(ctx, open, node, kept, scratch); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return scratch, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func readTunnels(ctx context.Context, inv *inventory.Inventory) (map[string]compose.FoundTunnel, error) {
|
func readTunnels(ctx context.Context, inv *inventory.Inventory) (map[string]compose.FoundTunnel, error) {
|
||||||
found, err := inv.Tunnels(ctx)
|
found, err := inv.Tunnels(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -6,11 +6,9 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// 1382: the adapter's Input names, for every node, what the inventory answers for it, each compared with the read named
|
// 1382: the adapter's Input names, for every node, what the inventory answers for it, each compared with the read named
|
||||||
@@ -132,69 +130,6 @@ func nodePorts(in compose.Input, name string) map[string]map[int]int {
|
|||||||
return n.Ports
|
return n.Ports
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1382/AC2: a node keeping a recorded module at the build it runs has that module in Kept and KeptBuilds, and the Input's
|
|
||||||
// Catalogue is the current shelf, not the kept manifest (R4a review round 1, finding 1).
|
|
||||||
func TestANodeKeepingARecordedBuildHasItInKeptAndTheCatalogueStaysCurrent(t *testing.T) {
|
|
||||||
open := aMesh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
inv := open.inventory
|
|
||||||
start := time.Now().Add(-time.Hour)
|
|
||||||
image := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64)
|
|
||||||
for _, b := range []link.BuildResult{
|
|
||||||
aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, map[string][2]string{"server": {image, ""}}),
|
|
||||||
} {
|
|
||||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := inv.Assign(ctx, "anchor", "postgres"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"postgres": "c1111111"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute),
|
|
||||||
map[string][2]string{"server": {image, "runtime"}})); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A send that is not a person's push keeps the build the machine runs.
|
|
||||||
in, err := readComposeInput(keepingRecorded(ctx), open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
anchor, _ := nodeOf(in.Input, "anchor")
|
|
||||||
if anchor.KeptBuilds["postgres"] != "c1111111" {
|
|
||||||
t.Fatalf("anchor keeps %v, want postgres at c1111111", anchor.KeptBuilds)
|
|
||||||
}
|
|
||||||
kept, found := anchor.Kept["postgres"]
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("the kept manifest of postgres is not in anchor's Kept: %v", anchor.Kept)
|
|
||||||
}
|
|
||||||
current, err := inv.Catalogue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(in.Catalogue["postgres"], current["postgres"]) {
|
|
||||||
t.Error("the Input's Catalogue is not the current shelf")
|
|
||||||
}
|
|
||||||
if reflect.DeepEqual(kept, current["postgres"]) {
|
|
||||||
t.Error("the kept manifest is the current one: nothing is kept")
|
|
||||||
}
|
|
||||||
if laptop, _ := nodeOf(in.Input, "laptop"); len(laptop.Kept) != 0 || len(laptop.KeptBuilds) != 0 {
|
|
||||||
t.Errorf("laptop keeps what only anchor runs: %v", laptop.KeptBuilds)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A person's push keeps nothing.
|
|
||||||
pushed, err := readComposeInput(ctx, open)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if a, _ := nodeOf(pushed.Input, "anchor"); len(a.Kept) != 0 || len(a.KeptBuilds) != 0 {
|
|
||||||
t.Errorf("a person's push keeps %v", a.KeptBuilds)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1382/AC3: the seats on record, the tunnels, the adopted tunnel, its carried peers and the nodes joining equal the
|
// 1382/AC3: the seats on record, the tunnels, the adopted tunnel, its carried peers and the nodes joining equal the
|
||||||
// inventory's answers; the range and the suffix are read from the environment once, here.
|
// inventory's answers; the range and the suffix are read from the environment once, here.
|
||||||
func TestTheAdaptersInputCarriesTheNetworkAsTheInventoryHoldsIt(t *testing.T) {
|
func TestTheAdaptersInputCarriesTheNetworkAsTheInventoryHoldsIt(t *testing.T) {
|
||||||
|
|||||||
@@ -39,24 +39,10 @@ func withInputCache(ctx context.Context) context.Context {
|
|||||||
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
|
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
|
||||||
}
|
}
|
||||||
|
|
||||||
// freshInput is ctx for one composition whose Input is read afresh and kept only until the composition makes something: a
|
|
||||||
// send that is composed after another one made and recorded things (a push's machines, one after the other).
|
|
||||||
func freshInput(ctx context.Context) context.Context {
|
|
||||||
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs
|
// keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs
|
||||||
// (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send.
|
// (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send.
|
||||||
func keptSignature(ctx context.Context) string {
|
func keptSignature(ctx context.Context) string {
|
||||||
skip, on := keptExcept(ctx)
|
return ""
|
||||||
var names []string
|
|
||||||
for m := range skip {
|
|
||||||
names = append(names, m)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
every, _ := ctx.Value(keepEveryBuildKey{}).(string)
|
|
||||||
settings, _ := ctx.Value(keepForSettingsKey{}).(string)
|
|
||||||
return fmt.Sprintf("kept=%v;except=%v;every=%s;settings=%s", on, names, every, settings)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// inputFor is the Input for ctx's operation.
|
// inputFor is the Input for ctx's operation.
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -86,26 +85,3 @@ func TestADeclarationThatStandsInForAValueIsStillForeseen(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1386: an Input read for a person's push is never the one read for a send that keeps recorded builds, and the cache is
|
|
||||||
// emptied by what a send made.
|
|
||||||
func TestAnInputIsCachedPerWhatItKeepsAndForgottenAfterASend(t *testing.T) {
|
|
||||||
ctx := withInputCache(context.Background())
|
|
||||||
plain := keptSignature(ctx)
|
|
||||||
kept := keptSignature(keepingRecorded(ctx))
|
|
||||||
bus := keptSignature(keepingRecorded(ctx, "nats"))
|
|
||||||
every := keptSignature(keepingEveryBuild(ctx, "anchor"))
|
|
||||||
for name, sig := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} {
|
|
||||||
for other, o := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} {
|
|
||||||
if name != other && sig == o {
|
|
||||||
t.Errorf("%s and %s read the same Input", name, other)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cache := ctx.Value(inputCacheKey{}).(*inputCache)
|
|
||||||
cache.by[plain] = &meshInput{}
|
|
||||||
forgetInput(ctx)
|
|
||||||
if len(cache.by) != 0 {
|
|
||||||
t.Fatal("what a send made left the Input it read in the cache")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A mesh that composes (novox/hq ADR 0539): the real store (theCatalogue: sudo from the catalogue, mesh-host from the
|
||||||
|
// node-engine's repository, the private network on anchor and laptop), the controller's signing key in its own
|
||||||
|
// identity store, and a real bus server this process composes onto, its lease taken by the act as a command's is. Every
|
||||||
|
// subject published on the bus is heard, so a test can say nothing went to a node's declaration subject.
|
||||||
|
type composingMesh struct {
|
||||||
|
open *stores
|
||||||
|
js jetstream.JetStream
|
||||||
|
conn *nats.Conn
|
||||||
|
mu sync.Mutex
|
||||||
|
heard []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func aComposingMesh(t *testing.T) *composingMesh {
|
||||||
|
t.Helper()
|
||||||
|
open := theCatalogue(t)
|
||||||
|
ident := identity.ForTest(t)
|
||||||
|
if _, err := ident.Establish(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s := testbus.Start(t)
|
||||||
|
t.Setenv(broker.NATSVar, s.ClientURL())
|
||||||
|
was := theLease
|
||||||
|
theLease = &actor{}
|
||||||
|
t.Cleanup(func() { theLease.release(); theLease = was })
|
||||||
|
conn, err := nats.Connect(s.ClientURL())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(conn.Close)
|
||||||
|
js, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := &composingMesh{open: open, js: js, conn: conn}
|
||||||
|
sub, err := conn.Subscribe(">", func(msg *nats.Msg) {
|
||||||
|
m.mu.Lock()
|
||||||
|
m.heard = append(m.heard, msg.Subject)
|
||||||
|
m.mu.Unlock()
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
legacy, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The controller's seat events, kept as the mesh's events stream keeps them.
|
||||||
|
if _, err := legacy.AddStream(&nats.StreamConfig{Name: "CONTROLLER_EVENTS", Subjects: []string{"mesh.seat.mesh-controller.event.>"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wasEvents := checkEvents
|
||||||
|
checkEvents = link.OverNATS{Conn: conn, JS: legacy}
|
||||||
|
t.Cleanup(func() { checkEvents = wasEvents })
|
||||||
|
withTrunkList(t, true)
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// trunkKeys are a done key and the indexer's trunk value naming it.
|
||||||
|
type trunkKeys struct {
|
||||||
|
keyBuckets
|
||||||
|
trunk sdkdelivery.Module
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k trunkKeys) ModuleAt(context.Context, string, string) (sdkdelivery.Module, error) {
|
||||||
|
return k.trunk, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC14: the declarations' stream stays while one node's have names another node-engine than the one registered,
|
||||||
|
// and goes, said once, when every node's names it (ADR 0539 clause 11).
|
||||||
|
func TestTheDeclarationsStreamGoesOnceEveryNodeRunsTheNewEngine(t *testing.T) {
|
||||||
|
m := aComposingMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
legacy, err := m.conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := legacy.AddStream(&nats.StreamConfig{Name: broker.DeclarationsStream, Subjects: []string{"mesh.node.*.declare"},
|
||||||
|
MaxMsgsPerSubject: 1}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The new node-engine registered at its fingerprint, from its done key.
|
||||||
|
n6 := fpOf('6')
|
||||||
|
keys, trunk := aKeyFor("mesh-host", "6", n6, "novox/mesh-host", "", "f00d0123456789ab")
|
||||||
|
if _, err := liveDoneKeySteps(m.open, trunkKeys{keyBuckets: keys, trunk: trunk}).take(ctx, "mesh-host"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := m.js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: sdkdelivery.HaveBucket, History: sdkdelivery.HaveHistory}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
haves, err := m.js.KeyValue(ctx, sdkdelivery.HaveBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
have := func(node, fp string) {
|
||||||
|
key, _ := sdkdelivery.HaveKey(node, "mesh-host")
|
||||||
|
raw, _ := sdkdelivery.EncodeHave(sdkdelivery.Have{FP: fp, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK})
|
||||||
|
if _, err := haves.Put(ctx, key, raw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
have("anchor", n6)
|
||||||
|
have("laptop", fpOf('5'))
|
||||||
|
if done, err := retireDeclarations(ctx, m.open); err != nil || done {
|
||||||
|
t.Fatalf("with laptop on the old node-engine: done %v (%v)", done, err)
|
||||||
|
}
|
||||||
|
if _, err := m.js.Stream(ctx, broker.DeclarationsStream); err != nil {
|
||||||
|
t.Fatalf("the stream went while laptop runs the old node-engine: %v", err)
|
||||||
|
}
|
||||||
|
have("laptop", n6)
|
||||||
|
if done, err := retireDeclarations(ctx, m.open); err != nil || !done {
|
||||||
|
t.Fatalf("with every node on the new node-engine: done %v (%v)", done, err)
|
||||||
|
}
|
||||||
|
if _, err := m.js.Stream(ctx, broker.DeclarationsStream); !errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
t.Errorf("the stream is still there: %v", err)
|
||||||
|
}
|
||||||
|
if done, err := retireDeclarations(ctx, m.open); err != nil || !done {
|
||||||
|
t.Errorf("a second look: done %v (%v)", done, err)
|
||||||
|
}
|
||||||
|
if !declarationsRetiredNow(ctx, m.open.inventory) {
|
||||||
|
t.Errorf("a raise after the retirement would assert the stream again")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,343 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The judgement of a move (novox/hq ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests): beside each composition's digest
|
||||||
|
// the controller writes whether moving the node from the composition it runs, as its have names it, to this one
|
||||||
|
// recreates a container, moves the module's data, or neither, or that it cannot be judged. The assigner holds a move of
|
||||||
|
// a module of policy record from it until a person allows it.
|
||||||
|
//
|
||||||
|
// ADR 0377 §3 over two compositions of one module: **recreates** when a container resource is changed, added or
|
||||||
|
// removed; **moves data** when, with no container changed, a directory or file resource is changed, added or removed
|
||||||
|
// that a container of the module mounts or that holds or lies within one of its directories (where a module keeps its
|
||||||
|
// data), when its `data` declaration changes, or, for a module that declares data, when one of its long-running
|
||||||
|
// processes changes; **neither** otherwise, and for a module the node has never run. **Cannot judge** when the
|
||||||
|
// composition the node runs is gone or cannot be read. A held composition never becomes what is judged against: with A
|
||||||
|
// running, B held and C composed, C is judged against A, the one the have names.
|
||||||
|
|
||||||
|
// judgeMove is the value written beside a composition whose digest is digest: its judgement against what the node's have
|
||||||
|
// names for the module, and the module's policy.
|
||||||
|
func judgeMove(ctx context.Context, at composedPlace, key string, to sdkdelivery.Composed, digest string) (sdkdelivery.ComposedDigest, error) {
|
||||||
|
out := sdkdelivery.ComposedDigest{Digest: digest}
|
||||||
|
if at.policy != nil && to.Module != "" {
|
||||||
|
out.Policy, out.PolicyFrom, out.PolicyWhy = at.policy(to.Module)
|
||||||
|
}
|
||||||
|
if to.Module == "" {
|
||||||
|
// The node's own composition: no module's containers or data are judged; its want is written as any.
|
||||||
|
out.Judgement = sdkdelivery.JudgedNeither
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
have, found, err := at.ran(ctx, to.Module)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
// A node whose have names no composition yet (the switch: it runs the declaration it last applied) is judged
|
||||||
|
// against that module's slice of that declaration (novox/hq ADR 0540 clause 9): equal is unchanged, a want that
|
||||||
|
// changes nothing on the node; otherwise by ADR 0377 §3's tests, as any move.
|
||||||
|
if found && have.Composed == "" && have.FP != "" && have.Phase != sdkdelivery.PhaseRemoved && at.lastApplied != nil {
|
||||||
|
slice, declared, applied, err := at.lastApplied(ctx, to.Module)
|
||||||
|
if err != nil {
|
||||||
|
// Unread is not judged: a person's yes holds it, and the words say why.
|
||||||
|
out.Judgement = sdkdelivery.JudgedCannot
|
||||||
|
out.Words = fmt.Sprintf("%s's have on %s names no composition, and the declaration it last applied cannot be read: %v",
|
||||||
|
to.Module, at.node, err)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if applied {
|
||||||
|
from := sdkdelivery.Composed{Node: at.node, Module: to.Module, FP: have.FP, Data: to.Data, Resources: slice}
|
||||||
|
if sameResources(from, to) {
|
||||||
|
out.Judgement, out.Words = sdkdelivery.JudgedUnchanged, "equal to the declaration "+at.node+" last applied"
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
// Against names the declaration, by its digest: the composition the node runs is that declaration's slice.
|
||||||
|
out.Against = declared
|
||||||
|
if onlyTheUserList(from, to) {
|
||||||
|
// Only the bus's user list, as against a composition (ADR 0539 clause 7): no yes waits on it at the switch.
|
||||||
|
out.Judgement, out.UserListOnly = sdkdelivery.JudgedNeither, true
|
||||||
|
out.Words = "only the bus's user list changes, against the declaration " + at.node + " last applied"
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
j, words := judged(from, to)
|
||||||
|
out.Judgement, out.Words = j, words+", against the declaration "+at.node+" last applied"
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found || have.Phase == sdkdelivery.PhaseRemoved || (have.Composed == "" && have.FP == "") {
|
||||||
|
out.Judgement, out.Words = sdkdelivery.JudgedNeither, "the node runs no composition of "+to.Module
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if have.Composed == "" {
|
||||||
|
out.Judgement = sdkdelivery.JudgedCannot
|
||||||
|
out.Words = fmt.Sprintf("%s's have on %s names no composition, only the fingerprint %s", to.Module, at.node, have.FP)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
out.Against = have.Composed
|
||||||
|
if have.Composed == digest {
|
||||||
|
out.Judgement, out.Words = sdkdelivery.JudgedNeither, "the node runs this composition"
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
from, found, err := composedByDigest(ctx, at, to.Module, have.FP, have.Composed)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
out.Judgement = sdkdelivery.JudgedCannot
|
||||||
|
out.Words = fmt.Sprintf("the composition %s runs on %s (%s) is gone", to.Module, at.node, short(strings.TrimPrefix(have.Composed, "sha256:")))
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if onlyTheUserList(from, to) {
|
||||||
|
// Only the bus's user list (ADR 0539 clause 7): a file change and a reload of the bus, no move of it, written
|
||||||
|
// into its want at once.
|
||||||
|
out.Judgement, out.UserListOnly, out.Words = sdkdelivery.JudgedNeither, true, "only the bus's user list changes"
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
out.Judgement, out.Words = judged(from, to)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// busUsersSuffix ends the id of the bus module's user list as the composer writes it: `<module>.bus-users`.
|
||||||
|
const busUsersSuffix = ".bus-users"
|
||||||
|
|
||||||
|
// onlyTheUserList says two compositions of a module differ, and only in its file of the bus's user list.
|
||||||
|
func onlyTheUserList(from, to sdkdelivery.Composed) bool {
|
||||||
|
if from.Data != to.Data {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
before, after := resourcesByID(from), resourcesByID(to)
|
||||||
|
if len(before) != len(after) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
changed := false
|
||||||
|
for id, now := range after {
|
||||||
|
was, ok := before[id]
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if bytes.Equal(was.raw, now.raw) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(id, busUsersSuffix) || now.kind != string(sdkdecl.TypeFile) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
return changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// composedByDigest finds, in the node's bucket, the composition of module at fingerprint fp whose digest is digest: the
|
||||||
|
// key's current value or one of the revisions it keeps.
|
||||||
|
func composedByDigest(ctx context.Context, at composedPlace, module, fp, digest string) (sdkdelivery.Composed, bool, error) {
|
||||||
|
if fp == "" {
|
||||||
|
return sdkdelivery.Composed{}, false, nil
|
||||||
|
}
|
||||||
|
key, err := sdkdelivery.ComposedKey(at.node, module, fp)
|
||||||
|
if err != nil {
|
||||||
|
return sdkdelivery.Composed{}, false, nil
|
||||||
|
}
|
||||||
|
values, err := historyOf(ctx, at.bucket, key)
|
||||||
|
if err != nil {
|
||||||
|
return sdkdelivery.Composed{}, false, err
|
||||||
|
}
|
||||||
|
for _, v := range values {
|
||||||
|
c, err := sdkdelivery.DecodeComposed(v)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if d, err := sdkdelivery.ComposedDigestOf(c); err == nil && d == digest {
|
||||||
|
return c, true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sdkdelivery.Composed{}, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// historyOf is every value a key keeps, the newest first; none when there is no such key.
|
||||||
|
func historyOf(ctx context.Context, kv composedBucketsKV, key string) ([][]byte, error) {
|
||||||
|
if h, ok := kv.(interface {
|
||||||
|
History(ctx context.Context, key string, opts ...jetstream.WatchOpt) ([]jetstream.KeyValueEntry, error)
|
||||||
|
}); ok {
|
||||||
|
entries, err := h.History(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := make([][]byte, 0, len(entries))
|
||||||
|
for i := len(entries) - 1; i >= 0; i-- {
|
||||||
|
if entries[i].Operation() == jetstream.KeyValuePut {
|
||||||
|
out = append(out, entries[i].Value())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return [][]byte{e.Value()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judged applies ADR 0377 §3 to two compositions of one module.
|
||||||
|
func judged(from, to sdkdelivery.Composed) (sdkdelivery.Judgement, string) {
|
||||||
|
before, after := resourcesByID(from), resourcesByID(to)
|
||||||
|
if ids := changedOf(before, after, func(kind string) bool { return kind == string(sdkdecl.TypeContainer) }); len(ids) > 0 {
|
||||||
|
return sdkdelivery.JudgedRecreates, "it recreates " + strings.Join(ids, ", ")
|
||||||
|
}
|
||||||
|
if from.Data != to.Data {
|
||||||
|
return sdkdelivery.JudgedMovesData, "its data declaration changes"
|
||||||
|
}
|
||||||
|
places := dataPlaces(before, after)
|
||||||
|
held := func(kind string) bool {
|
||||||
|
return kind == string(sdkdecl.TypeDirectory) || kind == string(sdkdecl.TypeFile)
|
||||||
|
}
|
||||||
|
var moved []string
|
||||||
|
for _, id := range changedOf(before, after, held) {
|
||||||
|
r := before[id]
|
||||||
|
if r == nil {
|
||||||
|
r = after[id]
|
||||||
|
}
|
||||||
|
if withinPlaces(r.path, places) {
|
||||||
|
moved = append(moved, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if to.Data || from.Data {
|
||||||
|
moved = append(moved, changedOf(before, after, func(kind string) bool { return kind == string(sdkdecl.TypeProcess) })...)
|
||||||
|
}
|
||||||
|
if len(moved) > 0 {
|
||||||
|
sort.Strings(moved)
|
||||||
|
return sdkdelivery.JudgedMovesData, "it changes where its data is kept: " + strings.Join(moved, ", ")
|
||||||
|
}
|
||||||
|
return sdkdelivery.JudgedNeither, "it recreates no container and moves no data"
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgedResource is a resource as the judgement reads it: its kind, the path it acts on, what its containers mount,
|
||||||
|
// and its canonical bytes.
|
||||||
|
type judgedResource struct {
|
||||||
|
kind string
|
||||||
|
path string
|
||||||
|
mounts []string
|
||||||
|
raw []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func resourcesByID(c sdkdelivery.Composed) map[string]*judgedResource {
|
||||||
|
out := map[string]*judgedResource{}
|
||||||
|
for _, raw := range c.Resources {
|
||||||
|
var head struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
// A container's volumes, `<path on the machine>:<path inside>[:<options>]`.
|
||||||
|
Volumes []string `json:"volumes"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &head); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r := &judgedResource{kind: head.Type, path: head.Path, raw: compactComposed(raw)}
|
||||||
|
for _, v := range head.Volumes {
|
||||||
|
source, _, _ := strings.Cut(v, ":")
|
||||||
|
r.mounts = append(r.mounts, source)
|
||||||
|
}
|
||||||
|
out[head.ID] = r
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// compactComposed is a resource's bytes canonical: its keys sorted, its spacing dropped, its numbers as written, so
|
||||||
|
// two compositions that say the same compare equal however each was written.
|
||||||
|
func compactComposed(raw []byte) []byte {
|
||||||
|
var v any
|
||||||
|
d := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
d.UseNumber()
|
||||||
|
if err := d.Decode(&v); err != nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
out, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// changedOf are the ids of the resources of the kinds counted that are changed, added or removed, sorted.
|
||||||
|
func changedOf(before, after map[string]*judgedResource, counted func(kind string) bool) []string {
|
||||||
|
var out []string
|
||||||
|
for id, now := range after {
|
||||||
|
if !counted(now.kind) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if was, ok := before[id]; !ok || !bytes.Equal(was.raw, now.raw) {
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for id, was := range before {
|
||||||
|
if _, ok := after[id]; !ok && counted(was.kind) {
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// dataPlaces are the paths where the module keeps what a move must not lose: what its containers mount, and its
|
||||||
|
// directories, before and after.
|
||||||
|
func dataPlaces(before, after map[string]*judgedResource) []string {
|
||||||
|
var out []string
|
||||||
|
for _, rs := range []map[string]*judgedResource{before, after} {
|
||||||
|
for _, r := range rs {
|
||||||
|
if r.kind == string(sdkdecl.TypeContainer) {
|
||||||
|
out = append(out, r.mounts...)
|
||||||
|
}
|
||||||
|
if r.kind == string(sdkdecl.TypeDirectory) && r.path != "" {
|
||||||
|
out = append(out, r.path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// withinPlaces says p is one of places, or lies within one, or holds one.
|
||||||
|
func withinPlaces(p string, places []string) bool {
|
||||||
|
if p == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
p = path.Clean(p)
|
||||||
|
for _, place := range places {
|
||||||
|
if place == "" || !strings.HasPrefix(place, "/") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
place = path.Clean(place)
|
||||||
|
if p == place || strings.HasPrefix(p, place+"/") || strings.HasPrefix(place, p+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// sameResources says two compositions of a module hold the same resources in the same order, each compared canonical.
|
||||||
|
func sameResources(a, b sdkdelivery.Composed) bool {
|
||||||
|
if len(a.Resources) != len(b.Resources) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range a.Resources {
|
||||||
|
if !bytes.Equal(compactComposed(a.Resources[i]), compactComposed(b.Resources[i])) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mailOn is a composition of mesh-mail on ace at fingerprint c, from the given resources, each saying its module.
|
||||||
|
func mailOn(t *testing.T, c byte, data bool, resources ...string) sdkdelivery.Composed {
|
||||||
|
t.Helper()
|
||||||
|
out := sdkdelivery.Composed{Node: "ace", Module: "mesh-mail", FP: sdkdelivery.SourcePrefix2 + strings.Repeat(string(c), 64), Data: data}
|
||||||
|
for _, r := range resources {
|
||||||
|
out.Resources = append(out.Resources, json.RawMessage(r))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
mailContainer = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:1111111111111111111111111111111111111111111111111111111111111111","volumes":["/var/lib/mesh-mail/spool:/spool"]}`
|
||||||
|
mailContainer2 = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:2222222222222222222222222222222222222222222222222222222222222222","volumes":["/var/lib/mesh-mail/spool:/spool"]}`
|
||||||
|
mailSpool = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/var/lib/mesh-mail/spool"}`
|
||||||
|
mailSpoolMoved = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/srv/mesh-mail/spool"}`
|
||||||
|
mailTools = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"a"}`
|
||||||
|
mailTools2 = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"b"}`
|
||||||
|
mailConfig = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"a"}`
|
||||||
|
mailConfig2 = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"b"}`
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1945/AC12: captured pairs of compositions are judged by ADR 0377 §3's tests: a changed container recreates; with no
|
||||||
|
// container changed, a directory moved, a file within the data a container mounts changed, or the data declaration
|
||||||
|
// changed moves data; a tools file nothing mounts changed is neither.
|
||||||
|
func TestAMoveIsJudgedByWhatItRecreatesOrMoves(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
from, to sdkdelivery.Composed
|
||||||
|
want sdkdelivery.Judgement
|
||||||
|
}{
|
||||||
|
{"a new image", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools), sdkdelivery.JudgedRecreates},
|
||||||
|
{"a container gone", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailSpool), sdkdelivery.JudgedRecreates},
|
||||||
|
{"its spool moved", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpoolMoved), sdkdelivery.JudgedMovesData},
|
||||||
|
{"a file in its spool", mailOn(t, 'a', true, mailContainer, mailSpool, mailConfig), mailOn(t, 'b', true, mailContainer, mailSpool, mailConfig2), sdkdelivery.JudgedMovesData},
|
||||||
|
{"its data declaration", mailOn(t, 'a', false, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpool), sdkdelivery.JudgedMovesData},
|
||||||
|
{"its tools only", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools2), sdkdelivery.JudgedNeither},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got, words := judged(c.from, c.to); got != c.want || words == "" {
|
||||||
|
t.Errorf("%s: judged %s (%q), want %s", c.name, got, words, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC12: a move is judged against the composition the node's have names: one the node never ran is neither; one
|
||||||
|
// whose composition is gone cannot be judged and says why; and with A running, B held and C composed, C is judged
|
||||||
|
// against A, and judged again when the have changes.
|
||||||
|
func TestAMoveIsJudgedAgainstWhatTheNodeRuns(t *testing.T) {
|
||||||
|
at, _, _, haves := aPlace(t, "ace")
|
||||||
|
ctx := context.Background()
|
||||||
|
a := mailOn(t, 'a', true, mailContainer, mailSpool, mailTools)
|
||||||
|
b := mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools)
|
||||||
|
c := mailOn(t, 'c', true, mailContainer, mailSpool, mailTools2)
|
||||||
|
write := func(comp sdkdelivery.Composed) (string, string) {
|
||||||
|
key, _ := sdkdelivery.ComposedKey("ace", comp.Module, comp.FP)
|
||||||
|
if _, err := writeComposed(ctx, at, key, comp); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d, _ := sdkdelivery.ComposedDigestOf(comp)
|
||||||
|
return key, d
|
||||||
|
}
|
||||||
|
judgedNow := func(key string) sdkdelivery.ComposedDigest {
|
||||||
|
e, err := at.digests.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
v, err := sdkdelivery.DecodeComposedDigest(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
keyA, digestA := write(a)
|
||||||
|
if v := judgedNow(keyA); v.Judgement != sdkdelivery.JudgedNeither || v.Against != "" {
|
||||||
|
t.Errorf("a module the node never ran: %+v", v)
|
||||||
|
}
|
||||||
|
if v := judgedNow(keyA); v.Policy != "roll" || v.PolicyFrom != "default" {
|
||||||
|
t.Errorf("the policy beside the judgement: %+v", v)
|
||||||
|
}
|
||||||
|
haves["mesh-mail"] = sdkdelivery.Have{FP: a.FP, Composed: digestA, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||||
|
keyB, _ := write(b)
|
||||||
|
if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedRecreates || v.Against != digestA {
|
||||||
|
t.Errorf("B against A: %+v", v)
|
||||||
|
}
|
||||||
|
// B is held (the assigner writes no want for it); the node still runs A.
|
||||||
|
keyC, digestC := write(c)
|
||||||
|
if v := judgedNow(keyC); v.Judgement != sdkdelivery.JudgedNeither || v.Against != digestA {
|
||||||
|
t.Errorf("C against A, B held: %+v", v)
|
||||||
|
}
|
||||||
|
// The have changes to C: every judgement of the module is written again, against what runs now (the have watch).
|
||||||
|
haves["mesh-mail"] = sdkdelivery.Have{FP: c.FP, Composed: digestC, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||||
|
if n, err := rejudge(ctx, at, "mesh-mail"); err != nil || n == 0 {
|
||||||
|
t.Fatalf("rejudged %d (%v)", n, err)
|
||||||
|
}
|
||||||
|
if v := judgedNow(keyC); v.Against != digestC || v.Judgement != sdkdelivery.JudgedNeither {
|
||||||
|
t.Errorf("C once the node runs it: %+v", v)
|
||||||
|
}
|
||||||
|
if v := judgedNow(keyB); v.Against != digestC || v.Judgement != sdkdelivery.JudgedRecreates {
|
||||||
|
t.Errorf("B once the node runs C: %+v", v)
|
||||||
|
}
|
||||||
|
// A person sets the module's policy: written beside every composition of it.
|
||||||
|
at.policy = func(string) (string, string, string) { return "record", "person", "its spool" }
|
||||||
|
if _, err := rejudge(ctx, at, "mesh-mail"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v := judgedNow(keyB); v.Policy != "record" || v.PolicyFrom != "person" || v.PolicyWhy != "its spool" {
|
||||||
|
t.Errorf("the person's policy beside B: %+v", v)
|
||||||
|
}
|
||||||
|
// A have naming a composition the buckets no longer hold.
|
||||||
|
haves["mesh-mail"] = sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("9", 64), Composed: "sha256:" + strings.Repeat("9", 64),
|
||||||
|
Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||||
|
write(b)
|
||||||
|
if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedCannot || !strings.Contains(v.Words, "gone") {
|
||||||
|
t.Errorf("against a composition that is gone: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
busServer = `{"id":"nats.server","type":"container","module":"nats","name":"nats","image":"nats@sha256:4444444444444444444444444444444444444444444444444444444444444444"}`
|
||||||
|
busServer2 = `{"id":"nats.server","type":"container","module":"nats","name":"nats","image":"nats@sha256:5555555555555555555555555555555555555555555555555555555555555555"}`
|
||||||
|
busUsers = `{"id":"nats.bus-users","type":"file","module":"nats","path":"/etc/nats/users.conf","content":"users: [a]"}`
|
||||||
|
busUsers2 = `{"id":"nats.bus-users","type":"file","module":"nats","path":"/etc/nats/users.conf","content":"users: [a, b]"}`
|
||||||
|
)
|
||||||
|
|
||||||
|
// busOn is a composition of the bus on ace from the given resources.
|
||||||
|
func busOn(t *testing.T, resources ...string) sdkdelivery.Composed {
|
||||||
|
t.Helper()
|
||||||
|
out := sdkdelivery.Composed{Node: "ace", Module: "nats", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64)}
|
||||||
|
for _, r := range resources {
|
||||||
|
out.Resources = append(out.Resources, json.RawMessage(r))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0540 clause 9: a have that names no composition, only a fingerprint (the switch: the node runs the declaration it
|
||||||
|
// last applied), is judged against the module's slice of that declaration, read from the kept stream of declarations:
|
||||||
|
// an equal slice is unchanged; a slice differing only in the bus's user list is that (round 1's A2, so no yes waits on it at the switch); a changed setting is judged as any move; no declaration to read cannot be judged.
|
||||||
|
func TestAHaveWithNoCompositionIsJudgedAgainstTheLastDeclaration(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
other := `{"id":"mesh-calendar.server","type":"container","module":"mesh-calendar","name":"calendar","image":"cal@sha256:3333333333333333333333333333333333333333333333333333333333333333"}`
|
||||||
|
unsaid := `{"id":"mesh-mail.tools","type":"file","path":"/usr/lib/mesh-mail/tools.json","content":"a"}`
|
||||||
|
declared := func(resources ...string) []byte {
|
||||||
|
body, err := json.Marshal(map[string]any{"node": "ace", "resources": json.RawMessage("[" + strings.Join(resources, ",") + "]")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
signed, err := json.Marshal(link.Signed{Declaration: body, Signature: []byte("s")}) // as link.Envelope sent it
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return signed
|
||||||
|
}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
last []byte // the node's last declaration on the stream; nil when there is none
|
||||||
|
to sdkdelivery.Composed
|
||||||
|
want sdkdelivery.Judgement
|
||||||
|
inWords string
|
||||||
|
userList bool
|
||||||
|
}{
|
||||||
|
{"an equal slice", declared(mailContainer, other, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedUnchanged, "last applied", false},
|
||||||
|
{"a declared resource saying no module is not equal", declared(mailContainer, other, mailSpool, unsaid), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedNeither, "last applied", false},
|
||||||
|
{"a changed image", declared(mailContainer, other, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools), sdkdelivery.JudgedRecreates, "last applied", false},
|
||||||
|
{"a changed setting in its spool", declared(mailContainer, mailSpool, mailConfig), mailOn(t, 'b', true, mailContainer, mailSpool, mailConfig2), sdkdelivery.JudgedMovesData, "last applied", false},
|
||||||
|
{"a changed tools file", declared(mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools2), sdkdelivery.JudgedNeither, "last applied", false},
|
||||||
|
{"only the bus's user list", declared(busServer, busUsers, other), busOn(t, busServer, busUsers2), sdkdelivery.JudgedNeither, "only the bus's user list", true},
|
||||||
|
{"the bus's user list and its container", declared(busServer, busUsers), busOn(t, busServer2, busUsers2), sdkdelivery.JudgedRecreates, "last applied", false},
|
||||||
|
{"no declaration kept", nil, mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedCannot, "names no composition", false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
at, _, _, haves := aPlace(t, "ace")
|
||||||
|
last := c.last
|
||||||
|
at.lastApplied = func(_ context.Context, module string) ([]json.RawMessage, string, bool, error) {
|
||||||
|
if last == nil {
|
||||||
|
return nil, "", false, nil
|
||||||
|
}
|
||||||
|
return sliceOfDeclaration(last, module)
|
||||||
|
}
|
||||||
|
haves[c.to.Module] = sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||||
|
key, _ := sdkdelivery.ComposedKey("ace", c.to.Module, c.to.FP)
|
||||||
|
digest, err := sdkdelivery.ComposedDigestOf(c.to)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", c.name, err)
|
||||||
|
}
|
||||||
|
got, err := judgeMove(ctx, at, key, c.to, digest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", c.name, err)
|
||||||
|
}
|
||||||
|
if got.UserListOnly != c.userList {
|
||||||
|
t.Errorf("%s: user list only %v, want %v", c.name, got.UserListOnly, c.userList)
|
||||||
|
}
|
||||||
|
if got.Judgement != c.want || !strings.Contains(got.Words, c.inWords) {
|
||||||
|
t.Errorf("%s: judged %s (%q), want %s", c.name, got.Judgement, got.Words, c.want)
|
||||||
|
}
|
||||||
|
if _, err := sdkdelivery.EncodeComposedDigest(got); err != nil {
|
||||||
|
t.Errorf("%s: the judgement written is refused: %v", c.name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,264 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A new fingerprint of a registered module, from its done key to its compositions (novox/hq ADR 0539 clauses 5 and 7,
|
||||||
|
// ADR 0501 §2). With no delivery plan left to take a module's done key in, a write of the builder's or the indexer's
|
||||||
|
// keys does it: the controller takes in the done key of the fingerprint the indexer's trunk value names (registers the
|
||||||
|
// module at it, as a plan's take-in did), composes every node the module is assigned to (which makes what a module's
|
||||||
|
// new manifest needs, a credential among it, as a send made it), composes the node holding the bus (its user list now
|
||||||
|
// carries the module's permissions at the new fingerprint: a new revision of the bus module's composition alone,
|
||||||
|
// written into its want at once by the assigner), and writes `grants.<module>.<fp>`. The assigner's sixth and seventh
|
||||||
|
// rules then let a want name the fingerprint. **No timer**: woken by the keys, and a starting controller looks at every
|
||||||
|
// module once (B5's start pass).
|
||||||
|
|
||||||
|
// doneKeySteps are what taking a module's done key in does, each a step a test may stand in for.
|
||||||
|
type doneKeySteps struct {
|
||||||
|
// held says the catalogue holds the module (a new one is the new-module take-in's, new_modules.go).
|
||||||
|
held func(ctx context.Context, module string) (bool, error)
|
||||||
|
// trunk is the indexer's trunk value of the module.
|
||||||
|
trunk func(ctx context.Context, module string) (sdkdelivery.Module, error)
|
||||||
|
// registeredAt says the module is registered at the commit.
|
||||||
|
registeredAt func(ctx context.Context, module, commit string) (bool, error)
|
||||||
|
// takeIn registers the module from the builder's done key of the trunk's fingerprint (takeInKey); errKeyWaits when
|
||||||
|
// the key is not done yet.
|
||||||
|
takeIn func(ctx context.Context, module string, trunk sdkdelivery.Module) error
|
||||||
|
// nodesOf are the nodes the module is assigned to; busHolder the node holding the bus, "" for none.
|
||||||
|
nodesOf func(ctx context.Context, module string) ([]string, error)
|
||||||
|
busHolder func(ctx context.Context) (string, error)
|
||||||
|
// compose writes the nodes' compositions; grants writes the grants issued.
|
||||||
|
compose func(ctx context.Context, nodes []string) error
|
||||||
|
grants func(ctx context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// take takes module's done key in and composes what it changed. It answers what it did in a line, "" when there was
|
||||||
|
// nothing to do.
|
||||||
|
func (s doneKeySteps) take(ctx context.Context, module string) (string, error) {
|
||||||
|
held, err := s.held(ctx, module)
|
||||||
|
if err != nil || !held {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
trunk, err := s.trunk(ctx, module)
|
||||||
|
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(trunk.FP2, sdkdelivery.SourcePrefix2) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
at, err := s.registeredAt(ctx, module, trunk.Commit)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
said := ""
|
||||||
|
if !at {
|
||||||
|
if err := s.takeIn(ctx, module, trunk); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
said = fmt.Sprintf("%s registered from the builder's done key %s (%s)", module, short(trunk.FP2), short(trunk.Commit))
|
||||||
|
}
|
||||||
|
nodes, err := s.nodesOf(ctx, module)
|
||||||
|
if err != nil {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return said, nil
|
||||||
|
}
|
||||||
|
// The module's nodes first: composing them makes what the new manifest needs, its credentials among it.
|
||||||
|
if err := s.compose(ctx, nodes); err != nil {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
// Then the bus's user list, which carries the module's permissions at its new fingerprint.
|
||||||
|
holder, err := s.busHolder(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
if holder != "" && !inList(nodes, holder) {
|
||||||
|
if err := s.compose(ctx, []string{holder}); err != nil {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Then the grant, which with the composition lets a want name the fingerprint (rules 6 and 7).
|
||||||
|
if err := s.grants(ctx); err != nil {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
if said == "" {
|
||||||
|
said = fmt.Sprintf("%s at %s", module, short(trunk.FP2))
|
||||||
|
}
|
||||||
|
return said + "; composed for " + strings.Join(nodes, ", ") + ", its grant written", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func inList(list []string, s string) bool {
|
||||||
|
for _, v := range list {
|
||||||
|
if v == s {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// doneKeysMu keeps one take-in of a module's done key at a time in this process, per module.
|
||||||
|
var doneKeysMu sync.Map
|
||||||
|
|
||||||
|
// takeDoneKey is a woken module's take-in on the live stores and buckets, said. A take-in that fails (its compositions
|
||||||
|
// or its grant not written) is tried again every doneKeyRetry while it fails, loudly: printed each time, and an urgent
|
||||||
|
// condition raised by the first until it has worked, as the chain's own moves are (layBuildsOutSoon). A key not done
|
||||||
|
// yet is no failure: its next write wakes this again.
|
||||||
|
func takeDoneKey(ctx context.Context, open *stores, module string) {
|
||||||
|
mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
|
||||||
|
mu.(*sync.Mutex).Lock()
|
||||||
|
defer mu.(*sync.Mutex).Unlock()
|
||||||
|
failed := false
|
||||||
|
for {
|
||||||
|
err := takeDoneKeyOnce(ctx, open, module)
|
||||||
|
o := doneKeyObservation(module)
|
||||||
|
if err == nil || errors.Is(err, errKeyWaits) {
|
||||||
|
if keeper := conditionsFrom; keeper != nil && failed {
|
||||||
|
_, _ = keeper.Clear(ctx, o.Key(), "composed and granted")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Printf("done keys: %s: %v; tried again in %s\n", module, err, doneKeyRetry)
|
||||||
|
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||||
|
o.Summary += err.Error()
|
||||||
|
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(doneKeyRetry):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// doneKeyRetry is how long a failed take-in of a done key waits before it tries again.
|
||||||
|
var doneKeyRetry = 30 * time.Second
|
||||||
|
|
||||||
|
// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
|
||||||
|
// they are, no want can name it.
|
||||||
|
const kindNotComposed = "fingerprint-not-composed"
|
||||||
|
|
||||||
|
func doneKeyObservation(module string) conditions.Observation {
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeBuild, ID: module, Token: kindNotComposed, Kind: kindNotComposed,
|
||||||
|
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys",
|
||||||
|
Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeDoneKeyOnce is one take-in of a module's done key.
|
||||||
|
func takeDoneKeyOnce(ctx context.Context, open *stores, module string) error {
|
||||||
|
keys, closeKeys, err := openBuilderKeys()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the builder's keys cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
defer closeKeys()
|
||||||
|
said, err := liveDoneKeySteps(open, keys).take(ctx, module)
|
||||||
|
if err == nil && said != "" {
|
||||||
|
fmt.Printf("done keys: %s\n", said)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// liveDoneKeySteps are the steps on the controller's own stores and the bus.
|
||||||
|
func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
|
||||||
|
inv := open.inventory
|
||||||
|
return doneKeySteps{
|
||||||
|
held: func(ctx context.Context, module string) (bool, error) {
|
||||||
|
_, err := inv.SourceOf(ctx, module)
|
||||||
|
if errors.Is(err, inventory.ErrNoSuchModule) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return err == nil, err
|
||||||
|
},
|
||||||
|
trunk: func(ctx context.Context, module string) (sdkdelivery.Module, error) {
|
||||||
|
return keys.ModuleAt(ctx, module, "")
|
||||||
|
},
|
||||||
|
registeredAt: func(ctx context.Context, module, commit string) (bool, error) {
|
||||||
|
src, err := inv.SourceOf(ctx, module)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return sameCommitID(src.BuiltFrom, commit), nil
|
||||||
|
},
|
||||||
|
takeIn: func(ctx context.Context, module string, trunk sdkdelivery.Module) error {
|
||||||
|
_, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||||
|
if errors.Is(err, inventory.ErrSuperseded) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
nodesOf: func(ctx context.Context, module string) ([]string, error) {
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.Manifest.Module == module {
|
||||||
|
return e.On, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
},
|
||||||
|
busHolder: func(ctx context.Context) (string, error) {
|
||||||
|
holders, err := seatHolders(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return holders[catalogue.BrokerSeat].Node, nil
|
||||||
|
},
|
||||||
|
compose: func(ctx context.Context, nodes []string) error {
|
||||||
|
_, err := composeModules(ctx, open, nodes)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
grants: func(ctx context.Context) error {
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return grantsIssued(ctx, inv, js, "done keys: ")
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchDoneKeys wakes the take-in of a module's done key for each module the builder or the indexer writes, until ctx
|
||||||
|
// ends: never on the watch's own goroutine.
|
||||||
|
func watchDoneKeys(ctx context.Context, open *stores) {
|
||||||
|
keys, closeKeys, err := openBuilderKeys()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("done keys: the builder's keys are not watched: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := builderkeys.Watch(ctx, keys.JS, func(module string) {
|
||||||
|
inBackground(func() { takeDoneKey(ctx, open, module) })
|
||||||
|
}); err != nil {
|
||||||
|
closeKeys()
|
||||||
|
fmt.Printf("done keys: the builder's keys are not watched: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
closeKeys()
|
||||||
|
}()
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||||
|
)
|
||||||
|
|
||||||
|
// stepsFor are doneKeySteps over a module registered at registered, whose trunk value names fp at commit, assigned to
|
||||||
|
// nodes, the bus held on holder; every act is written to acts in order.
|
||||||
|
func stepsFor(acts *[]string, registered, commit, fp string, nodes []string, holder string) doneKeySteps {
|
||||||
|
return doneKeySteps{
|
||||||
|
held: func(_ context.Context, module string) (bool, error) { return module == "mesh-mail", nil },
|
||||||
|
trunk: func(context.Context, string) (sdkdelivery.Module, error) {
|
||||||
|
if fp == "" {
|
||||||
|
return sdkdelivery.Module{}, builderkeys.ErrNoValue
|
||||||
|
}
|
||||||
|
return sdkdelivery.Module{Commit: commit, FP2: fp}, nil
|
||||||
|
},
|
||||||
|
registeredAt: func(_ context.Context, _ string, c string) (bool, error) { return c == registered, nil },
|
||||||
|
takeIn: func(_ context.Context, module string, trunk sdkdelivery.Module) error {
|
||||||
|
*acts = append(*acts, "take in "+module+" at "+trunk.Commit)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
nodesOf: func(context.Context, string) ([]string, error) { return nodes, nil },
|
||||||
|
busHolder: func(context.Context) (string, error) { return holder, nil },
|
||||||
|
compose: func(_ context.Context, ns []string) error {
|
||||||
|
*acts = append(*acts, "compose "+strings.Join(ns, ","))
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
grants: func(context.Context) error { *acts = append(*acts, "grants"); return nil },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC3 (unit half): a done key of a new fingerprint of an assigned module is taken in, then the module's nodes are
|
||||||
|
// composed, then the node holding the bus (its user list), then the grants: the grant and the composition exist
|
||||||
|
// before the assigner's rules let a want name the fingerprint. A module registered at the trunk's commit already is
|
||||||
|
// composed and granted without a second take-in; one assigned nowhere, unregistered, or with no trunk value is left
|
||||||
|
// alone.
|
||||||
|
func TestADoneKeyIsTakenInComposedAndGranted(t *testing.T) {
|
||||||
|
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64)
|
||||||
|
var acts []string
|
||||||
|
said, err := stepsFor(&acts, "old", "new", fp, []string{"ace", "g14"}, "novox").take(context.Background(), "mesh-mail")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := strings.Join(acts, "; "); got != "take in mesh-mail at new; compose ace,g14; compose novox; grants" {
|
||||||
|
t.Errorf("the acts were %q", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "registered from the builder's done key") {
|
||||||
|
t.Errorf("said %q", said)
|
||||||
|
}
|
||||||
|
|
||||||
|
acts = nil
|
||||||
|
if _, err := stepsFor(&acts, "new", "new", fp, []string{"novox"}, "novox").take(context.Background(), "mesh-mail"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := strings.Join(acts, "; "); got != "compose novox; grants" {
|
||||||
|
t.Errorf("a module registered at its trunk already: %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, s := range map[string]doneKeySteps{
|
||||||
|
"assigned nowhere": stepsFor(&acts, "old", "new", fp, nil, "novox"),
|
||||||
|
"no trunk value": stepsFor(&acts, "old", "new", "", []string{"ace"}, "novox"),
|
||||||
|
} {
|
||||||
|
acts = nil
|
||||||
|
if _, err := s.take(context.Background(), "mesh-mail"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if name == "assigned nowhere" && strings.Join(acts, "; ") != "take in mesh-mail at new" {
|
||||||
|
t.Errorf("%s: %v", name, acts)
|
||||||
|
}
|
||||||
|
if name == "no trunk value" && len(acts) != 0 {
|
||||||
|
t.Errorf("%s: %v", name, acts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
acts = nil
|
||||||
|
if _, err := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox").take(context.Background(), "unknown"); err != nil || len(acts) != 0 {
|
||||||
|
t.Errorf("a module the catalogue does not hold: %v %v", acts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC4: the bus's user list is a file of the bus module's composition (ADR 0539 clause 7): a change of the composed
|
||||||
|
// users is a new revision of that composition alone, every other composition of the node keeping its digest.
|
||||||
|
func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testing.T) {
|
||||||
|
body, fps := capturedBody(t, "g14")
|
||||||
|
fps["nats"] = sdkdelivery.SourcePrefix2 + strings.Repeat("e", 64)
|
||||||
|
withUsers := func(users string) map[string]sdkdelivery.Composed {
|
||||||
|
var d map[string]any
|
||||||
|
if err := json.Unmarshal(body, &d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// As the composer writes it for the module holding mesh-broker (mesh-sdk catalogue: BusUsersID, owned by the module).
|
||||||
|
d["resources"] = append(d["resources"].([]any), map[string]any{"id": "nats.bus-users", "type": "file",
|
||||||
|
"module": "nats", "path": "/var/lib/mesh/nats/users.conf", "content": users, "mode": "0600"})
|
||||||
|
raw, _ := json.Marshal(d)
|
||||||
|
comps, _, err := compositionsOf("g14", raw, fps, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return comps
|
||||||
|
}
|
||||||
|
before, after := withUsers("authorization { users = [ {user: a} ] }\n"), withUsers("authorization { users = [ {user: a}, {user: b} ] }\n")
|
||||||
|
natsKey, _ := sdkdelivery.ComposedKey("g14", "nats", fps["nats"])
|
||||||
|
for key, c := range before {
|
||||||
|
d1, _ := sdkdelivery.ComposedDigestOf(c)
|
||||||
|
d2, _ := sdkdelivery.ComposedDigestOf(after[key])
|
||||||
|
if changed := d1 != d2; changed != (key == natsKey) {
|
||||||
|
t.Errorf("%s changed %v; only the bus module's composition changes with its user list", key, changed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945 (the review's D1): a grant that is not written is the take-in's error, never swallowed: with no send left,
|
||||||
|
// nothing else would write it, and no want could name the fingerprint.
|
||||||
|
func TestAGrantNotWrittenIsTheTakeInsError(t *testing.T) {
|
||||||
|
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64)
|
||||||
|
var acts []string
|
||||||
|
s := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox")
|
||||||
|
s.grants = func(context.Context) error { return errors.New("the bucket mesh-controller_grants: not found") }
|
||||||
|
if _, err := s.take(context.Background(), "mesh-mail"); err == nil || !strings.Contains(err.Error(), "grants") {
|
||||||
|
t.Errorf("a grant not written: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,362 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// composeModules composes each named node and writes its compositions that differ (ADR 0539 clause 5): what each act
|
||||||
|
// that once sent a declaration calls instead. A node that cannot compose is that node's: the others are written, and
|
||||||
|
// every refusal is named in the error. It answers the keys it wrote, by node. Under the lease only: a controller that
|
||||||
|
// does not hold it writes nothing and says so.
|
||||||
|
func composeModules(ctx context.Context, open *stores, nodes []string) (map[string][]string, error) {
|
||||||
|
if _, err := theLease.epoch(ctx); err != nil {
|
||||||
|
return nil, fmt.Errorf("nothing is composed: %w", err)
|
||||||
|
}
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("nothing is composed: the bus: %w", err)
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sign, err := controllerSigner(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("nothing is composed: the controller's signing key: %w", err)
|
||||||
|
}
|
||||||
|
fps, err := currentFingerprints(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
manifests := make(map[string]catalogue.Manifest, len(entries))
|
||||||
|
for _, e := range entries {
|
||||||
|
manifests[e.Manifest.Module] = e.Manifest
|
||||||
|
}
|
||||||
|
upgrades, err := open.inventory.Upgrades(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
policy := func(module string) (string, string, string) {
|
||||||
|
u, ok := upgrades[module]
|
||||||
|
if !ok {
|
||||||
|
return "", "", ""
|
||||||
|
}
|
||||||
|
return u.Policy(), u.From, u.Why
|
||||||
|
}
|
||||||
|
sort.Strings(nodes)
|
||||||
|
out := map[string][]string{}
|
||||||
|
var errs []error
|
||||||
|
for _, node := range nodes {
|
||||||
|
at, err := openComposedPlace(ctx, js, bus.Conn(), node)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at.sign, at.policy = sign, policy
|
||||||
|
if composeTrace != nil {
|
||||||
|
traceInput(ctx, open, node)
|
||||||
|
}
|
||||||
|
declared, err := composeSend(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
body, err := declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// The provisioner's plans of the modules the node's composition hands to it, for the assignments mirror.
|
||||||
|
if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s: its provisioner's plans: %w", node, err))
|
||||||
|
}
|
||||||
|
comps, moved, err := compositionsOf(node, body, fps, manifests, declared.modules)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(moved) > 0 {
|
||||||
|
fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n",
|
||||||
|
node, strings.Join(moved, ", "))
|
||||||
|
}
|
||||||
|
if composeTrace != nil {
|
||||||
|
traceComposed(ctx, at, node, comps)
|
||||||
|
}
|
||||||
|
wrote, err := writeNodeCompositions(ctx, at, comps)
|
||||||
|
if len(wrote) > 0 {
|
||||||
|
out[node] = wrote
|
||||||
|
fmt.Printf("compositions: %s: wrote %s\n", node, strings.Join(wrote, ", "))
|
||||||
|
}
|
||||||
|
// And where a data-keeping provision bound each consumer: what the next resolution keeps it at (novox/hq ADR 0232).
|
||||||
|
if len(declared.Bindings) > 0 {
|
||||||
|
if err := open.inventory.RecordBindings(ctx, node, declared.Bindings); err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s: where its consumers are bound to their data: %w", node, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeAndSay is composeModules for an act that has nothing to do with its answer but say it: what each former send
|
||||||
|
// of a verb or an event calls.
|
||||||
|
func composeAndSay(ctx context.Context, open *stores, nodes []string) error {
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := composeModules(ctx, open, nodes)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("compositions: %v\n", err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// openComposedPlace opens a node's buckets: its compositions, the digests, its files, and reads its want and have.
|
||||||
|
func openComposedPlace(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, node string) (composedPlace, error) {
|
||||||
|
if err := broker.EnsureComposed(ctx, js, node); err != nil {
|
||||||
|
return composedPlace{}, err
|
||||||
|
}
|
||||||
|
name, err := sdkdelivery.ComposedBucketFor(node)
|
||||||
|
if err != nil {
|
||||||
|
return composedPlace{}, err
|
||||||
|
}
|
||||||
|
bucket, err := js.KeyValue(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return composedPlace{}, fmt.Errorf("the bucket %s: %w", name, err)
|
||||||
|
}
|
||||||
|
digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket)
|
||||||
|
if err != nil {
|
||||||
|
return composedPlace{}, fmt.Errorf("the bucket %s: %w", sdkdelivery.ControllerComposedBucket, err)
|
||||||
|
}
|
||||||
|
objects, err := openBucket(ctx, conn, node)
|
||||||
|
if err != nil {
|
||||||
|
return composedPlace{}, err
|
||||||
|
}
|
||||||
|
return composedPlace{node: node, bucket: bucket, digests: digests, objects: objects,
|
||||||
|
wanted: func(ctx context.Context, module string) (string, error) { return wantedOn(ctx, js, node, module) },
|
||||||
|
ran: func(ctx context.Context, module string) (sdkdelivery.Have, bool, error) {
|
||||||
|
return haveOn(ctx, js, node, module)
|
||||||
|
},
|
||||||
|
lastApplied: lastAppliedOn(js, node),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantedOn is the fingerprint the assigner's want names for a module on a node; "" when there is none or the bucket is
|
||||||
|
// not there yet.
|
||||||
|
func wantedOn(ctx context.Context, js jetstream.JetStream, node, module string) (string, error) {
|
||||||
|
kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket)
|
||||||
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
key, err := sdkdelivery.WantKey(node, module)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
w, err := sdkdelivery.DecodeWant(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
return "", nil // a want of release 5, naming no composition: no fingerprint a composition must be kept for
|
||||||
|
}
|
||||||
|
return w.FP, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// haveOn is what a node's node-engine says it runs of a module.
|
||||||
|
func haveOn(ctx context.Context, js jetstream.JetStream, node, module string) (sdkdelivery.Have, bool, error) {
|
||||||
|
kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket)
|
||||||
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||||
|
return sdkdelivery.Have{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return sdkdelivery.Have{}, false, err
|
||||||
|
}
|
||||||
|
key, err := sdkdelivery.HaveKey(node, module)
|
||||||
|
if err != nil {
|
||||||
|
return sdkdelivery.Have{}, false, err
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return sdkdelivery.Have{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return sdkdelivery.Have{}, false, err
|
||||||
|
}
|
||||||
|
h, err := sdkdelivery.DecodeHave(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
return sdkdelivery.Have{}, false, nil
|
||||||
|
}
|
||||||
|
return h, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantedOnTheBus is every fingerprint a want names, as `<module> <fingerprint>`: what the keep set keeps the builds of
|
||||||
|
// (ADR 0539 clause 1). Read from the assigner's bucket on each look; no bucket yet names none.
|
||||||
|
func wantedOnTheBus(ctx context.Context) (map[string]bool, error) {
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket)
|
||||||
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||||
|
return map[string]bool{}, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
lister, err := kv.ListKeys(ctx)
|
||||||
|
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||||
|
return map[string]bool{}, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]bool{}
|
||||||
|
for key := range lister.Keys() {
|
||||||
|
_, module, err := sdkdelivery.ParseWantKey(key)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
w, err := sdkdelivery.DecodeWant(e.Value())
|
||||||
|
if err != nil || w.FP == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[module+" "+w.FP] = true
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// plansKept is a composition's provisioner's plans as the store keeps them: by module, each plan and what requires a
|
||||||
|
// module only pulled in by another (design 57 P6, issue 765); nil when it hands none.
|
||||||
|
func plansKept(d sendable) []byte {
|
||||||
|
if len(d.plans) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
kept := map[string]composedPlan{}
|
||||||
|
for module, p := range d.plans {
|
||||||
|
kept[module] = composedPlan{Plan: p, RequiredBy: d.requiredBy[module]}
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(kept) // plain strings and slices: it marshals
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// composedPlan is one handed module's plan as the store keeps it, the shape inventory.SentProvisioningPlans reads.
|
||||||
|
type composedPlan struct {
|
||||||
|
catalogue.Plan
|
||||||
|
RequiredBy string `json:"required-by,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastAppliedOn reads, once, the declaration the node was last sent from the kept stream of declarations (the last
|
||||||
|
// message on `mesh.node.<node>.declare`), and answers each module's slice of it (its resources that say the module, or
|
||||||
|
// whose id names it) with the declaration's digest. None when the stream or the node's message is gone.
|
||||||
|
func lastAppliedOn(js jetstream.JetStream, node string) func(ctx context.Context, module string) ([]json.RawMessage, string, bool, error) {
|
||||||
|
var once sync.Once
|
||||||
|
var last []byte
|
||||||
|
var found bool
|
||||||
|
var readErr error
|
||||||
|
return func(ctx context.Context, module string) ([]json.RawMessage, string, bool, error) {
|
||||||
|
once.Do(func() { last, found, readErr = lastDeclaration(ctx, js, node) })
|
||||||
|
if readErr != nil || !found {
|
||||||
|
return nil, "", false, readErr
|
||||||
|
}
|
||||||
|
return sliceOfDeclaration(last, module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastDeclaration is the node's last signed declaration on the kept stream, as the stream holds it.
|
||||||
|
func lastDeclaration(ctx context.Context, js jetstream.JetStream, node string) ([]byte, bool, error) {
|
||||||
|
stream, err := js.Stream(ctx, broker.DeclarationsStream)
|
||||||
|
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
msg, err := stream.GetLastMsgForSubject(ctx, "mesh.node."+node+".declare")
|
||||||
|
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return msg.Data, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sliceOfDeclaration is module's slice of a signed declaration as the stream holds it, and the digest of the
|
||||||
|
// declaration it was signed over.
|
||||||
|
func sliceOfDeclaration(data []byte, module string) ([]json.RawMessage, string, bool, error) {
|
||||||
|
var signed link.Signed
|
||||||
|
if err := json.Unmarshal(data, &signed); err != nil {
|
||||||
|
return nil, "", false, fmt.Errorf("the last declaration is not readable: %w", err)
|
||||||
|
}
|
||||||
|
var envelope struct {
|
||||||
|
Resources []json.RawMessage `json:"resources"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(signed.Declaration, &envelope); err != nil {
|
||||||
|
return nil, "", false, fmt.Errorf("the last declaration is not readable: %w", err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(signed.Declaration)
|
||||||
|
return sliceOfModule(envelope.Resources, module), "sha256:" + hex.EncodeToString(sum[:]), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sliceOfModule is, of a declaration's resources in their order, those of module: saying it, or with no module said
|
||||||
|
// and an id that names it (sdkdecl.ModuleGuess).
|
||||||
|
func sliceOfModule(resources []json.RawMessage, module string) []json.RawMessage {
|
||||||
|
var out []json.RawMessage
|
||||||
|
for _, r := range resources {
|
||||||
|
var head struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(r, &head) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
owner := head.Module
|
||||||
|
if owner == "" {
|
||||||
|
owner, _ = sdkdecl.ModuleGuess(head.ID)
|
||||||
|
}
|
||||||
|
if owner == module {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1945/AC13: the node-engine's composition on a node carries the operator's public keys as its file operator-keys,
|
||||||
|
// from its setting operator-signers, beside the controller's key it already trusts (ADR 0539 clause 3, decided with the
|
||||||
|
// lead 2026-10-24: the switch's last declaration carries it because today's controller composes the same file from
|
||||||
|
// the same manifest). The manifest is mesh-host's module.json at 20d0998c (testdata/mesh-host-module.json), never
|
||||||
|
// typed by hand.
|
||||||
|
func TestTheNodeEnginesCompositionCarriesTheOperatorsKeys(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("testdata/mesh-host-module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m catalogue.Manifest
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Its archive names a build artifact, which only its registration turns into a reference: left out here, the rest of
|
||||||
|
// the manifest as captured.
|
||||||
|
var kept []map[string]any
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r["type"] != "archive" {
|
||||||
|
kept = append(kept, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.Resources = kept
|
||||||
|
const key = "MCowBQYDK2VwAyEAb3BlcmF0b3Itc2lnbmluZy1rZXktZm9yLXRlc3RzLW9ubHk="
|
||||||
|
in := compose.Input{Catalogue: map[string]catalogue.Manifest{m.Module: m},
|
||||||
|
Nodes: []compose.Node{{Name: "ace", Assigned: []string{m.Module}, Stated: true,
|
||||||
|
Settings: map[string][]catalogue.Layer{m.Module: {{From: "mesh", Values: map[string]any{"operator-signers": key}}}}}}}
|
||||||
|
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("6", 64)
|
||||||
|
all, _, err := compose.Compositions(in, "ace", map[string]string{m.Module: fp}, compose.Reading)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
k, _ := sdkdelivery.ComposedKey("ace", m.Module, fp)
|
||||||
|
var file struct{ Path, Content string }
|
||||||
|
for _, r := range all[k].Resources {
|
||||||
|
var head struct{ ID, Path, Content string }
|
||||||
|
_ = json.Unmarshal(r, &head)
|
||||||
|
if strings.HasSuffix(head.ID, "operator-keys") {
|
||||||
|
file.Path, file.Content = head.Path, head.Content
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if file.Path != "/var/lib/mesh-host/operator-keys" || file.Content != key+"\n" {
|
||||||
|
t.Errorf("the node-engine's composition carries operator-keys at %q with %q, want the operator's key", file.Path, file.Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1945 (dev-r6-controller's finding): the provisioner's plans a composition hands are kept in the shape the
|
||||||
|
// assignments mirror reads (inventory.SentProvisioningPlans: each module's plan and what requires it), and a
|
||||||
|
// composition that hands none keeps none.
|
||||||
|
func TestAComposedNodesProvisionerPlansAreKept(t *testing.T) {
|
||||||
|
if plansKept(sendable{}) != nil {
|
||||||
|
t.Errorf("a composition that hands nothing kept plans")
|
||||||
|
}
|
||||||
|
d := sendable{plans: map[string]catalogue.Plan{"umami": {Placement: map[string]string{"dir:state": "/var/lib/umami"}}},
|
||||||
|
requiredBy: map[string]string{"umami": "analytics"}}
|
||||||
|
var kept map[string]struct {
|
||||||
|
Placement map[string]string `json:"placement"`
|
||||||
|
RequiredBy string `json:"required-by"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(plansKept(d), &kept); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if kept["umami"].Placement["dir:state"] != "/var/lib/umami" || kept["umami"].RequiredBy != "analytics" {
|
||||||
|
t.Errorf("kept %+v", kept)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The judgement beside each composition is rewritten when what it is judged against or with changes (novox/hq ADR
|
||||||
|
// 0539 clauses 2 and 14): when the node's have of the module changes (it runs another composition, or holds), and when
|
||||||
|
// a person sets the module's policy. Heard as the have bucket's writes and the `upgrade` verb, never on a timer.
|
||||||
|
|
||||||
|
// rejudge writes again the judgement and policy beside every composition of module on the node at, where they differ.
|
||||||
|
func rejudge(ctx context.Context, at composedPlace, module string) (int, error) {
|
||||||
|
lister, err := at.bucket.ListKeys(ctx)
|
||||||
|
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
wrote := 0
|
||||||
|
var errs []error
|
||||||
|
for key := range lister.Keys() {
|
||||||
|
node, m, _, isNode, err := sdkdelivery.ParseComposedKey(key)
|
||||||
|
if err != nil || isNode || node != at.node || m != module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c, revision, err := readComposed(ctx, at.bucket, key)
|
||||||
|
if err != nil || revision == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.Signature = nil
|
||||||
|
digest, err := sdkdelivery.ComposedDigestOf(c)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
judged, err := judgeMove(ctx, at, key, c, digest)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
w, err := keepDigest(ctx, at.digests, key, judged)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
if w {
|
||||||
|
wrote++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return wrote, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// rejudgeOn opens the node's buckets and rejudges module there, under the lease.
|
||||||
|
func rejudgeOn(ctx context.Context, open *stores, nodes []string, module string) error {
|
||||||
|
if _, err := theLease.epoch(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
upgrades, err := open.inventory.Upgrades(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var errs []error
|
||||||
|
for _, node := range nodes {
|
||||||
|
at, err := openComposedPlace(ctx, js, bus.Conn(), node)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at.policy = func(m string) (string, string, string) {
|
||||||
|
u, ok := upgrades[m]
|
||||||
|
if !ok {
|
||||||
|
return "", "", ""
|
||||||
|
}
|
||||||
|
return u.Policy(), u.From, u.Why
|
||||||
|
}
|
||||||
|
if _, err := rejudge(ctx, at, module); err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// rejudgeModule rejudges a module on every node it is assigned to: what a person's policy changes.
|
||||||
|
func rejudgeModule(ctx context.Context, open *stores, module string) {
|
||||||
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.Manifest.Module == module && len(e.On) > 0 {
|
||||||
|
if err := rejudgeOn(ctx, open, e.On, module); err != nil {
|
||||||
|
fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchHavesForJudgement rejudges a node's module each time its have changes, until ctx ends: never on the watch's own
|
||||||
|
// goroutine.
|
||||||
|
func watchHavesForJudgement(ctx context.Context, open *stores) {
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
bus.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket)
|
||||||
|
if err != nil {
|
||||||
|
bus.Close()
|
||||||
|
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||||
|
if err != nil {
|
||||||
|
bus.Close()
|
||||||
|
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer bus.Close()
|
||||||
|
defer w.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case e, ok := <-w.Updates():
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
node, module, isNode, err := sdkdelivery.ParseHaveKey(e.Key())
|
||||||
|
if err != nil || isNode {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
inBackground(func() {
|
||||||
|
if err := rejudgeOn(ctx, open, []string{node}, module); err != nil {
|
||||||
|
fmt.Printf("compositions: %s on %s is not judged again: %v\n", module, node, err)
|
||||||
|
}
|
||||||
|
// A node's node-engine moved: the declarations' stream goes once every node runs the new one.
|
||||||
|
if module == nodeEngineModule {
|
||||||
|
if _, err := retireDeclarations(ctx, open); err != nil {
|
||||||
|
fmt.Printf("compositions: whether the declarations' stream may go cannot be read: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
@@ -0,0 +1,615 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||||
|
"github.com/nats-io/nats-server/v2/server"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||||
|
"github.com/novox/mesh-controller/internal/chain"
|
||||||
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The compositions through the controller whole (novox/hq ADR 0539; the review's G1): the real store, the controller's
|
||||||
|
// signing key, and a bus that runs the user list the controller composes from that store, so a subject release 6 uses
|
||||||
|
// and the list does not grant is refused as it would be on the mesh. serve() runs whole on it; the builder and the
|
||||||
|
// indexer are their own accounts writing their own keys; a rotation is the operator's verb.
|
||||||
|
type servedMesh struct {
|
||||||
|
open *stores
|
||||||
|
controller jetstream.JetStream
|
||||||
|
builder jetstream.JetStream
|
||||||
|
indexer jetstream.JetStream
|
||||||
|
}
|
||||||
|
|
||||||
|
// keyOfManifest is a done key of a module at fp with no artifacts, its manifest the given one, and the indexer's trunk
|
||||||
|
// value naming it in the catalogue's repository at commit.
|
||||||
|
func keyOfManifest(t *testing.T, m catalogue.Manifest, fp, commit string) (raw []byte, k sdkdelivery.BuilderBuild, trunk sdkdelivery.Module) {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
digest := sdkdelivery.ManifestDigest(raw)
|
||||||
|
return raw, sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "anchor", Manifest: digest},
|
||||||
|
sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: commit, Path: "modules/" + m.Module}
|
||||||
|
}
|
||||||
|
|
||||||
|
func aServedMesh(t *testing.T) *servedMesh {
|
||||||
|
t.Helper()
|
||||||
|
open := theCatalogue(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
withTrunkList(t, true)
|
||||||
|
ident := identity.ForTest(t)
|
||||||
|
if _, err := ident.Establish(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bus, running the user list the controller composes from the store, with the builder and the indexer as the
|
||||||
|
// accounts their own manifests make (each writes its own state).
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var controller broker.Principal
|
||||||
|
for _, p := range users {
|
||||||
|
if p.Kind == broker.KindController {
|
||||||
|
controller = p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: controller.Username(), Kind: inventory.BusController})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
builder := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.BuilderModule, State: []string{"builds"}}
|
||||||
|
indexer := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.IndexerModule, State: []string{"modules", "manifests"}}
|
||||||
|
kept, err := inv.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hashes := map[string]string{}
|
||||||
|
for name, u := range kept {
|
||||||
|
hashes[name] = u.PasswordHash
|
||||||
|
}
|
||||||
|
fixture, _ := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||||
|
hashes[builder.Username()], hashes[indexer.Username()] = string(fixture), string(fixture)
|
||||||
|
filled, missing := broker.WithPasswords([]broker.Principal{controller, builder, indexer}, hashes)
|
||||||
|
if len(missing) > 0 {
|
||||||
|
t.Fatalf("no password for %v", missing)
|
||||||
|
}
|
||||||
|
accounts, err := broker.ComposeAccounts(filled)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||||
|
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
opts, err := server.ProcessConfigFile(conf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the composed user list does not parse: %v", err)
|
||||||
|
}
|
||||||
|
opts.NoLog, opts.NoSigs = true, true
|
||||||
|
s, err := server.NewServer(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go s.Start()
|
||||||
|
if !s.ReadyForConnections(10 * time.Second) {
|
||||||
|
t.Fatal("the bus did not come up")
|
||||||
|
}
|
||||||
|
t.Cleanup(s.Shutdown)
|
||||||
|
u, _ := url.Parse(s.ClientURL())
|
||||||
|
u.User = url.UserPassword(controller.Username(), password)
|
||||||
|
t.Setenv(broker.NATSVar, u.String())
|
||||||
|
|
||||||
|
connect := func(user, pw string) jetstream.JetStream {
|
||||||
|
t.Helper()
|
||||||
|
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, pw), nats.CustomInboxPrefix("_INBOX."+user))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s cannot connect: %v", user, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(nc.Close)
|
||||||
|
js, err := jetstream.New(nc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return js
|
||||||
|
}
|
||||||
|
m := &servedMesh{open: open, controller: connect(controller.Username(), password),
|
||||||
|
builder: connect(builder.Username(), "pw"), indexer: connect(indexer.Username(), "pw")}
|
||||||
|
|
||||||
|
// The test's own inventory on the bus's chain, as the serving controller's will be: what is registered and assigned
|
||||||
|
// below is in the chain it reads (assignments and builds live on the bus, ADR 0408).
|
||||||
|
chainConn, err := nats.Connect(s.ClientURL(), nats.UserInfo(controller.Username(), password),
|
||||||
|
nats.CustomInboxPrefix("_INBOX."+controller.Username()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(chainConn.Close)
|
||||||
|
// The controller's own buckets, the chain's among them, as it asserts them at its start.
|
||||||
|
asserter, err := broker.Dial(u.String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(asserter.Close)
|
||||||
|
if err := asserter.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatalf("the controller's buckets: %v", err)
|
||||||
|
}
|
||||||
|
c, err := chain.OnTheBus(ctx, chainConn, func(context.Context) error { return nil })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inv.UseChain(c)
|
||||||
|
|
||||||
|
// A provider and its consumer (bindings_test.go's store and board), registered from done keys of the catalogue's
|
||||||
|
// repository, so each has a fingerprint of the new form; the store holds the mesh's seat on anchor, the board on
|
||||||
|
// laptop is bound to it.
|
||||||
|
shelf := map[string]catalogue.Manifest{}
|
||||||
|
for _, m := range storeManifests() {
|
||||||
|
shelf[m.Module] = m
|
||||||
|
}
|
||||||
|
// The board is given its credential in a file, as a consumer of a database is.
|
||||||
|
board := shelf["board"]
|
||||||
|
board.Secrets = map[string]string{"postgres-database": "/var/lib/board/database-password"}
|
||||||
|
shelf["board"] = board
|
||||||
|
for i, module := range []string{"store", "board"} {
|
||||||
|
raw, k, trunk := keyOfManifest(t, shelf[module], fpOf(byte('a'+i)), "c0ffee0123456789")
|
||||||
|
keys := trunkKeys{keyBuckets: keyBuckets{keys: map[string]builderkeys.Key{module: aBuilderKey(module, trunk.FP2, k)},
|
||||||
|
manifests: map[string]string{trunk.Manifest: string(raw)}}, trunk: trunk}
|
||||||
|
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
|
||||||
|
t.Fatalf("%s is not registered from its key: %v", module, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The private network on both machines, as aMesh assigns it, now on the chain the controller reads.
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, node, overlay.Name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.Assign(ctx, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.Assign(ctx, "laptop", "board"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The builder's and the indexer's buckets as their own parts make them, there before the controller starts (its
|
||||||
|
// watch of their keys begins at its start); and the plans' buckets a controller of release 5 left.
|
||||||
|
for _, b := range []string{sdkdelivery.BuilderBuildsBucket, sdkdelivery.ModulesBucket, sdkdelivery.ManifestsBucket} {
|
||||||
|
if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
|
||||||
|
t.Fatalf("the bucket %s: %v", b, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, b := range broker.PlanStateBuckets {
|
||||||
|
if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
keepWhatServeSets(t)
|
||||||
|
serving, stop := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- serve(serving) }()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(30 * time.Second):
|
||||||
|
t.Log("the controller did not stop within 30 s")
|
||||||
|
}
|
||||||
|
// A stopping controller refuses every verb from then on as a handover (issue 289), which its process exits
|
||||||
|
// after; this one is the test process, so a later test's verb would be refused without running (issue 1945:
|
||||||
|
// the conditions verb's own process was never started). Its stop marks it once the serving context ends.
|
||||||
|
for deadline := time.Now().Add(5 * time.Second); !handingOver.Load() && time.Now().Before(deadline); {
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
handingOver.Store(false)
|
||||||
|
})
|
||||||
|
// Served once both nodes have their own composition and their module's: the start's pass wrote them all. A node's
|
||||||
|
// compositions are written in key order, so anchor's own comes before its store's (issue 1945: a test that reads the
|
||||||
|
// revisions at once must not catch the pass midway).
|
||||||
|
for _, k := range []struct{ node, module, fp string }{{"anchor", "", ""}, {"laptop", "", ""}, {"anchor", "store", fpOf('a')},
|
||||||
|
{"laptop", "board", fpOf('b')}} {
|
||||||
|
key, _ := sdkdelivery.ComposedNodeKey(k.node)
|
||||||
|
if k.module != "" {
|
||||||
|
key, _ = sdkdelivery.ComposedKey(k.node, k.module, k.fp)
|
||||||
|
}
|
||||||
|
m.waitFor(t, done, k.node, key, 0)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// aBuilderKey is the builder's done key of module at fp, as the take-in reads it.
|
||||||
|
func aBuilderKey(module, fp string, b sdkdelivery.BuilderBuild) builderkeys.Key {
|
||||||
|
return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, Build: b}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitFor waits until the node's composition bucket holds key at a revision above after, failing when the controller
|
||||||
|
// stops or a minute passes: a test's wait for what an event made, never a loop of the code under test.
|
||||||
|
func (m *servedMesh) waitFor(t *testing.T, done chan error, node, key string, after uint64) uint64 {
|
||||||
|
t.Helper()
|
||||||
|
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||||
|
deadline := time.Now().Add(time.Minute)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("the controller stopped: %v", err)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil {
|
||||||
|
if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after {
|
||||||
|
return e.Revision()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("%s in %s was not written within a minute; %s", key, bucket, m.whatIsThere(t, node))
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// whatIsThere says, for a failure, what the node's bucket holds, the ids its own composition carries and the
|
||||||
|
// fingerprints the controller has registered.
|
||||||
|
func (m *servedMesh) whatIsThere(t *testing.T, node string) string {
|
||||||
|
t.Helper()
|
||||||
|
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||||
|
out := "keys:"
|
||||||
|
kv, err := m.controller.KeyValue(context.Background(), bucket)
|
||||||
|
if err != nil {
|
||||||
|
return out + " " + err.Error()
|
||||||
|
}
|
||||||
|
lister, err := kv.ListKeys(context.Background())
|
||||||
|
if err == nil {
|
||||||
|
for k := range lister.Keys() {
|
||||||
|
out += " " + k
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if lister, err := kv.ListKeys(context.Background()); err == nil {
|
||||||
|
for k := range lister.Keys() {
|
||||||
|
if e, err := kv.Get(context.Background(), k); err == nil {
|
||||||
|
if c, err := sdkdelivery.DecodeComposed(e.Value()); err == nil {
|
||||||
|
out += fmt.Sprintf("; %s at revision %d carries", k, e.Revision())
|
||||||
|
for _, r := range c.Resources {
|
||||||
|
var head struct{ ID string }
|
||||||
|
_ = json.Unmarshal(r, &head)
|
||||||
|
out += " " + head.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
holders, herr := m.open.inventory.HoldersOf(context.Background(), "postgres-database", "")
|
||||||
|
out += fmt.Sprintf("; holders of postgres-database: %+v (%v)", holders, herr)
|
||||||
|
own, _ := sdkdelivery.ComposedNodeKey(node)
|
||||||
|
if e, err := kv.Get(context.Background(), own); err == nil {
|
||||||
|
if c, err := sdkdelivery.DecodeComposed(e.Value()); err == nil {
|
||||||
|
out += "; its own carries:"
|
||||||
|
for _, r := range c.Resources {
|
||||||
|
var head struct{ ID string }
|
||||||
|
_ = json.Unmarshal(r, &head)
|
||||||
|
out += " " + head.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fps, err := currentFingerprints(context.Background(), m.open.inventory)
|
||||||
|
out += fmt.Sprintf("; fingerprints registered: %v (%v)", fps, err)
|
||||||
|
// And what composing the node says now, in this process, under the serving controller's lease.
|
||||||
|
_, cerr := composeModules(context.Background(), m.open, []string{node})
|
||||||
|
return out + fmt.Sprintf("; composing %s now: %v", node, cerr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// revisions are a node's composition keys and their revisions.
|
||||||
|
func (m *servedMesh) revisions(t *testing.T, node string) map[string]uint64 {
|
||||||
|
t.Helper()
|
||||||
|
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||||
|
kv, err := m.controller.KeyValue(t.Context(), bucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lister, err := kv.ListKeys(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := map[string]uint64{}
|
||||||
|
for k := range lister.Keys() {
|
||||||
|
if e, err := kv.Get(t.Context(), k); err == nil {
|
||||||
|
out[k] = e.Revision()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// declarationsSent is how many declarations the bus's stream of them holds: none is ever sent in release 6.
|
||||||
|
func (m *servedMesh) declarationsSent(t *testing.T) uint64 {
|
||||||
|
t.Helper()
|
||||||
|
s, err := m.controller.Stream(t.Context(), broker.DeclarationsStream)
|
||||||
|
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := s.Info(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return info.State.Msgs
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC6, AC14: started whole on a bus running its own user list, the controller composes every node and every
|
||||||
|
// assigned module, deletes the plans' buckets and says each as bucket-deleted, and sends no declaration.
|
||||||
|
func TestServedTheStartComposesEveryNodeAndDeletesThePlansBuckets(t *testing.T) {
|
||||||
|
m := aServedMesh(t)
|
||||||
|
for _, k := range []struct{ node, module string }{{"anchor", "store"}, {"laptop", "board"}} {
|
||||||
|
fp := map[string]string{"store": fpOf('a'), "board": fpOf('b')}[k.module]
|
||||||
|
key, _ := sdkdelivery.ComposedKey(k.node, k.module, fp)
|
||||||
|
m.waitFor(t, nil, k.node, key, 0)
|
||||||
|
}
|
||||||
|
for _, b := range broker.PlanStateBuckets {
|
||||||
|
if _, err := m.controller.KeyValue(t.Context(), b); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||||
|
t.Errorf("%s is still there: %v", b, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := m.declarationsSent(t); n != 0 {
|
||||||
|
t.Errorf("%d declarations were sent", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC3: the builder writes the done key of a new fingerprint of an assigned module and the indexer its trunk
|
||||||
|
// value: the controller, hearing the keys, registers it, writes its composition on the node it is assigned to and
|
||||||
|
// `grants.<module>.<fp>`; nothing is sent.
|
||||||
|
func TestServedADoneKeyIsComposedAndGranted(t *testing.T) {
|
||||||
|
m := aServedMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
shelf := map[string]catalogue.Manifest{}
|
||||||
|
for _, mf := range storeManifests() {
|
||||||
|
shelf[mf.Module] = mf
|
||||||
|
}
|
||||||
|
next := shelf["board"]
|
||||||
|
next.Secrets = map[string]string{"postgres-database": "/var/lib/board/database-password"}
|
||||||
|
next.Version = "2"
|
||||||
|
fp := fpOf('c')
|
||||||
|
raw, k, trunk := keyOfManifest(t, next, fp, "d00d0123456789ab")
|
||||||
|
manifests, err := m.indexer.KeyValue(ctx, sdkdelivery.ManifestsBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
value, _ := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: "board", JSON: string(raw)})
|
||||||
|
if _, err := manifests.Put(ctx, trunk.Manifest, value); err != nil {
|
||||||
|
t.Fatalf("the indexer cannot write its manifest: %v", err)
|
||||||
|
}
|
||||||
|
builds, err := m.builder.KeyValue(ctx, sdkdelivery.BuilderBuildsBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
key, _ := sdkdelivery.BuilderBuildKey("board", fp)
|
||||||
|
value, _ = sdkdelivery.EncodeBuilderBuild(k)
|
||||||
|
if _, err := builds.Put(ctx, key, value); err != nil {
|
||||||
|
t.Fatalf("the builder cannot write its key: %v", err)
|
||||||
|
}
|
||||||
|
modules, err := m.indexer.KeyValue(ctx, sdkdelivery.ModulesBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mk, _ := sdkdelivery.ModulesKey("board")
|
||||||
|
value, _ = sdkdelivery.EncodeModule(trunk)
|
||||||
|
if _, err := modules.Put(ctx, mk, value); err != nil {
|
||||||
|
t.Fatalf("the indexer cannot write its trunk value: %v", err)
|
||||||
|
}
|
||||||
|
composed, _ := sdkdelivery.ComposedKey("laptop", "board", fp)
|
||||||
|
m.waitFor(t, nil, "laptop", composed, 0)
|
||||||
|
grants, err := m.controller.KeyValue(ctx, broker.GrantsBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gk, _ := sdkdelivery.ControllerGrantKey("board", fp)
|
||||||
|
// The take-in writes the grant after the compositions (composed_keys.go), so it is waited for, not read in the instant
|
||||||
|
// the composition shows: a loaded build node read it a moment early (issue 1945).
|
||||||
|
becomes(t, 10*time.Second, "grants."+gk+" written", func() bool {
|
||||||
|
_, err := grants.Get(ctx, gk)
|
||||||
|
return err == nil
|
||||||
|
})
|
||||||
|
if n := m.declarationsSent(t); n != 0 {
|
||||||
|
t.Errorf("%d declarations were sent", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC2: the operator rotates the store's credential, the real verb: the compositions that carry it (the store's
|
||||||
|
// grants on anchor, and the board's on laptop when it carries the credential) get a new revision, every other
|
||||||
|
// composition keeps its own, and nothing is sent.
|
||||||
|
func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) {
|
||||||
|
m := aServedMesh(t)
|
||||||
|
// Everything that writes or discards a credential row, and what composing each node saw and decided, in order, with the
|
||||||
|
// callers: one failure shows the cause (issue 1945).
|
||||||
|
var traceMu sync.Mutex
|
||||||
|
var trace []string
|
||||||
|
note := func(line string) {
|
||||||
|
traceMu.Lock()
|
||||||
|
defer traceMu.Unlock()
|
||||||
|
trace = append(trace, time.Now().Format("15:04:05.000")+" "+line)
|
||||||
|
}
|
||||||
|
inventory.SecretTrace = func(op, what string) { note("secret " + op + ": " + what) }
|
||||||
|
composeTrace = func(line string) { note("compose " + line) }
|
||||||
|
t.Cleanup(func() { inventory.SecretTrace, composeTrace = nil, nil })
|
||||||
|
traced := func() string {
|
||||||
|
traceMu.Lock()
|
||||||
|
defer traceMu.Unlock()
|
||||||
|
return strings.Join(trace, "\n ")
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if t.Failed() {
|
||||||
|
t.Logf("TRACE\n %s", traced())
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
before := map[string]map[string]uint64{"anchor": m.revisions(t, "anchor"), "laptop": m.revisions(t, "laptop")}
|
||||||
|
holders, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "")
|
||||||
|
var waited uint64
|
||||||
|
// The verb runs here as its child process runs it: its inventory takes the chain from the bus the served controller keeps it
|
||||||
|
// on (the registrations and assignments are there), not the one in memory the fixture's store was made with.
|
||||||
|
inventory.OnTheBusChainForTest(t)
|
||||||
|
said, err := capturedOutput(func() error {
|
||||||
|
if err := rotateCommand(t.Context(), []string{"postgres-database"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The serving controller's own lines while the rotation lands are captured too.
|
||||||
|
boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b'))
|
||||||
|
waited = m.waitForOrSay(t, "laptop", boardKey, before["laptop"][boardKey])
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("rotate: %v\n%s", err, said)
|
||||||
|
}
|
||||||
|
t.Logf("rotate said (holders before: %+v):\n%s", holders, said)
|
||||||
|
defer func() {
|
||||||
|
if t.Failed() {
|
||||||
|
t.Logf("rotate said:\n%s", said)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b'))
|
||||||
|
storeKey, _ := sdkdelivery.ComposedKey("anchor", "store", fpOf('a'))
|
||||||
|
// The board's sealed credential file is new: its composition moves with the rotation.
|
||||||
|
if rev := waited; rev == 0 {
|
||||||
|
var kept []string
|
||||||
|
for _, line := range strings.Split(said, "\n") {
|
||||||
|
for _, w := range []string{"board", "postgres", "secret", "credential", "laptop", "withdr", "retir", "error", "refused"} {
|
||||||
|
if strings.Contains(line, w) {
|
||||||
|
kept = append(kept, line)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
said = strings.Join(kept, " | ")
|
||||||
|
after, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "")
|
||||||
|
in, ierr := inputFor(t.Context(), m.open)
|
||||||
|
var credentials string
|
||||||
|
if ierr == nil {
|
||||||
|
credentials = fmt.Sprintf("would make %+v; credentials %+v", in.wouldMake, in.Credentials)
|
||||||
|
}
|
||||||
|
state, serr := m.open.inventory.PairCredentialState(t.Context(), inventory.PairCredential{Name: "postgres-database",
|
||||||
|
Consumer: "laptop", ConsumerModule: "board", Provider: "anchor"})
|
||||||
|
d, derr := composeSend(t.Context(), m.open, "laptop")
|
||||||
|
var sealed []string
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if r["id"] == "board.secret-postgres-database" {
|
||||||
|
sealed = append(sealed, fmt.Sprint(r["sealed"])[:min(24, len(fmt.Sprint(r["sealed"])))])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diag := fmt.Sprintf("state %+v %v; compose now %v sealed %v; %s %v", state, serr, derr, sealed, credentials, ierr)
|
||||||
|
// One line: the verdict keeps only a failure's first line (issue 1921).
|
||||||
|
t.Fatalf("the board's composition did not move within a minute after the rotation (revision before %d); TRACE %s; LINES %s; "+
|
||||||
|
"holders before %+v, after %+v; DIAG %s; %s", before["laptop"][boardKey],
|
||||||
|
strings.Join(strings.Fields(strings.ReplaceAll(traced(), "\n", " | ")), " "), strings.Join(strings.Fields(said), " "),
|
||||||
|
holders, after, diag, strings.Join(strings.Fields(m.whatIsThere(t, "laptop")), " "))
|
||||||
|
}
|
||||||
|
moved := map[string]bool{}
|
||||||
|
for node, was := range before {
|
||||||
|
for k, rev := range m.revisions(t, node) {
|
||||||
|
if rev != was[k] {
|
||||||
|
moved[k] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for k := range moved {
|
||||||
|
if k != boardKey && k != storeKey {
|
||||||
|
t.Errorf("%s moved: only the compositions carrying the credential do", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !moved[boardKey] {
|
||||||
|
t.Errorf("the board's composition did not move: %v", moved)
|
||||||
|
}
|
||||||
|
if n := m.declarationsSent(t); n != 0 {
|
||||||
|
t.Errorf("%d declarations were sent", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForOrSay is waitFor that answers 0 instead of failing, for a caller that says more.
|
||||||
|
func (m *servedMesh) waitForOrSay(t *testing.T, node, key string, after uint64) uint64 {
|
||||||
|
t.Helper()
|
||||||
|
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||||
|
deadline := time.Now().Add(time.Minute)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil {
|
||||||
|
if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after {
|
||||||
|
return e.Revision()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// capturedOutput runs act with this process's standard output captured, and answers what it printed.
|
||||||
|
func capturedOutput(act func() error) (string, error) {
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
return "", act()
|
||||||
|
}
|
||||||
|
was := os.Stdout
|
||||||
|
os.Stdout = w
|
||||||
|
done := make(chan string)
|
||||||
|
go func() {
|
||||||
|
var b strings.Builder
|
||||||
|
buf := make([]byte, 4096)
|
||||||
|
for {
|
||||||
|
n, err := r.Read(buf)
|
||||||
|
b.Write(buf[:n])
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
done <- b.String()
|
||||||
|
}()
|
||||||
|
actErr := act()
|
||||||
|
os.Stdout = was
|
||||||
|
w.Close()
|
||||||
|
return <-done, actErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepWhatServeSets gives back, once the served controller stopped, what serve sets for the rest of its process, which
|
||||||
|
// a controller's process exits after and the test process does not (issue 1945): the served mark in the environment, the
|
||||||
|
// hand-act log's connection, the keeper, the status and the event publishers. Registered before the controller starts, so
|
||||||
|
// it runs after the cleanup that stops it.
|
||||||
|
func keepWhatServeSets(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv(servedVar, "")
|
||||||
|
unheld, status, given, check, follow, keeper, hand := logUnheldChanges, statusFrom, givenEvents, checkEvents,
|
||||||
|
link.Calls.Follow, conditionsFrom, handActConn
|
||||||
|
t.Cleanup(func() {
|
||||||
|
logUnheldChanges, statusFrom, givenEvents, checkEvents, link.Calls.Follow, conditionsFrom, handActConn =
|
||||||
|
unheld, status, given, check, follow, keeper, hand
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,341 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the controller does at its start under the lease (novox/hq ADR 0539 clauses 10 and 13):
|
||||||
|
//
|
||||||
|
// - it deletes the buckets of the plans' state this release deleted, each said once in its log and as its event
|
||||||
|
// `bucket-deleted`: a later start finds none and says nothing;
|
||||||
|
// - it composes every node, so every assigned module has its composition at the fingerprint it is registered at
|
||||||
|
// and each node its own (ADR 0539 clause 11: the new controller writes every composition at its start);
|
||||||
|
// - and it composes, on the control-node, the controller it replaced at that controller's own fingerprint, from that
|
||||||
|
// build's manifest, so the operator's way back is one key at the terminal (clause 10).
|
||||||
|
|
||||||
|
// planStateBucketDeleter is what deleting the plans' buckets needs of the bus: jetstream.JetStream, or a test's.
|
||||||
|
type planStateBucketDeleter interface {
|
||||||
|
DeleteKeyValue(ctx context.Context, bucket string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// deletePlanStateBuckets deletes each bucket of the plans' state the bus still holds and says each it deleted; it
|
||||||
|
// answers those it deleted. A bucket not there is no error and is not said.
|
||||||
|
func deletePlanStateBuckets(ctx context.Context, js planStateBucketDeleter, say func(bucket string)) ([]string, error) {
|
||||||
|
var deleted []string
|
||||||
|
var errs []error
|
||||||
|
for _, bucket := range broker.PlanStateBuckets {
|
||||||
|
err := js.DeleteKeyValue(ctx, bucket)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, jetstream.ErrBucketNotFound), errors.Is(err, jetstream.ErrStreamNotFound):
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
errs = append(errs, fmt.Errorf("deleting %s: %w", bucket, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
deleted = append(deleted, bucket)
|
||||||
|
say(bucket)
|
||||||
|
}
|
||||||
|
return deleted, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BucketDeleted is the body of the controller's event `bucket-deleted`.
|
||||||
|
type BucketDeleted struct {
|
||||||
|
Bucket string `json:"bucket"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayBucketDeleted says a deleted bucket in the log and as the controller's event.
|
||||||
|
func sayBucketDeleted(ctx context.Context, bucket string) {
|
||||||
|
sayDeleted(ctx, bucket, "a bucket of the plans' state, which release 6 deleted (novox/hq ADR 0539 clause 13)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayDeleted says a deleted bucket or stream, with why, in the log and as the controller's event bucket-deleted.
|
||||||
|
func sayDeleted(ctx context.Context, bucket, why string) {
|
||||||
|
fmt.Printf("start: deleted %s: %s\n", bucket, why)
|
||||||
|
bus := checkEvents
|
||||||
|
if bus == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(BucketDeleted{Bucket: bucket, Why: why, At: time.Now().UTC()})
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
saying, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
|
defer stop()
|
||||||
|
if err := bus.PublishSeatEvent(saying, link.MeshControllerSeat, "bucket-deleted", body); err != nil {
|
||||||
|
fmt.Printf("start: that %s was deleted could not be said as the event bucket-deleted: %v\n", bucket, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeAtStart is the start's pass: the plans' buckets deleted, every node composed, the replaced controller
|
||||||
|
// composed. Said, never fatal: what is not composed now is composed by the next act that touches it.
|
||||||
|
func composeAtStart(ctx context.Context, open *stores) {
|
||||||
|
if bus, err := aBus(); err != nil {
|
||||||
|
fmt.Printf("start: the plans' buckets are not looked at: %v\n", err)
|
||||||
|
} else {
|
||||||
|
if js, err := jetstream.New(bus.Conn()); err == nil {
|
||||||
|
if _, err := deletePlanStateBuckets(ctx, js, func(b string) { sayBucketDeleted(ctx, b) }); err != nil {
|
||||||
|
fmt.Printf("start: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bus.Close()
|
||||||
|
}
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("start: nothing is composed: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(nodes))
|
||||||
|
for _, n := range nodes {
|
||||||
|
names = append(names, n.Name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
if _, err := composeModules(ctx, open, names); err != nil {
|
||||||
|
fmt.Printf("start: %v\n", err)
|
||||||
|
}
|
||||||
|
if err := composeReplacedController(ctx, open); err != nil {
|
||||||
|
fmt.Printf("start: the controller this one replaced is not composed: %v\n", err)
|
||||||
|
}
|
||||||
|
if _, err := retireDeclarations(ctx, open); err != nil {
|
||||||
|
fmt.Printf("start: whether the declarations' stream may go cannot be read: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeEngineModule is the node-engine's module.
|
||||||
|
const nodeEngineModule = "mesh-host"
|
||||||
|
|
||||||
|
// declarationsDone says every node runs the node-engine at fingerprint current, as its have names: no node still reads
|
||||||
|
// a declaration, and none away will need its last one (ADR 0539 clause 11). No node, or no fingerprint of the new
|
||||||
|
// form, is not done.
|
||||||
|
func declarationsDone(nodes []string, runs map[string]string, current string) bool {
|
||||||
|
if len(nodes) == 0 || !strings.HasPrefix(current, sdkdelivery.SourcePrefix2) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if runs[n] != current {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// declarationsRetiredOn reads whether the declarations are done on the mesh now: every node's have of the node-engine
|
||||||
|
// against the fingerprint it is registered at.
|
||||||
|
func declarationsRetiredOn(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream) (bool, error) {
|
||||||
|
fps, err := currentFingerprints(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(nodes))
|
||||||
|
runs := map[string]string{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
names = append(names, n.Name)
|
||||||
|
h, found, err := haveOn(ctx, js, n.Name, nodeEngineModule)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
runs[n.Name] = h.FP
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return declarationsDone(names, runs, fps[nodeEngineModule]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// retireDeclarations deletes the declarations' stream, with every node's consumer on it, once every node runs the new
|
||||||
|
// node-engine, and says it once; a stream already gone is said by nobody. It answers whether the declarations are done.
|
||||||
|
func retireDeclarations(ctx context.Context, open *stores) (bool, error) {
|
||||||
|
if _, err := theLease.epoch(ctx); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
done, err := declarationsRetiredOn(ctx, open.inventory, js)
|
||||||
|
if err != nil || !done {
|
||||||
|
return done, err
|
||||||
|
}
|
||||||
|
err = js.DeleteStream(ctx, broker.DeclarationsStream)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, jetstream.ErrStreamNotFound):
|
||||||
|
return true, nil
|
||||||
|
case err != nil:
|
||||||
|
return true, fmt.Errorf("deleting the stream %s: %w", broker.DeclarationsStream, err)
|
||||||
|
}
|
||||||
|
sayDeleted(ctx, broker.DeclarationsStream, "the stream of each node's last declaration: every node runs the new "+
|
||||||
|
"node-engine, which reads none (novox/hq ADR 0539 clause 11)")
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// replacedController is the controller a control-node ran before this one: its fingerprint and manifest.
|
||||||
|
type replacedController struct {
|
||||||
|
node, fp string
|
||||||
|
manifest catalogue.Manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
// replacedControllers are, for each node the controller is assigned to, the controller it replaced: the newest build of
|
||||||
|
// the controller that worked, whose fingerprint is of the new form and not the one the controller is registered at now
|
||||||
|
// (the assignments name no build since ADR 0539 §12, so the build records say it). None when there is no such build.
|
||||||
|
func replacedControllers(ctx context.Context, open *stores) ([]replacedController, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
fps, err := currentFingerprints(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var on []string
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.Manifest.Module == controllerModule {
|
||||||
|
on = e.On
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(on) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
builds, err := inv.Builds(ctx, controllerModule, 20)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
before, found := replacedBuild(builds, fps[controllerModule])
|
||||||
|
if !found {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var m catalogue.Manifest
|
||||||
|
if err := json.Unmarshal(before.Manifest, &m); err != nil {
|
||||||
|
return nil, fmt.Errorf("the manifest of %s's build %s: %w", controllerModule, before.ID, err)
|
||||||
|
}
|
||||||
|
var out []replacedController
|
||||||
|
for _, node := range on {
|
||||||
|
out = append(out, replacedController{node: node, fp: before.SourceFingerprint, manifest: m})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// replacedBuild is, of a module's builds newest first, the newest that worked, kept its manifest, and names a
|
||||||
|
// fingerprint of the new form other than current.
|
||||||
|
func replacedBuild(builds []inventory.Build, current string) (inventory.Build, bool) {
|
||||||
|
for _, b := range builds {
|
||||||
|
if b.Worked() && b.SourceFingerprint != current && strings.HasPrefix(b.SourceFingerprint, sdkdelivery.SourcePrefix2) &&
|
||||||
|
len(b.Manifest) > 0 {
|
||||||
|
return b, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inventory.Build{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeReplacedController writes, on each control-node, the controller's composition at the fingerprint of the one
|
||||||
|
// it replaced: the node composed with that build's manifest in place of the controller's registered one, in the
|
||||||
|
// reading mode (nothing made), and only the controller's composition of it kept.
|
||||||
|
func composeReplacedController(ctx context.Context, open *stores) error {
|
||||||
|
if _, err := theLease.epoch(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
replaced, err := replacedControllers(ctx, open)
|
||||||
|
if err != nil || len(replaced) == 0 {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sign, err := controllerSigner(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var errs []error
|
||||||
|
for _, r := range replaced {
|
||||||
|
in, err := inputFor(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, err := replacedComposition(in.Input, r)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s: %w", r.node, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at, err := openComposedPlace(ctx, js, bus.Conn(), r.node)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at.sign = sign
|
||||||
|
key, _ := sdkdelivery.ComposedKey(r.node, controllerModule, r.fp)
|
||||||
|
if wrote, err := writeComposed(ctx, at, key, c); err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
} else if wrote {
|
||||||
|
fmt.Printf("start: composed %s, the controller this one replaced, on %s: the way back is want.%s.%s naming it\n",
|
||||||
|
short(r.fp), r.node, r.node, controllerModule)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// replacedComposition is the controller's composition on r's node at r's fingerprint: in composed with r's manifest
|
||||||
|
// in place of the controller's registered one.
|
||||||
|
func replacedComposition(in compose.Input, r replacedController) (sdkdelivery.Composed, error) {
|
||||||
|
shelf := make(map[string]catalogue.Manifest, len(in.Catalogue))
|
||||||
|
for k, v := range in.Catalogue {
|
||||||
|
shelf[k] = v
|
||||||
|
}
|
||||||
|
shelf[controllerModule] = r.manifest
|
||||||
|
in.Catalogue = shelf
|
||||||
|
all, _, err := compose.Compositions(in, r.node, map[string]string{controllerModule: r.fp}, compose.Reading)
|
||||||
|
if err != nil {
|
||||||
|
return sdkdelivery.Composed{}, err
|
||||||
|
}
|
||||||
|
key, _ := sdkdelivery.ComposedKey(r.node, controllerModule, r.fp)
|
||||||
|
c, ok := all[key]
|
||||||
|
if !ok {
|
||||||
|
return sdkdelivery.Composed{}, fmt.Errorf("%s composes no %s with the replaced build's manifest", r.node, controllerModule)
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// declarationsRetiredNow says the declarations are done on the mesh now, for a raise of the bus's objects; a bus or a
|
||||||
|
// store it cannot read is not done, so the stream is asserted as before.
|
||||||
|
func declarationsRetiredNow(ctx context.Context, inv *inventory.Inventory) bool {
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
done, err := declarationsRetiredOn(ctx, inv, js)
|
||||||
|
return err == nil && done
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeBuckets is the bus's buckets by name, for deleting.
|
||||||
|
type fakeBuckets map[string]bool
|
||||||
|
|
||||||
|
func (f fakeBuckets) DeleteKeyValue(_ context.Context, bucket string) error {
|
||||||
|
if !f[bucket] {
|
||||||
|
return jetstream.ErrBucketNotFound
|
||||||
|
}
|
||||||
|
delete(f, bucket)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC14 (unit half): the first start deletes the merge window's, the silent sends' and the building plans' buckets
|
||||||
|
// and says each once; a later start finds none and says nothing; a bucket of anything else is left alone.
|
||||||
|
func TestTheFirstStartDeletesThePlansBucketsAndSaysEachOnce(t *testing.T) {
|
||||||
|
bus := fakeBuckets{"mesh-controller_merge-window": true, "mesh-controller_rollout-silent": true,
|
||||||
|
"mesh-controller_plans-building": true, "mesh-controller_grants": true}
|
||||||
|
var said []string
|
||||||
|
deleted, err := deletePlanStateBuckets(context.Background(), bus, func(b string) { said = append(said, b) })
|
||||||
|
if err != nil || len(deleted) != 3 || strings.Join(said, " ") != strings.Join(broker.PlanStateBuckets, " ") {
|
||||||
|
t.Fatalf("first start: deleted %v, said %v, %v", deleted, said, err)
|
||||||
|
}
|
||||||
|
if !bus["mesh-controller_grants"] || len(bus) != 1 {
|
||||||
|
t.Errorf("the bus holds %v after the first start", bus)
|
||||||
|
}
|
||||||
|
said = nil
|
||||||
|
if deleted, err := deletePlanStateBuckets(context.Background(), bus, func(b string) { said = append(said, b) }); err != nil || len(deleted)+len(said) != 0 {
|
||||||
|
t.Errorf("a later start deleted %v and said %v (%v)", deleted, said, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC6 (unit half): the controller this one replaced is composed on its node from its own build's manifest at its
|
||||||
|
// own fingerprint: the composition holds what that manifest declares, not what the controller registered now declares.
|
||||||
|
func TestTheReplacedControllerIsComposedFromItsOwnManifest(t *testing.T) {
|
||||||
|
now := catalogue.Manifest{Module: controllerModule, Version: "6", Resources: []map[string]any{
|
||||||
|
{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "six\n"}}}
|
||||||
|
before := catalogue.Manifest{Module: controllerModule, Version: "5", Resources: []map[string]any{
|
||||||
|
{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "five\n"}}}
|
||||||
|
in := compose.Input{Catalogue: map[string]catalogue.Manifest{controllerModule: now},
|
||||||
|
Nodes: []compose.Node{{Name: "anchor", Assigned: []string{controllerModule}, Stated: true}}}
|
||||||
|
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("5", 64)
|
||||||
|
c, err := replacedComposition(in, replacedController{node: "anchor", fp: fp, manifest: before})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c.Module != controllerModule || c.FP != fp || c.Node != "anchor" || len(c.Resources) != 1 {
|
||||||
|
t.Fatalf("composed %s at %s on %s with %d resources", c.Module, c.FP, c.Node, len(c.Resources))
|
||||||
|
}
|
||||||
|
var f struct{ Content string }
|
||||||
|
_ = json.Unmarshal(c.Resources[0], &f)
|
||||||
|
if f.Content != "five\n" {
|
||||||
|
t.Errorf("the replaced controller's composition holds %q, its own manifest says five", f.Content)
|
||||||
|
}
|
||||||
|
if in.Catalogue[controllerModule].Version != "6" {
|
||||||
|
t.Errorf("composing the replaced controller changed the Input's catalogue")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC6 (unit half): the controller replaced is the newest earlier build of the controller that worked and names a
|
||||||
|
// fingerprint of the new form, not the one registered now.
|
||||||
|
func TestTheReplacedControllerIsItsNewestEarlierBuild(t *testing.T) {
|
||||||
|
now, before, older := sdkdelivery.SourcePrefix2+strings.Repeat("6", 64), sdkdelivery.SourcePrefix2+strings.Repeat("5", 64),
|
||||||
|
sdkdelivery.SourcePrefix2+strings.Repeat("4", 64)
|
||||||
|
m := []byte(`{"module":"mesh-controller","version":"5"}`)
|
||||||
|
builds := []inventory.Build{
|
||||||
|
{ID: "c6", SourceFingerprint: now, Manifest: m},
|
||||||
|
{ID: "c5-failed", SourceFingerprint: before, Failed: "no", Manifest: m},
|
||||||
|
{ID: "c5", SourceFingerprint: before, Manifest: m},
|
||||||
|
{ID: "c4", SourceFingerprint: older, Manifest: m},
|
||||||
|
}
|
||||||
|
if b, ok := replacedBuild(builds, now); !ok || b.ID != "c5" {
|
||||||
|
t.Errorf("the replaced build is %+v (%v), want c5", b, ok)
|
||||||
|
}
|
||||||
|
if _, ok := replacedBuild(builds[:1], now); ok {
|
||||||
|
t.Errorf("a controller with no earlier build replaced one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC14 (unit half): the declarations' stream stays while one node's have names another node-engine than the one
|
||||||
|
// registered, or none, and goes once every node's names it.
|
||||||
|
func TestTheDeclarationsGoOnlyOnceEveryNodeRunsTheNewEngine(t *testing.T) {
|
||||||
|
n6 := sdkdelivery.SourcePrefix2 + strings.Repeat("6", 64)
|
||||||
|
nodes := []string{"ace", "g14", "novox"}
|
||||||
|
if declarationsDone(nodes, map[string]string{"ace": n6, "novox": n6}, n6) {
|
||||||
|
t.Errorf("done while g14 says nothing (it sleeps)")
|
||||||
|
}
|
||||||
|
if declarationsDone(nodes, map[string]string{"ace": n6, "g14": "src2:old", "novox": n6}, n6) {
|
||||||
|
t.Errorf("done while g14 runs the old node-engine")
|
||||||
|
}
|
||||||
|
if !declarationsDone(nodes, map[string]string{"ace": n6, "g14": n6, "novox": n6}, n6) {
|
||||||
|
t.Errorf("not done with every node on the new node-engine")
|
||||||
|
}
|
||||||
|
if declarationsDone(nodes, map[string]string{"ace": "", "g14": "", "novox": ""}, "") || declarationsDone(nil, nil, n6) {
|
||||||
|
t.Errorf("done with no fingerprint registered, or no node")
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user