Compare commits
174
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d20a077096 | ||
|
|
e0ce2236dd | ||
|
|
9873b3bf13 | ||
|
|
541603c15c | ||
|
|
106507b1d3 | ||
|
|
e610f2d92c | ||
|
|
f80b6cdbd1 | ||
|
|
5dcf33db45 | ||
|
|
6abce7e956 | ||
|
|
0d2b304f08 | ||
|
|
bdfbd0254f | ||
|
|
16c5e78fa8 | ||
|
|
ed90771382 | ||
|
|
672d1f4ca1 | ||
|
|
208901c6cc | ||
|
|
4ac5cfe3a7 | ||
|
|
22660dc274 | ||
|
|
d7f359c498 | ||
|
|
ed25fd68e2 | ||
|
|
01c5ab2aab | ||
|
|
bb3cd6437b | ||
|
|
0b07e68cb8 | ||
|
|
625d02862c | ||
|
|
e8478e208b | ||
|
|
5761737687 | ||
|
|
89ec48b9a9 | ||
|
|
869fb6d6bf | ||
|
|
d25b69178b | ||
|
|
a7bb1e0b1c | ||
|
|
5eaed84271 | ||
|
|
326b1aec14 | ||
|
|
134d039ff8 | ||
|
|
d90c6ab93a | ||
|
|
6b7d2ec49b | ||
|
|
4ed1057df3 | ||
|
|
1f4c67a01b | ||
|
|
5474ea2d41 | ||
|
|
b7912172af | ||
|
|
b36babcd7d | ||
|
|
49cf0aa562 | ||
|
|
5a4f73f761 | ||
|
|
6af891e358 | ||
|
|
568f55fd2e | ||
|
|
35314175f2 | ||
|
|
ec2e6255a9 | ||
|
|
f3f34a170e | ||
|
|
e2622fd031 | ||
|
|
3ee32970ef | ||
|
|
11b654499b | ||
|
|
d69e19103c | ||
|
|
10f948e970 | ||
|
|
f421d4588c | ||
|
|
74b0dab34c | ||
|
|
41b20b2782 | ||
|
|
912e9f4e85 | ||
|
|
babd7b2f47 | ||
|
|
892dfd1d08 | ||
|
|
cfac579392 | ||
|
|
fe0d295490 | ||
|
|
d8a0238e02 | ||
|
|
dcf710a8d5 | ||
|
|
a2003ab616 | ||
|
|
1363a2fe27 | ||
|
|
f19a2254ac | ||
|
|
7d46e48b26 | ||
|
|
78915f9f7a | ||
|
|
17b8f14fe1 | ||
|
|
42c394acc2 | ||
|
|
b9ad7a2948 | ||
|
|
cde22ff627 | ||
|
|
79993fb498 | ||
|
|
aec55b7072 | ||
|
|
c7884f5a72 | ||
|
|
580c4d66a7 | ||
|
|
63bfc5fda5 | ||
|
|
02e3482eb5 | ||
|
|
294e83dab1 | ||
|
|
b5438bb331 | ||
|
|
c23be73d4d | ||
|
|
d2d171f2d2 | ||
|
|
73fa64ea68 | ||
|
|
1a13dbeb17 | ||
|
|
e11caecdad | ||
|
|
7bb9e55d0b | ||
|
|
00037608ae | ||
|
|
cea59428b1 | ||
|
|
5c832f2d19 | ||
|
|
cdebb7d1a5 | ||
|
|
6a803ea5b3 | ||
|
|
9745c1ab31 | ||
|
|
c0c3c3fed4 | ||
|
|
c1449fffe9 | ||
|
|
f873c97db5 | ||
|
|
b0b3d87fe2 | ||
|
|
ba189e6943 | ||
|
|
cadf74a176 | ||
|
|
74efe8e2e7 | ||
|
|
68af9eff44 | ||
|
|
518eeb7941 | ||
|
|
bf2da878a0 | ||
|
|
50cf253a43 | ||
|
|
980a0dee93 | ||
|
|
ac9c2d57be | ||
|
|
8b016cc62b | ||
|
|
cf2bb3b87d | ||
|
|
473376259b | ||
|
|
e1293fb0ad | ||
|
|
6784efae75 | ||
|
|
d86baebe9a | ||
|
|
82481099b7 | ||
|
|
b127f005c3 | ||
|
|
58b4fcb8c8 | ||
|
|
796f6410c1 | ||
|
|
e67c58cd98 | ||
|
|
85873b19e1 | ||
|
|
2ebbb79937 | ||
|
|
9204190445 | ||
|
|
06ea2168d8 | ||
|
|
2b149dd43e | ||
|
|
17dba2a34c | ||
|
|
11e4bc0ba1 | ||
|
|
e56f3aa1cb | ||
|
|
f966693583 | ||
|
|
5acc763992 | ||
|
|
4f009fff83 | ||
|
|
f27e31954f | ||
|
|
62650cd48c | ||
|
|
408f6dbad9 | ||
|
|
eae0577567 | ||
|
|
de26918c52 | ||
|
|
e01d18e548 | ||
|
|
59166b1031 | ||
|
|
c294949f2a | ||
|
|
28853a251b | ||
|
|
76a8b8df9e | ||
|
|
f86f6a74f0 | ||
|
|
a3e8a4185b | ||
|
|
78de54381b | ||
|
|
ff5ef0ab60 | ||
|
|
a5d6a1187c | ||
|
|
06d0a4bfe8 | ||
|
|
d293a0deaf | ||
|
|
11a44e1ec4 | ||
|
|
28c7f05b39 | ||
|
|
93a0c6202e | ||
|
|
7d82751862 | ||
|
|
905f3363c9 | ||
|
|
9f9d9b3b25 | ||
|
|
bde4b61b3b | ||
|
|
d07018f3c5 | ||
|
|
729a5f9e6c | ||
|
|
773b561f5e | ||
|
|
ca7e81e964 | ||
|
|
08bb56f1d3 | ||
|
|
1a44d281c2 | ||
|
|
1c8fe65601 | ||
|
|
8eb6c3e94a | ||
|
|
9d4d847dc4 | ||
|
|
bea1a1c513 | ||
|
|
b1df688c62 | ||
|
|
21d38c9b0e | ||
|
|
1f86ed4135 | ||
|
|
689dd060b0 | ||
|
|
99c4c4ef04 | ||
|
|
e5e1666f91 | ||
|
|
bd58d1c3ef | ||
|
|
d134c89a7c | ||
|
|
a9307d9f33 | ||
|
|
5eb1c9c2b7 | ||
|
|
37b8edeec6 | ||
|
|
424406b2d6 | ||
|
|
90455c61f6 | ||
|
|
1f6793cf0c | ||
|
|
6ef522fbda |
+8
-2
@@ -1,5 +1,11 @@
|
||||
ARG GO_BASE=golang:1.25-alpine
|
||||
# The control plane's image.
|
||||
# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq
|
||||
# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how
|
||||
# `make image` broke once before (issue 146).
|
||||
ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||
# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go
|
||||
# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it.
|
||||
# Genesis builds this file and raises it as the container the process replaces on the first push
|
||||
# (mesh-host internal/bootstrap, novox/hq issue 223).
|
||||
#
|
||||
# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
|
||||
# machine where no mesh exists yet, and run before there is anything to check it against. So it
|
||||
|
||||
@@ -27,17 +27,21 @@ build:
|
||||
IMAGE ?= mesh-controller:$(VERSION)
|
||||
DEV_TAG ?= mesh-controller:development
|
||||
|
||||
# The base the module declares, read from the manifest rather than written here twice.
|
||||
# The Go base the image is built on.
|
||||
#
|
||||
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
|
||||
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
|
||||
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
|
||||
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
|
||||
# what it cost).
|
||||
GO_BASE ?= $(shell python3 -c "import json;print(next(o['image'] for o in json.load(open('module.json'))['build']['on'] if o['arg']=='GO_BASE'))" 2>/dev/null)
|
||||
#
|
||||
# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds
|
||||
# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab —
|
||||
# still needs a Go base. The digest is the one the manifest declared until then.
|
||||
GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||
|
||||
image:
|
||||
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
|
||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -48,7 +52,7 @@ BUILDER_IMAGE ?= mesh-builder:$(VERSION)
|
||||
BUILDER_DEV_TAG ?= mesh-builder:development
|
||||
|
||||
builder-image:
|
||||
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
|
||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -59,7 +63,7 @@ PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
|
||||
PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development
|
||||
|
||||
provisioner-image:
|
||||
docker build -f examples/postgres-provisioner/Dockerfile \
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/postgres-provisioner/Dockerfile \
|
||||
-t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -70,7 +74,7 @@ OBJECTSTORE_IMAGE ?= mesh-provision-objectstore:$(VERSION)
|
||||
OBJECTSTORE_DEV_TAG ?= mesh-provision-objectstore:development
|
||||
|
||||
objectstore-image:
|
||||
docker build -f examples/objectstore-provisioner/Dockerfile \
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/objectstore-provisioner/Dockerfile \
|
||||
-t $(OBJECTSTORE_IMAGE) -t $(OBJECTSTORE_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(OBJECTSTORE_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -81,7 +85,7 @@ REDIS_PROVISIONER_IMAGE ?= mesh-provision-redis:$(VERSION)
|
||||
REDIS_PROVISIONER_DEV_TAG ?= mesh-provision-redis:development
|
||||
|
||||
redis-provisioner-image:
|
||||
docker build -f examples/redis-provisioner/Dockerfile \
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/redis-provisioner/Dockerfile \
|
||||
-t $(REDIS_PROVISIONER_IMAGE) -t $(REDIS_PROVISIONER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
@@ -91,7 +95,7 @@ PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||
|
||||
proxy-image:
|
||||
docker build -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
|
||||
@@ -193,6 +193,13 @@ passes every check that only looks at the message.
|
||||
|
||||
## The image
|
||||
|
||||
**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go
|
||||
bundle, `controller`, which the host on the controller's machine unpacks and runs as the process
|
||||
`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what
|
||||
still runs a container of the controller: genesis, which raises the first controller from it and
|
||||
installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the
|
||||
mesh's own build any more — `make image` builds it.
|
||||
|
||||
`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
|
||||
manager, no libc, no CA certificates.
|
||||
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
|
||||
//
|
||||
// **The queue is the controller's; the build running here is this machine's.** The controller can
|
||||
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
|
||||
// on this machine and containers in this machine's runtime, and only this machine can end them. So
|
||||
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
|
||||
// it, pause, resume.
|
||||
//
|
||||
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
|
||||
// set per build — so "the build running here" is one or none, and kill names it by id so a call
|
||||
// that arrives as one build ends and the next begins cannot end the wrong one.
|
||||
|
||||
// holder is this machine's state as a holder of the build seat.
|
||||
type holder struct {
|
||||
on, seat string
|
||||
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
|
||||
// rather than silently taking work again.
|
||||
workspace string
|
||||
// say publishes this machine's state: whether it takes work (link.HolderState).
|
||||
say func(link.HolderState) error
|
||||
// remove runs what a kill needs outside the build: the containers left behind.
|
||||
remove builder.Runner
|
||||
|
||||
mu sync.Mutex
|
||||
paused bool
|
||||
running *running
|
||||
}
|
||||
|
||||
// running is the build this machine is doing.
|
||||
type running struct {
|
||||
request link.BuildRequest
|
||||
step string
|
||||
started time.Time
|
||||
cancel context.CancelFunc
|
||||
killed bool
|
||||
// returned is the build's own work having ended, before a kill or not; outcome is what was then
|
||||
// announced, and announced whether it went out.
|
||||
returned bool
|
||||
outcome string
|
||||
announced bool
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// pausedFile is where the flag lives in the workspace.
|
||||
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
|
||||
|
||||
// newHolder reads the paused flag the workspace keeps.
|
||||
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
|
||||
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
|
||||
if _, err := os.Stat(pausedFile(workspace)); err == nil {
|
||||
h.paused = true
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Paused is asked by the taking loop before every fetch.
|
||||
func (h *holder) Paused() bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.paused
|
||||
}
|
||||
|
||||
// setPaused records the flag in the workspace first and then in memory, so what this holder says
|
||||
// it is and what it would be after a restart never differ.
|
||||
func (h *holder) setPaused(paused bool) error {
|
||||
path := pausedFile(h.workspace)
|
||||
if paused {
|
||||
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
|
||||
}
|
||||
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.paused = paused
|
||||
h.mu.Unlock()
|
||||
h.announce()
|
||||
return nil
|
||||
}
|
||||
|
||||
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
|
||||
// controller reading an older state is a plan read as late rather than a build lost.
|
||||
func (h *holder) announce() {
|
||||
if h.say == nil {
|
||||
return
|
||||
}
|
||||
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
|
||||
// a pause outlives the events stream's retention.
|
||||
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||
h.announce()
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if h.Paused() {
|
||||
h.announce()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// begin records the build this machine took, with the cancel that ends it.
|
||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||
h.mu.Lock()
|
||||
h.running = r
|
||||
h.mu.Unlock()
|
||||
return r
|
||||
}
|
||||
|
||||
// end clears it, and lets a kill waiting on it know it is over.
|
||||
func (h *holder) end(r *running) {
|
||||
h.mu.Lock()
|
||||
if h.running == r {
|
||||
h.running = nil
|
||||
}
|
||||
h.mu.Unlock()
|
||||
close(r.done)
|
||||
}
|
||||
|
||||
// stepped records the step a build is at, for `current`.
|
||||
func (h *holder) stepped(r *running, step string) {
|
||||
h.mu.Lock()
|
||||
r.step = step
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// currentBuild is what `current` answers.
|
||||
type currentBuild struct {
|
||||
On string `json:"on"`
|
||||
Paused bool `json:"paused"`
|
||||
Running *struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
} `json:"running,omitempty"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
func (h *holder) current() currentBuild {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out := currentBuild{On: h.on, Paused: h.paused}
|
||||
taking := "taking builds"
|
||||
if h.paused {
|
||||
taking = "paused, taking no new build"
|
||||
}
|
||||
if h.running == nil {
|
||||
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
|
||||
return out
|
||||
}
|
||||
r := h.running
|
||||
elapsed := time.Since(r.started).Round(time.Second)
|
||||
out.Running = &struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
|
||||
r.started.UTC().Format(time.RFC3339), elapsed.String()}
|
||||
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
|
||||
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
|
||||
return out
|
||||
}
|
||||
|
||||
// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
|
||||
// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
|
||||
// (killAnswer in the controller's queue.go, 75s).
|
||||
var (
|
||||
killRemoves = 15 * time.Second
|
||||
killWaits = 20 * time.Second
|
||||
)
|
||||
|
||||
// kill ends the build with this id, if it is the one running here and still working: its context
|
||||
// cancelled — which kills each command's process group — and the containers it started removed by
|
||||
// their label, once at once and again after the build has ended, so one created while it was being
|
||||
// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
|
||||
// ask so it is not redelivered; the answer says whether that happened.
|
||||
func (h *holder) kill(id string) (string, error) {
|
||||
h.mu.Lock()
|
||||
r := h.running
|
||||
if r == nil || r.request.ID != id {
|
||||
doing := "nothing"
|
||||
if r != nil {
|
||||
doing = r.request.ID
|
||||
}
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
|
||||
}
|
||||
if r.returned {
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
|
||||
}
|
||||
r.killed = true
|
||||
cancel, done := r.cancel, r.done
|
||||
h.mu.Unlock()
|
||||
|
||||
cancel()
|
||||
removed, removeErr := h.removeContainers(id)
|
||||
ended := false
|
||||
select {
|
||||
case <-done:
|
||||
ended = true
|
||||
case <-time.After(killWaits):
|
||||
}
|
||||
again, againErr := h.removeContainers(id)
|
||||
removed += again
|
||||
if removeErr == nil {
|
||||
removeErr = againErr
|
||||
}
|
||||
containers := fmt.Sprintf("%d container(s) it started removed", removed)
|
||||
if removeErr != nil {
|
||||
containers = "its containers could not all be listed or removed: " + removeErr.Error()
|
||||
}
|
||||
if !ended {
|
||||
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
|
||||
"its outcome is in", id, h.on, containers), nil
|
||||
}
|
||||
h.mu.Lock()
|
||||
outcome, announced := r.outcome, r.announced
|
||||
h.mu.Unlock()
|
||||
if !announced {
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
|
||||
"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
|
||||
containers), nil
|
||||
}
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
|
||||
"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
|
||||
}
|
||||
|
||||
// removeContainers is one pass of removing what the build left, bounded.
|
||||
func (h *holder) removeContainers(id string) (int, error) {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
||||
defer stop()
|
||||
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||
}
|
||||
|
||||
// returned records that the build's work ended, and says whether a kill came first — only then is
|
||||
// the build killed; an error it ended with on its own is its own outcome.
|
||||
func (h *holder) returned(r *running) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
r.returned = true
|
||||
return r.killed
|
||||
}
|
||||
|
||||
// said records the outcome announced, and whether it went out, for a kill to answer with.
|
||||
func (h *holder) said(r *running, outcome string, announced bool) {
|
||||
h.mu.Lock()
|
||||
r.outcome, r.announced = outcome, announced
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// handlers are the seat's verbs, as this machine answers them.
|
||||
func (h *holder) handlers() map[string]link.ToolHandler {
|
||||
return map[string]link.ToolHandler{
|
||||
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
|
||||
return nil, errors.New("kill needs the build's id")
|
||||
}
|
||||
said, err := h.kill(strings.TrimSpace(args.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": said}, nil
|
||||
},
|
||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
said := h.on + " is paused: it takes no new build until resumed"
|
||||
if c := h.current(); c.Running != nil {
|
||||
said += "; " + c.Running.ID + " runs on and finishes"
|
||||
}
|
||||
return map[string]any{"said": said, "paused": true}, nil
|
||||
},
|
||||
"resume": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func orNothing(s string) string {
|
||||
if s == "" {
|
||||
return "its start"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
|
||||
// log it would be is the one being ended.
|
||||
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
|
||||
// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
|
||||
// the module answering, the seat, scope node, the machine, its description and argument schema — so
|
||||
// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
|
||||
//
|
||||
// One service per machine, named for the seat and identified by the machine, so the answer is this
|
||||
// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
|
||||
// store, and what it serves is what this binary was built to serve.
|
||||
func announcement(seat, module, node string) micro.Info {
|
||||
s, _ := catalogue.SeatNamed(seat)
|
||||
var endpoints []micro.EndpointInfo
|
||||
for _, v := range s.Serves {
|
||||
schema, _ := json.Marshal(v.Input)
|
||||
endpoints = append(endpoints, micro.EndpointInfo{
|
||||
Name: seat + "__" + v.Name,
|
||||
Subject: link.NodeSeatToolSubject(seat, v.Name, node),
|
||||
// The queue group the verbs are served in, as every runtime announces its own.
|
||||
QueueGroup: "seat." + seat,
|
||||
Metadata: map[string]string{
|
||||
"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
|
||||
"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
|
||||
},
|
||||
})
|
||||
}
|
||||
return micro.Info{
|
||||
ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
|
||||
Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
|
||||
Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
}
|
||||
|
||||
// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
|
||||
func moduleOf(user string) string {
|
||||
if _, module, ok := strings.Cut(user, "."); ok && module != "" {
|
||||
return module
|
||||
}
|
||||
return "build-agent"
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
||||
// 0219), and what it says about itself follows.
|
||||
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
var said []link.HolderState
|
||||
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
||||
said = append(said, s)
|
||||
return nil
|
||||
})
|
||||
if h.Paused() {
|
||||
t.Fatal("a new holder starts paused")
|
||||
}
|
||||
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
||||
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
||||
}
|
||||
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
||||
if !again.Paused() {
|
||||
t.Fatal("restarted, the holder forgot it was paused")
|
||||
}
|
||||
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
||||
t.Fatal("resumed, the holder came back paused")
|
||||
}
|
||||
}
|
||||
|
||||
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
||||
// label, and refuses an id it is not building.
|
||||
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
var removed []string
|
||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "ps" {
|
||||
return "c1\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
kill := h.handlers()["kill"]
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
||||
t.Fatal("killed a build while none ran")
|
||||
}
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
||||
h.stepped(r, "image")
|
||||
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
||||
t.Fatalf("current says %+v", c)
|
||||
}
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "build-1") {
|
||||
t.Fatalf("killed another id: %v", err)
|
||||
}
|
||||
// The build's own goroutine: it ends when its context does, as a build's commands do, and
|
||||
// announces what came of it.
|
||||
go func() {
|
||||
<-building.Done()
|
||||
if h.returned(r) {
|
||||
h.said(r, link.KilledByHand, true)
|
||||
}
|
||||
h.end(r)
|
||||
}()
|
||||
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if building.Err() == nil {
|
||||
t.Fatal("the build's context was not cancelled")
|
||||
}
|
||||
if !r.killed {
|
||||
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
||||
}
|
||||
said := answer.(map[string]any)["said"].(string)
|
||||
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
|
||||
t.Errorf("kill said %q", said)
|
||||
}
|
||||
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
||||
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
||||
t.Errorf("removed %v", removed)
|
||||
}
|
||||
if c := h.current(); c.Running != nil {
|
||||
t.Errorf("after the kill current says %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
|
||||
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
|
||||
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
|
||||
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
|
||||
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
|
||||
}
|
||||
kill := info.Endpoints[1]
|
||||
md := kill.Metadata
|
||||
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
|
||||
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
|
||||
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
|
||||
t.Fatalf("kill is announced as %+v", kill)
|
||||
}
|
||||
body, err := json.Marshal(info)
|
||||
if err != nil || len(body) > 8*1024 {
|
||||
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
|
||||
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
|
||||
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
|
||||
_, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
|
||||
if killed := h.returned(r); killed {
|
||||
t.Fatal("a build nobody killed reads as killed")
|
||||
}
|
||||
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
|
||||
t.Fatalf("killed a build that had ended: %v", err)
|
||||
}
|
||||
h.end(r)
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
|
||||
go func() {
|
||||
<-building.Done()
|
||||
if h.returned(r) {
|
||||
h.said(r, link.KilledByHand, false)
|
||||
}
|
||||
h.end(r)
|
||||
}()
|
||||
said, err := h.kill("build-2")
|
||||
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
|
||||
t.Fatalf("kill said %q (%v)", said, err)
|
||||
}
|
||||
}
|
||||
+112
-12
@@ -19,11 +19,13 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
@@ -107,43 +109,96 @@ func run() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
machine, err := takeWorkFrom(credential, on)
|
||||
js, seat, err := dialFor(credential)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
|
||||
// paused flag is read from the workspace before anything is taken, so a holder restarted while
|
||||
// paused takes nothing.
|
||||
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
|
||||
body, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
|
||||
return err
|
||||
})
|
||||
if h.Paused() {
|
||||
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||
}
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
// one serves none, and a subscription its holder has no grant for would be refused for ever.
|
||||
if seat == link.TheBuildMachine {
|
||||
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
|
||||
log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopServing()
|
||||
// And says so, for the console to find (novox/hq ADR 0197).
|
||||
stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
|
||||
announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopAnnouncing()
|
||||
go h.stateWhilePaused(ctx, time.Hour)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
answer(ctx, publisher, on, workspace, work)
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
|
||||
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
|
||||
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
|
||||
// holds.
|
||||
//
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||
// machine told about both would take work from one and answer on the other, and every log line would
|
||||
// say it was fine.
|
||||
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
|
||||
func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
||||
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
||||
// and that is enough to dial it, pinned.
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
return link.MachineOverNATS(js, on), nil
|
||||
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
||||
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
||||
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
||||
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
||||
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
||||
return js, seat, nil
|
||||
}
|
||||
|
||||
// answer does one build and says what happened, whichever way it went.
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
|
||||
request := work.Request()
|
||||
|
||||
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
|
||||
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
|
||||
// reaches it through the parent, and that keeps today's meaning below.
|
||||
building, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
var mine *running
|
||||
if h != nil {
|
||||
mine = h.begin(request, cancel)
|
||||
defer h.end(mine)
|
||||
}
|
||||
|
||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||
@@ -157,6 +212,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
say := func(step, message string) {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
h.stepped(mine, step)
|
||||
}
|
||||
}
|
||||
builder.Said = say
|
||||
defer func() { builder.Said = nil }()
|
||||
@@ -166,7 +224,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
|
||||
result := link.BuildResult{
|
||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||
Ref: request.Ref, On: on, Source: request.Source,
|
||||
Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
|
||||
}
|
||||
what := "building " + request.Repository
|
||||
if request.Path != "" {
|
||||
@@ -183,11 +241,24 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||
// after a clone that then fails at npm ci.
|
||||
built, err = builder.Build(ctx, builder.Command, publisher,
|
||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||
// docker client (ADR 0219).
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
if err != nil {
|
||||
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
||||
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
||||
killed := false
|
||||
if mine != nil {
|
||||
killed = h.returned(mine) && err != nil
|
||||
}
|
||||
if killed {
|
||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||
// error it ended with is the kill's consequence, not a fault of the source.
|
||||
result.Failed = link.KilledByHand
|
||||
say("failed", link.KilledByHand)
|
||||
} else if err != nil {
|
||||
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
||||
// builder that is not running, and those want completely different responses.
|
||||
result.Failed = err.Error()
|
||||
@@ -212,7 +283,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
}
|
||||
|
||||
if err := work.Announce(ctx, result); err != nil {
|
||||
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
|
||||
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
|
||||
// to be built again. A machine being stopped is the other case and keeps its meaning: the
|
||||
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
|
||||
announcing := ctx
|
||||
if killed {
|
||||
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
announcing = fresh
|
||||
}
|
||||
announceErr := work.Announce(announcing, result)
|
||||
if mine != nil {
|
||||
h.said(mine, result.Failed, announceErr == nil)
|
||||
}
|
||||
if err := announceErr; err != nil {
|
||||
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||
// nothing was said at all, so another machine can try.
|
||||
@@ -453,6 +538,21 @@ type Credential struct {
|
||||
// as two fields and this machine joins them once, here, to dial.
|
||||
User string `json:"user,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
// Claims are the seats the module this credential was issued for claims, as the mesh writes
|
||||
// them beside the credential (novox/hq ADR 0159). The first is the build role this machine
|
||||
// serves; a credential naming none is from before claims travelled in it.
|
||||
Claims []struct {
|
||||
Seat string `json:"seat"`
|
||||
} `json:"claims,omitempty"`
|
||||
}
|
||||
|
||||
// seatsClaimed is the seats the credential names, in order.
|
||||
func (c Credential) seatsClaimed() []string {
|
||||
out := make([]string, 0, len(c.Claims))
|
||||
for _, claim := range c.Claims {
|
||||
out = append(out, claim.Seat)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover).
|
||||
func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) {
|
||||
var held Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x",
|
||||
"claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" {
|
||||
t.Errorf("the old builder's credential serves %q", got)
|
||||
}
|
||||
var bare Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine {
|
||||
t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine)
|
||||
}
|
||||
}
|
||||
+183
-20
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -38,7 +40,10 @@ import (
|
||||
//
|
||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||
// machines this just blocked is worth more than the milliseconds.
|
||||
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("assign %s names no module", node)
|
||||
}
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
@@ -46,6 +51,16 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
||||
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
||||
// resources are applied through a seat nothing on the node holds is refused, because that order
|
||||
// — the service manager, the package manager and the runtime before anything that installs,
|
||||
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
|
||||
// holders that depend on each other go on in one command.
|
||||
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
||||
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
||||
// assignment resolves against a record rather than against a coincidence.
|
||||
@@ -53,58 +68,176 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
var lines []string
|
||||
var added []string
|
||||
for _, module := range modules {
|
||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||
if err != nil {
|
||||
return strings.Join(lines, "\n"), err
|
||||
}
|
||||
if !fresh {
|
||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||
lines = append(lines, fmt.Sprintf(
|
||||
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
|
||||
continue
|
||||
}
|
||||
added = append(added, module)
|
||||
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
|
||||
}
|
||||
if !fresh {
|
||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||
return fmt.Sprintf("%s already runs %s — one node runs one of each (ADR 0115); nothing changed",
|
||||
node, module), nil
|
||||
if len(added) == 0 {
|
||||
return strings.Join(lines, "\n"), nil
|
||||
}
|
||||
said := fmt.Sprintf("%s is assigned %s", node, module)
|
||||
answer := strings.Join(lines, "\n")
|
||||
for _, line := range settled {
|
||||
said += "\n " + line
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||
// node's list, and every other node's, is `status`'s.
|
||||
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||
for _, module := range added {
|
||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||
answer += "\n " + line
|
||||
}
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||
return said + blockedElsewhere(ctx, open, node), err
|
||||
return answer + blockedElsewhere(ctx, open, node), err
|
||||
}
|
||||
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
||||
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
||||
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
||||
isAdded := map[string]bool{}
|
||||
for _, m := range added {
|
||||
isAdded[m] = true
|
||||
}
|
||||
for _, u := range plan.Unhostable {
|
||||
if u.Module != module {
|
||||
if !isAdded[u.Module] {
|
||||
continue
|
||||
}
|
||||
for _, c := range u.Missing {
|
||||
said += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||
}
|
||||
}
|
||||
return said + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||
blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// unassign takes a module off a node. What it leaves behind is the host's business: a directory
|
||||
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
||||
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
|
||||
// and are not judged again.
|
||||
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
|
||||
map[string]catalogue.Manifest, []string, error) {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
already := map[string]bool{}
|
||||
for _, a := range assigned {
|
||||
already[a] = true
|
||||
}
|
||||
var adding []string
|
||||
for _, m := range modules {
|
||||
if !already[m] {
|
||||
adding = append(adding, m)
|
||||
}
|
||||
}
|
||||
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
|
||||
// with everything else this act changed, so they are not said twice.
|
||||
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Two modules declaring one package, path or unit is refused before anything is recorded
|
||||
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
||||
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return shelf, assigned, nil
|
||||
}
|
||||
|
||||
// unassign takes modules off a node. What they leave behind is the host's business: a directory
|
||||
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
||||
//
|
||||
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
||||
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
||||
// about the command's own output would ever have said so.
|
||||
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
//
|
||||
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
|
||||
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, module, node) + blockedElsewhere(ctx, open, node), nil
|
||||
assigned, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Every one checked before any is taken off, so a refusal leaves the node as it was.
|
||||
runs := map[string]bool{}
|
||||
for _, a := range assigned {
|
||||
runs[a] = true
|
||||
}
|
||||
for _, module := range modules {
|
||||
if !runs[module] {
|
||||
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
||||
}
|
||||
}
|
||||
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, module := range modules {
|
||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, strings.Join(modules, ", "), node)
|
||||
var left []string
|
||||
for _, a := range assigned {
|
||||
if !slices.Contains(modules, a) {
|
||||
left = append(left, a)
|
||||
}
|
||||
}
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||
// command API and the controller seat's verbs as they do from the command line.
|
||||
func splitModules(field string) []string {
|
||||
var out []string
|
||||
for _, m := range strings.Split(field, ",") {
|
||||
if m = strings.TrimSpace(m); m != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
||||
@@ -152,3 +285,33 @@ func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
||||
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
||||
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
|
||||
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
|
||||
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
|
||||
// module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
m, known := shelf[module]
|
||||
if !known || mayIssue(m) != nil {
|
||||
return ""
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||
return ""
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err == nil {
|
||||
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
|
||||
"`push %s` refuses to send %s until it is", err, module, node, node, module)
|
||||
}
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
@@ -145,7 +145,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// container is told so beside the sealed connection genesis wrote.
|
||||
// process is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -157,8 +157,8 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
|
||||
Reference: "registry.example/control@" + aDigest}})
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -175,6 +175,12 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -194,12 +200,12 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.server" {
|
||||
if r["id"] == "mesh-controller.controller" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's container is not in its own node's declaration")
|
||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
@@ -232,7 +238,7 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
out := m
|
||||
out.Resources = nil
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
if r["type"] != "container" && r["type"] != "process" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -94,6 +94,8 @@ func showFiltering(f inventory.Filtering, adopted bool) {
|
||||
switch {
|
||||
case fw.Active:
|
||||
fmt.Printf(" found firewall %s is ACTIVE on this converged machine; the next apply retires it again\n", fw.Kind)
|
||||
case fw.RetiredBy == "removed":
|
||||
fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0180)\n", fw.Kind)
|
||||
case fw.RetiredBy == inventory.FilterMesh || fw.RetiredBy == "mesh":
|
||||
fmt.Printf(" found firewall %s, retired by the mesh; its configuration stays on disk\n", fw.Kind)
|
||||
case fw.RetiredBy != "":
|
||||
|
||||
@@ -95,10 +95,10 @@ func commands(who Authenticator) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return assign(ctx, open, in.Node, in.Module)
|
||||
return assign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||
}))
|
||||
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return unassign(ctx, open, in.Node, in.Module)
|
||||
return unassign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||
}))
|
||||
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
||||
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
|
||||
+120
-18
@@ -148,6 +148,11 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
// rebuild the graph rather than a list of names.
|
||||
Path: result.Path,
|
||||
}
|
||||
// When it was asked, which is what orders it against another build of the same module
|
||||
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
||||
if asked, ok := link.BuildAskedAt(result.ID); ok {
|
||||
kept.Asked = asked
|
||||
}
|
||||
for _, ref := range result.Against {
|
||||
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
||||
}
|
||||
@@ -387,34 +392,47 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
//
|
||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
||||
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
||||
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
||||
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
||||
dryRun bool) (string, error) {
|
||||
if dryRun && wait != 0 {
|
||||
return "", errors.New("a dry run waited for is `build --dry-run`")
|
||||
}
|
||||
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
||||
// the reason, rather than sent to a machine to fail at `git clone`.
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
// Correlated by something the control plane makes, not by the module's name: two builds of one
|
||||
// module can be in flight, and the second answer is not the first one's.
|
||||
request := link.BuildRequest{
|
||||
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Path: path,
|
||||
Ref: ref,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
DryRun: dryRun,
|
||||
}
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
@@ -430,11 +448,13 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
ask, err := askOver(server)
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
fmt.Printf(" of %s\n", seat)
|
||||
|
||||
if wait == 0 {
|
||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||
@@ -442,26 +462,31 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
||||
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
|
||||
// follows the build by its id instead.
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
||||
"its outcome is not taken in\n", request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
|
||||
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
|
||||
return nil
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
result, err := ask.Submit(ctx, request, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
return request.ID, err
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return request.ID, err
|
||||
}
|
||||
defer open.Close()
|
||||
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
||||
if err != nil {
|
||||
return err
|
||||
return request.ID, err
|
||||
}
|
||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||
for _, made := range kept.Made {
|
||||
@@ -471,7 +496,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
||||
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
|
||||
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
||||
return nil
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
|
||||
@@ -522,17 +547,39 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
recorded := inventory.Source{
|
||||
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
|
||||
BuiltFrom: result.Commit, Head: result.Commit,
|
||||
// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
|
||||
Against: kept.Against,
|
||||
// When it was asked, so an older request heard later does not replace a newer one
|
||||
// (novox/hq 04-ISSUES/219).
|
||||
Asked: kept.Asked,
|
||||
}
|
||||
if result.Source != nil && result.Source.Seat != "" {
|
||||
recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
|
||||
}
|
||||
// **A build at a commit does not change the branch a module follows** (novox/hq 04-ISSUES/215):
|
||||
// the commit is built and recorded as what it was built from, and the module keeps following
|
||||
// what it followed before — the repository's default branch for one new to the catalogue.
|
||||
if followedBranch(result.Ref) == "" && result.Ref != "" {
|
||||
recorded.Ref = ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
recorded.Ref = followedBranch(was.Ref)
|
||||
}
|
||||
}
|
||||
if err := namesNoInstallation(manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
result.On, result.Repository, short(result.Commit), err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||
if errors.Is(err, inventory.ErrSuperseded) {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
|
||||
result.On, manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||
// worked and the module is registered.
|
||||
collect(ctx, inv)
|
||||
return manifest, kept, nil
|
||||
}
|
||||
|
||||
@@ -553,16 +600,17 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
ask, err := askOver(server)
|
||||
ask, err := askOverOn(buildSeatHeld(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
|
||||
result, err := ask.Submit(ctx, link.BuildRequest{
|
||||
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository, Path: path, Ref: ref,
|
||||
Held: heldBy(ctx), Seats: seatBases(ctx),
|
||||
DryRun: true,
|
||||
}, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -598,6 +646,8 @@ type answers struct {
|
||||
reported []inventory.Reported
|
||||
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
||||
plans []inventory.Plan
|
||||
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||
paused pauseView
|
||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||
// other answer here: those are about a machine that was told something, and this is about one
|
||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||
@@ -621,6 +671,11 @@ type answers struct {
|
||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||
untaken map[string]map[string]int
|
||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||
unheld []catalogue.Unheld
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
@@ -666,12 +721,56 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOver(_ *link.Server) (link.Builders, error) {
|
||||
func askOverOn(seat string) (link.Builders, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.BuildsOverNATS(address)
|
||||
return link.BuildsOverNATSOn(address, seat)
|
||||
}
|
||||
|
||||
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
|
||||
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
|
||||
// moment the first build-agent is assigned — and a controller that asked the new role before then
|
||||
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
|
||||
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
|
||||
func buildSeatHeld(ctx context.Context) string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
return buildSeatAmong(entries)
|
||||
}
|
||||
|
||||
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
|
||||
// assigned module claims it; else the retired role while an assigned module still claims that; else
|
||||
// the current role, which is where every ask goes once the handover is done.
|
||||
func buildSeatAmong(entries []inventory.Entry) string {
|
||||
heldBefore := false
|
||||
for _, e := range entries {
|
||||
if len(e.On) == 0 {
|
||||
continue
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
|
||||
heldBefore = true
|
||||
}
|
||||
}
|
||||
if heldBefore {
|
||||
return link.TheBuildMachineBefore
|
||||
}
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
|
||||
// buildLog prints everything a build machine said about one build, read back from the bus.
|
||||
@@ -691,10 +790,13 @@ func buildLog(ctx context.Context, id string) error {
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
sub, err := js.Context().PullSubscribe(link.BuildLog(id), "",
|
||||
// Under whichever build role did it: a build asked of the retired role during the handover
|
||||
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
|
||||
lines := link.BuildLogOf("*", id)
|
||||
sub, err := js.Context().PullSubscribe(lines, "",
|
||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read %s from the bus: %w", link.BuildLog(id), err)
|
||||
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func claiming(module, seat string, on ...string) inventory.Entry {
|
||||
return inventory.Entry{
|
||||
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
|
||||
On: on,
|
||||
}
|
||||
}
|
||||
|
||||
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
|
||||
// one while only the builder is assigned, the current one from the first build-agent on, and the
|
||||
// current one when nothing holds either — where every ask goes once the handover is done.
|
||||
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
|
||||
onlyTheBuilder := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
|
||||
}
|
||||
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
|
||||
t.Errorf("with only the builder assigned, asked %q", got)
|
||||
}
|
||||
bothHeld := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine, "home-server"),
|
||||
}
|
||||
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
|
||||
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
|
||||
}
|
||||
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
|
||||
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
|
||||
t.Errorf("with no holder of either, asked %q, want the current role", got)
|
||||
}
|
||||
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
|
||||
t.Errorf("an empty catalogue asks %q", got)
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -63,3 +66,87 @@ func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the
|
||||
// module keeps following the branch it followed — a new one, the default branch.
|
||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||
result := func(id, ref, commit string) link.BuildResult {
|
||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-1", "main", "1111111aaaa")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||
}
|
||||
|
||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||
r.Manifest, r.Path = other, "modules/letta"
|
||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered,
|
||||
// so a push sends what the newer request built.
|
||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
result := func(asked time.Time, image string) link.BuildResult {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result(newer, "4bcd5f73")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||
t.Errorf("the late build was not recorded: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||
at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC)
|
||||
if got, ok := link.BuildAskedAt(link.NewBuildID(at)); !ok || !got.Equal(at) {
|
||||
t.Errorf("read back %v %v; want %v", got, ok, at)
|
||||
}
|
||||
if got, ok := link.BuildAskedAt("build-1791064317392539762"); !ok || got.Format(time.TimeOnly) != "21:51:57" {
|
||||
t.Errorf("the incident's id reads as %v %v", got, ok)
|
||||
}
|
||||
for _, id := range []string{"b-1", "build-2", "build-", "build-x", ""} {
|
||||
if _, ok := link.BuildAskedAt(id); ok {
|
||||
t.Errorf("%q read as a request time", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
@@ -90,6 +91,17 @@ func moduleCheck(paths []string, out io.Writer) error {
|
||||
if len(m.Invokes) > 0 {
|
||||
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
||||
}
|
||||
// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
|
||||
if len(m.State) > 0 {
|
||||
kept := make([]string, 0, len(m.State))
|
||||
for _, s := range m.State {
|
||||
kept = append(kept, s.Name)
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||
}
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if failed > 0 {
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Letting the artifact store go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||
//
|
||||
// **Run where the records change.** A build is the moment new bytes landed in the store and the
|
||||
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
||||
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
||||
//
|
||||
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
||||
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
||||
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
||||
// having failed.
|
||||
|
||||
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
||||
// what it let go of. Says what it did and what it could not; returns nothing, because nothing
|
||||
// upstream should branch on it.
|
||||
func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||
references, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
|
||||
return
|
||||
}
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
|
||||
return
|
||||
}
|
||||
if len(references) == 0 && len(kept) == 0 {
|
||||
return
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue to find the artifact store: %v\n", err)
|
||||
return
|
||||
}
|
||||
// As the mesh reaches it from the network. Empty means the store is not on the network — on a
|
||||
// mesh being raised it is not yet, and there the store holds one build of anything and has
|
||||
// nothing to collect.
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil || address == "" {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not find the artifact store to collect from: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
|
||||
// never collected has the whole history to get through, and a person waiting on `build` should
|
||||
// not pay for it. Two bounds, and what is left over is simply offered again next time —
|
||||
// builds are frequent, and the point is that the store stops growing, not that it empties
|
||||
// tonight.
|
||||
within, stop := context.WithTimeout(ctx, sweepBudget)
|
||||
defer stop()
|
||||
store := artifacts.Store{Address: address}
|
||||
|
||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||
wrote, missing, err := holdKept(within, store, kept)
|
||||
if wrote > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
|
||||
}
|
||||
if missing > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||
"`collection` lists them\n", missing)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
var done []string
|
||||
var left, skipped int
|
||||
for i, reference := range references {
|
||||
if i >= mostPerSweep || within.Err() != nil {
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
err := store.LetGo(within, reference)
|
||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||
// again for ever.
|
||||
done = append(done, reference)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||
// reason to stop, because the store was never asked. One of these at the front of
|
||||
// the oldest-first order ended every sweep until this.
|
||||
skipped++
|
||||
if skipped == 1 {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"the sweep will not address %s and went on: %v\n", reference, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||
// front of whoever was building something.
|
||||
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
|
||||
reference, err)
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
|
||||
if len(done) > 0 {
|
||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||
// the sweep ran out of budget would mean asking about them again for ever.
|
||||
if err := inv.MarkCollected(ctx, done); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
|
||||
len(done), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
|
||||
len(done))
|
||||
}
|
||||
if left > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
|
||||
}
|
||||
}
|
||||
|
||||
// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
|
||||
// Answers how many holders it wrote and how many kept archives the store does not have.
|
||||
//
|
||||
// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
|
||||
// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
|
||||
// reference the store cannot be asked about is skipped as the deletion loop skips one
|
||||
// (novox/hq issue 226).
|
||||
func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
|
||||
for _, reference := range kept {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
|
||||
}
|
||||
did, err := store.Hold(ctx, reference)
|
||||
switch {
|
||||
case err == nil:
|
||||
if did {
|
||||
wrote++
|
||||
}
|
||||
case errors.Is(err, artifacts.Gone):
|
||||
missing++
|
||||
case errors.Is(err, artifacts.ErrNotOurs):
|
||||
default:
|
||||
return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
|
||||
}
|
||||
}
|
||||
return wrote, missing, nil
|
||||
}
|
||||
|
||||
// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
|
||||
// several times a day converges within days of this landing; small enough that no single build
|
||||
// waits on the whole backlog.
|
||||
const mostPerSweep = 200
|
||||
|
||||
// sweepBudget is the longest a sweep will keep a build waiting.
|
||||
const sweepBudget = 60 * time.Second
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
)
|
||||
|
||||
// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
|
||||
// (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The question to answer before the store's collector runs for real.** The collector deletes
|
||||
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
|
||||
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
|
||||
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
|
||||
// dry run go.
|
||||
//
|
||||
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
|
||||
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
|
||||
// new verb in the seat's row.
|
||||
|
||||
type collectionReport struct {
|
||||
// Store is the artifact store as this machine reached it; empty when it is not on the network.
|
||||
Store string `json:"store"`
|
||||
// KeptArchives is how many archives the mesh keeps, for either reason.
|
||||
KeptArchives int `json:"kept_archives"`
|
||||
// Held is how many of them the store holds by their manifest.
|
||||
Held int `json:"held"`
|
||||
// Unheld are the kept archives the collector would delete tonight if it ran for real.
|
||||
Unheld []string `json:"unheld"`
|
||||
// Missing are kept archives the store does not have at all.
|
||||
Missing []string `json:"missing"`
|
||||
// Unasked is how many could not be asked about, and why the asking stopped.
|
||||
Unasked int `json:"unasked"`
|
||||
Stopped string `json:"stopped,omitempty"`
|
||||
// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
|
||||
// collected — split by kind.
|
||||
Eligible int `json:"eligible"`
|
||||
EligibleImages int `json:"eligible_images"`
|
||||
EligibleArchives int `json:"eligible_archives"`
|
||||
// SafeToCollect is whether every kept archive was asked about and every one is held.
|
||||
SafeToCollect bool `json:"safe_to_collect"`
|
||||
}
|
||||
|
||||
func collectionCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("collection", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("collection [--json]")
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eligible, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
|
||||
for _, reference := range eligible {
|
||||
if strings.Contains(reference, "/blobs/") {
|
||||
report.EligibleArchives++
|
||||
} else {
|
||||
report.EligibleImages++
|
||||
}
|
||||
}
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if report.Store == "" {
|
||||
report.Unasked = len(kept)
|
||||
report.Stopped = "this mesh has no artifact store on its network"
|
||||
} else {
|
||||
report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
|
||||
}
|
||||
report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
|
||||
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(report)
|
||||
}
|
||||
printCollection(report)
|
||||
return nil
|
||||
}
|
||||
|
||||
// askHeld asks the store about each kept archive, stopping at the first answer that is not about
|
||||
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
|
||||
// identical failures says less than one line.
|
||||
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
|
||||
for i, reference := range kept {
|
||||
held, err := store.Held(ctx, reference)
|
||||
switch {
|
||||
case err == nil && held:
|
||||
report.Held++
|
||||
case err == nil:
|
||||
report.Unheld = append(report.Unheld, reference)
|
||||
case errors.Is(err, artifacts.Gone):
|
||||
report.Missing = append(report.Missing, reference)
|
||||
case errors.Is(err, artifacts.ErrNotOurs):
|
||||
// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
|
||||
report.Unasked++
|
||||
default:
|
||||
return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
|
||||
}
|
||||
}
|
||||
return report.Unasked, ""
|
||||
}
|
||||
|
||||
func printCollection(r collectionReport) {
|
||||
store := r.Store
|
||||
if store == "" {
|
||||
store = "(not on the network)"
|
||||
}
|
||||
fmt.Printf("artifact store %s\n", store)
|
||||
fmt.Printf("kept archives %d\n", r.KeptArchives)
|
||||
fmt.Printf(" held %d\n", r.Held)
|
||||
fmt.Printf(" unheld %d\n", len(r.Unheld))
|
||||
fmt.Printf(" missing %d\n", len(r.Missing))
|
||||
if r.Unasked > 0 {
|
||||
fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped)
|
||||
}
|
||||
fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
|
||||
if len(r.Unheld) > 0 {
|
||||
fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
|
||||
for _, reference := range r.Unheld {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
}
|
||||
if len(r.Missing) > 0 {
|
||||
fmt.Println("\nmissing — kept by the mesh, not in the store:")
|
||||
for _, reference := range r.Missing {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
if r.SafeToCollect {
|
||||
fmt.Println("every kept archive is held: the store's collector may run for real")
|
||||
} else {
|
||||
fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A fresh assignment is pushed before its credential exists (novox/hq issue 203): `assign` recorded
|
||||
// the module, `push` sealed a random own secret where the bus credential belongs, and the process
|
||||
// crash-looped until a person ran `module issue` and pushed again. Now assigning a module that speaks
|
||||
// on the bus issues its credential in the same act — or, when the bus cannot be reached from here,
|
||||
// says which verb to run — and a push never seals a placeholder in a credential's place.
|
||||
|
||||
func aTalker() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "talker", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/talker/broker"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/talker", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAssigningAModuleThatSpeaksOnTheBusNamesItsCredential(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
|
||||
// No bus is known to this process, so the credential cannot be issued here: the assignment
|
||||
// stands and says exactly what must happen before a push — never silently.
|
||||
said, err := assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "module issue talker --node laptop") {
|
||||
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
|
||||
}
|
||||
|
||||
// And the push refuses to send it, naming the same verb, rather than sealing a placeholder.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err == nil {
|
||||
t.Fatal("a push sealed a placeholder where talker's bus credential belongs")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "module issue talker --node laptop") || !strings.Contains(err.Error(), "issue 203") {
|
||||
t.Fatalf("the refusal does not say what to run: %v", err)
|
||||
}
|
||||
|
||||
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
|
||||
// does not mint again: a credential rotates on purpose, never by habit.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "laptop.talker", Kind: inventory.BusModule, Node: "laptop", Module: "talker"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err = assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "module issue") {
|
||||
t.Fatalf("a module with a minted credential was told to issue one:\n%s", said)
|
||||
}
|
||||
again, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again != hash {
|
||||
t.Fatal("re-assigning rotated the credential")
|
||||
}
|
||||
}
|
||||
|
||||
// A module that declares no broker secret is left alone: nothing to issue, nothing said.
|
||||
func TestAssigningAModuleThatDoesNotSpeakSaysNothingOfCredentials(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
register(t, open, helloWeb())
|
||||
said, err := assign(t.Context(), open, "laptop", "hello-web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "credential") {
|
||||
t.Fatalf("a module without a broker secret was told about credentials:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
|
||||
type recordingDelivery struct {
|
||||
did []string
|
||||
grantErr error
|
||||
declareErr error
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
|
||||
for _, s := range sending {
|
||||
r.did = append(r.did, "grant "+s.node)
|
||||
}
|
||||
return r.grantErr
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
|
||||
if r.declareErr != nil {
|
||||
return "", r.declareErr
|
||||
}
|
||||
r.did = append(r.did, "declare "+s.node)
|
||||
return "digest-" + s.node, nil
|
||||
}
|
||||
|
||||
func ready(names ...string) []readyNode {
|
||||
var out []readyNode
|
||||
for _, n := range names {
|
||||
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
|
||||
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
|
||||
// carries the controller's own right to issue them, and goes first.
|
||||
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
|
||||
d := &recordingDelivery{}
|
||||
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
|
||||
if !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
|
||||
}
|
||||
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
|
||||
t.Fatalf("the digests sent were not answered: %v", digests)
|
||||
}
|
||||
|
||||
held := &recordingDelivery{}
|
||||
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
|
||||
if !reflect.DeepEqual(held.did, want) {
|
||||
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
|
||||
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
|
||||
// grant that would let the controller issue memberships is never held behind them.
|
||||
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
|
||||
// A failure naming no machine (the buckets): everything but the bus's machine.
|
||||
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
|
||||
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
|
||||
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
|
||||
!strings.Contains(err.Error(), "anchor, laptop not sent") {
|
||||
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
|
||||
}
|
||||
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
|
||||
}
|
||||
|
||||
// A membership that failed for one machine: that machine alone.
|
||||
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
|
||||
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
|
||||
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
|
||||
t.Fatalf("one machine's refused membership was not said: %v", err)
|
||||
}
|
||||
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
|
||||
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
|
||||
}
|
||||
|
||||
// The announced upgrade that hit it is asked again.
|
||||
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
|
||||
t.Fatal("an announcement whose send stopped at its grants is not asked again")
|
||||
}
|
||||
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
|
||||
t.Fatal("any failure is asked again, not only a grant's")
|
||||
}
|
||||
|
||||
// Nothing to send is nothing granted either.
|
||||
none := &recordingDelivery{grantErr: errors.New("never asked")}
|
||||
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
|
||||
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Whether the bus's machine goes first is read from the user list alone, by its digest.
|
||||
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
list := "users: [a, b]"
|
||||
if userListBehind(list, digestOf([]byte(list))) {
|
||||
t.Fatal("the list it was sent reads as behind")
|
||||
}
|
||||
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
|
||||
t.Fatal("a changed or never-sent list reads as current")
|
||||
}
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
// checked against. Among the machines it is moved to the front; not among them it is added only
|
||||
// when it is behind.
|
||||
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
names []string
|
||||
holder string
|
||||
behind bool
|
||||
want []string
|
||||
}{
|
||||
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
|
||||
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
|
||||
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
|
||||
} {
|
||||
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
|
||||
// store at all, so anything that tried to record or register would fail rather than pass quietly.
|
||||
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
|
||||
err := builds{}.Built(t.Context(), link.BuildResult{
|
||||
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
|
||||
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The mark survives the wire both ways: asked as a dry run, answered as one.
|
||||
func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
|
||||
raw, _ := json.Marshal(link.BuildRequest{ID: "build-1", Repository: "r", DryRun: true})
|
||||
var asked link.BuildRequest
|
||||
if err := json.Unmarshal(raw, &asked); err != nil || !asked.DryRun {
|
||||
t.Fatalf("the request lost its dry-run mark: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(link.BuildResult{ID: "build-1", DryRun: true})
|
||||
var answered link.BuildResult
|
||||
if err := json.Unmarshal(raw, &answered); err != nil || !answered.DryRun {
|
||||
t.Fatalf("the outcome lost its dry-run mark: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(link.BuildResult{ID: "build-2"})
|
||||
if string(raw) != `{"id":"build-2","repository":"","on":""}` {
|
||||
t.Fatalf("an ordinary outcome carries a dry-run mark: %s", raw)
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package main
|
||||
|
||||
// The broker opening belongs only on the node that listens on it (novox/hq: it leaked onto
|
||||
// every enrolled node's declaration, opening a from-anywhere hole for a port nothing there
|
||||
// serves). foundationPortsFor is the scope.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
func TestTheBrokerHostGetsTheFoundationOpening(t *testing.T) {
|
||||
broker := catalogue.Manifest{Module: "lavinmq", Listens: []catalogue.Listening{
|
||||
{Port: 5671, Protocol: "tcp", From: "mesh"},
|
||||
{Port: 5672, Protocol: "tcp", From: "mesh"},
|
||||
}}
|
||||
got := foundationPortsFor(5671, []catalogue.Manifest{broker})
|
||||
if len(got) != 1 || got[0] != 5671 {
|
||||
t.Fatalf("the node that listens on the broker port keeps it; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatOnlyDialsTheBrokerGetsNoOpening(t *testing.T) {
|
||||
// ace's set: things that reach the broker as a client, none listening on 5671.
|
||||
ace := []catalogue.Manifest{
|
||||
{Module: "plex", Listens: []catalogue.Listening{{Port: 32400, Protocol: "tcp", From: "anywhere"}}},
|
||||
{Module: "postgres", Listens: []catalogue.Listening{{Port: 5432, Protocol: "tcp", From: "mesh"}}},
|
||||
}
|
||||
if got := foundationPortsFor(5671, ace); got != nil {
|
||||
t.Fatalf("a node that only dials out opens nothing for the broker; got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -18,16 +18,16 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||
plain := func(context.Context, string) (sendable, error) {
|
||||
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
||||
}
|
||||
failing := func(readyNode, []byte) error { return errors.New("the broker went away") }
|
||||
fine := func(readyNode, []byte) error { return nil }
|
||||
failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
|
||||
fine := &recordingDelivery{}
|
||||
|
||||
for name, round := range map[string]func() error{
|
||||
"a body that cannot be marshalled": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
|
||||
return err
|
||||
},
|
||||
"a send that fails": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
|
||||
return err
|
||||
},
|
||||
} {
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
|
||||
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
|
||||
// declaration composed before an assignment changed and sent after a newer one carried the higher
|
||||
// number — and the machine, which refuses a lower number, took the older content as the mesh's
|
||||
// newest word. Taken before the composition reads anything, the order of numbers is the order of
|
||||
// compositions, and the host's refusal does what it is for.
|
||||
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
|
||||
allot := numbered()
|
||||
var composed []string
|
||||
compose := func(stamp string) func(string) (sendable, error) {
|
||||
return func(node string) (sendable, error) {
|
||||
composed = append(composed, stamp)
|
||||
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
|
||||
}
|
||||
}
|
||||
// Composed first — before an assignment changed — and sent last.
|
||||
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
|
||||
// Composed after the change, sent first.
|
||||
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
|
||||
|
||||
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
|
||||
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
|
||||
stale[0].declared.Sequence, fresh[0].declared.Sequence)
|
||||
}
|
||||
// Sent in the other order, the numbers do not change — so the machine that has applied the
|
||||
// fresh one (2) refuses the stale one (1) when it arrives late.
|
||||
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
|
||||
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
|
||||
}
|
||||
if len(composed) != 2 || composed[0] != "before" {
|
||||
t.Fatalf("compositions happened in an unexpected order: %v", composed)
|
||||
}
|
||||
}
|
||||
|
||||
// The number is taken before the first read of the composition, not after it: an allotter that
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (int64, error) {
|
||||
if node == "anchor" {
|
||||
return 0, context.DeadlineExceeded
|
||||
}
|
||||
return 7, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
if node == "anchor" {
|
||||
t.Fatal("anchor was composed although its number could not be taken")
|
||||
}
|
||||
return sendable{}, nil
|
||||
})
|
||||
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
|
||||
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
|
||||
}
|
||||
if len(refusals) != 1 {
|
||||
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
|
||||
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
|
||||
// current" over a machine that had just been sent something else.
|
||||
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
t.Fatalf("recording a send after cancellation failed: %v", err)
|
||||
}
|
||||
outstanding, err := inv.Outstanding(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outstanding != digest || digest != digestOf(body) {
|
||||
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -72,6 +73,23 @@ func run() error {
|
||||
return askCommand(ctx, args[1:])
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return queueCommand(ctx, args[1:])
|
||||
case "cancel":
|
||||
return cancelCommand(ctx, args[1:])
|
||||
case "clear":
|
||||
return clearCommand(ctx, args[1:])
|
||||
case "rebuild":
|
||||
return rebuildCommand(ctx, args[1:])
|
||||
case "replay":
|
||||
return replayCommand(ctx, args[1:])
|
||||
case "kill":
|
||||
return killCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
return collectionCommand(ctx, args[1:])
|
||||
case "plans":
|
||||
return plansCommand(ctx, args[1:])
|
||||
case "pin":
|
||||
@@ -178,8 +196,8 @@ func usage() {
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||
assign <node> <module> put a module on a node
|
||||
unassign <node> <module> take it off
|
||||
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||
unassign <node> <module>... take them off
|
||||
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
||||
what it declares; --yes <digest> cuts it over as previewed
|
||||
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
||||
@@ -199,6 +217,15 @@ func usage() {
|
||||
build --behind build every module the mesh holds older than its source
|
||||
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
||||
builds [<module>] what has been built lately, and what came of it
|
||||
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
|
||||
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
|
||||
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
|
||||
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
|
||||
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
@@ -252,26 +279,42 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
||||
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
||||
// became of a build nobody was watching.
|
||||
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
|
||||
// unmerged branch was heard here like any build, registered, and its definition reached a machine
|
||||
// before anyone had reviewed it.
|
||||
if result.DryRun {
|
||||
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
switch {
|
||||
case err != nil && result.Failed != "":
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
if result.Module != "" {
|
||||
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed)
|
||||
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
|
||||
} else {
|
||||
planFailedBuild(ctx, b.open, result)
|
||||
}
|
||||
return nil
|
||||
case errors.Is(err, inventory.ErrSuperseded):
|
||||
// Not a failure: the module is already at what a later request built. A plan that asked
|
||||
// before that later request is answered by it; one that asked after it ignores this.
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
return nil
|
||||
case err != nil:
|
||||
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
|
||||
if manifest.Module != "" {
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error())
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "")
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -147,6 +147,40 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** (novox/hq ADR 0195): what each module
|
||||
// is, where it runs, whether it is current, and what it says of itself.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Built string `json:"built,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Current bool `json:"current"`
|
||||
Provided bool `json:"provided,omitempty"`
|
||||
// Tools says whether the module answers tools anywhere it runs: a list of its own,
|
||||
// a bundle the runtime serves, or a seat's verbs it claims (novox/hq ADR 0197) —
|
||||
// what the console checks the bus's answers against.
|
||||
Tools bool `json:"tools"`
|
||||
On []string `json:"on"`
|
||||
Provides []string `json:"provides,omitempty"`
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
Claims []string `json:"claims,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
out := make([]listed, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||
for _, c := range m.Claims {
|
||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
fmt.Println("this mesh knows about no modules yet")
|
||||
return nil
|
||||
@@ -300,8 +334,10 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return fmt.Errorf("%s <node> <module>", verb)
|
||||
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
|
||||
// service manager and the package manager — can only go on, or come off, together.
|
||||
if len(args) < 2 {
|
||||
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -315,7 +351,7 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
if verb == "unassign" {
|
||||
act = unassign
|
||||
}
|
||||
said, err := act(ctx, open, args[0], args[1])
|
||||
said, err := act(ctx, open, args[0], args[1:]...)
|
||||
if said != "" {
|
||||
fmt.Println(said)
|
||||
}
|
||||
@@ -557,7 +593,7 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module,
|
||||
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -577,6 +613,16 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||
}
|
||||
|
||||
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
|
||||
// the runtime (novox/hq ADR 0175), a module otherwise. The username is the same either way — the
|
||||
// runtime is issued through this same path — and the kind is what a reader of the records sees.
|
||||
func busKindOf(module string) string {
|
||||
if module == catalogue.RuntimeModule {
|
||||
return inventory.BusNodeTools
|
||||
}
|
||||
return inventory.BusModule
|
||||
}
|
||||
|
||||
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
|
||||
// secret, and the module's consumer created where the bus can be reached. Split from the minting
|
||||
// so the move can issue every module against a bus whose address it worked out itself
|
||||
@@ -723,3 +769,23 @@ func claimsFor(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaresTools is whether a module answers tools wherever it runs (novox/hq ADR 0197): it names
|
||||
// tools of its own, its build delivers a bundle the node's runtime serves, or it claims a seat
|
||||
// whose verbs it serves. A module with none is never expected to announce anything.
|
||||
func declaresTools(m catalogue.Manifest) bool {
|
||||
if len(m.Tools) > 0 {
|
||||
return true
|
||||
}
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if len(c.Serves) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -253,11 +254,10 @@ func theResolver(t *testing.T) catalogue.Manifest {
|
||||
return m
|
||||
}
|
||||
|
||||
// The resolver is handed every machine on the private network as a wildcard, the same set and the
|
||||
// same source as the hosts file, and is handed it again when a machine leaves — through the
|
||||
// module's own manifest asking for the fact, with no module of the mesh's own in between (hal
|
||||
// dnsmasq-app conversion, novox/hq 08-connectivity). The runtime on that machine is pointed at the
|
||||
// machine's own address, where the resolver answers for its containers.
|
||||
// The resolver is handed every machine on the private network as a wildcard, and is handed it again
|
||||
// when a machine leaves — through the module's own manifest asking for the fact, with no module of the
|
||||
// mesh's own in between (hal dnsmasq-app conversion, novox/hq 08-connectivity). It is the mesh's one
|
||||
// resolver (ADR 0194), and the container runtime is given no resolver of its own (ADR 0196).
|
||||
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -265,6 +265,12 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
if _, err := assign(ctx, open, "anchor", "dnsmasq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its bus credential, as assigning issues it where the bus is reachable (novox/hq issue 203):
|
||||
// no bus is known to this test, so it is minted here, or composing refuses the placeholder.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "anchor.dnsmasq", Kind: inventory.BusModule, Node: "anchor", Module: "dnsmasq"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zones := func() string {
|
||||
t.Helper()
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
@@ -286,11 +292,12 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
|
||||
}
|
||||
}
|
||||
// The container runtime is given no resolver of its own (novox/hq ADR 0196): it copies its
|
||||
// machine's, which name the mesh's resolver first. A `dns` key would be a second account of where a
|
||||
// container asks, read only when the runtime starts.
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "dnsmasq.runtime-dns" {
|
||||
if !strings.Contains(r["content"].(string), `"10.77.0.1"`) || r["into"] != "json" {
|
||||
t.Errorf("the runtime is not pointed at this machine's own address, written into its file: %v", r)
|
||||
}
|
||||
t.Errorf("the resolver still writes the runtime's own dns: %v", r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,3 +310,28 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
|
||||
}
|
||||
}
|
||||
|
||||
// The roster is the machines and nothing else (novox/hq ADR 0191): each node's internal domain covers
|
||||
// every route on it, and a node's public domains are public DNS's. A routed name in `.Names` was a
|
||||
// private answer for a public name, handed by a resolver serving a LAN to a phone that could not use it.
|
||||
func TestTheRosterNamesOnlyTheMachines(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(with.Names, with.Machines) {
|
||||
t.Fatalf("%s's roster names more than the machines:\n names %v\n machines %v", node, with.Names, with.Machines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -44,6 +45,21 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** — the console's discovery reads it
|
||||
// (novox/hq ADR 0195), and a reader that parses a printed column breaks when it is reworded.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Name string `json:"name"`
|
||||
Heard string `json:"heard"`
|
||||
Mode string `json:"mode"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
out := make([]listed, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
out = append(out, listed{Name: n.Name, Heard: heardFrom(n), Mode: modeOf(n), ID: n.ID})
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Said rather than printed as nothing: an empty list and a failed read must never
|
||||
// look the same, and this command answering "none" is only honest because getting
|
||||
@@ -370,8 +386,12 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "accounts" {
|
||||
return busAccounts(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||
return consumerReset(args[1:])
|
||||
}
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file>")
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||
"broker consumer-reset <stream> <consumer>")
|
||||
}
|
||||
known, err := broker.FromEnvironment()
|
||||
if errors.Is(err, broker.ErrNotConfigured) {
|
||||
@@ -391,6 +411,34 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||
func consumerReset(args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
before, after, err := js.ResetConsumer(args[0], args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("consumer %s on %s re-made to deliver from now\n", args[1], args[0])
|
||||
fmt.Printf(" before: delivers %s, delivered to %d, acknowledged to %d, %d pending, %d unacknowledged\n",
|
||||
before.DeliverPolicy, before.Delivered, before.AckFloor, before.Pending, before.AckPending)
|
||||
fmt.Printf(" after: delivers %s, %d pending; what was pending is dropped. A holder bound to it may need its "+
|
||||
"process restarted to bind again\n", after.DeliverPolicy, after.Pending)
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
||||
//
|
||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
||||
@@ -500,3 +548,13 @@ func orNotReported(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||
func printJSON(v any) error {
|
||||
body, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -146,6 +147,14 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
||||
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
|
||||
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
|
||||
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
|
||||
// novox/hq issue 252: a module the mesh has never registered is still a module, when the merge
|
||||
// shows it is one — and a directory that may be shared code is still shared.
|
||||
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
|
||||
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
|
||||
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
|
||||
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), "gitea,keycloak"},
|
||||
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), "gitea,keycloak"},
|
||||
{"the root's files still", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
@@ -191,7 +200,7 @@ func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
|
||||
t.Fatalf("the source records as %+v", from)
|
||||
}
|
||||
// A manifest with no provenance at all is legitimate: fixing something in a hurry.
|
||||
if from, err := whereItComesFrom("", "", "", "", false); err != nil || from != (inventory.Source{}) {
|
||||
if from, err := whereItComesFrom("", "", "", "", false); err != nil || !reflect.DeepEqual(from, inventory.Source{}) {
|
||||
t.Fatalf("a manifest handed over with no provenance was refused: %+v, %v", from, err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
@@ -210,3 +219,27 @@ func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a module once built at a commit still follows its branch — a merge into it
|
||||
// matches the module, and a plan re-asks the branch, not the old commit.
|
||||
func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main"}
|
||||
pinned := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a"}
|
||||
if !sourceIs(pinned, m) {
|
||||
t.Error("a module whose record names a commit is left out of a merge into its branch")
|
||||
}
|
||||
full := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a1d2c3b4a5f60718293a4b5c6d7e8f9012"}
|
||||
if !sourceIs(full, m) {
|
||||
t.Error("a full commit hash is read as a branch")
|
||||
}
|
||||
if got := followedBranch("9c97a8a"); got != "" {
|
||||
t.Errorf("a plan would re-ask the old commit %q", got)
|
||||
}
|
||||
if got := followedBranch("release"); got != "release" {
|
||||
t.Errorf("a branch is not followed as named: %q", got)
|
||||
}
|
||||
// A module that follows another branch is still not this merge's.
|
||||
if sourceIs(inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "release"}, m) {
|
||||
t.Error("a module following another branch was matched")
|
||||
}
|
||||
}
|
||||
|
||||
+186
-105
@@ -8,16 +8,16 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// working out what one machine should be.
|
||||
@@ -129,6 +129,7 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
// a mesh-wide gatherer may pass over — see notResolvable.
|
||||
return catalogue.Resolution{}, nil, notResolvable{err}
|
||||
}
|
||||
logUnheld(nodeName, resolved.Unheld)
|
||||
|
||||
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
|
||||
// password a provider is told to create is the one its consumer was given — and sealed to
|
||||
@@ -397,7 +398,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
return sendable{}, err
|
||||
}
|
||||
return sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
|
||||
}
|
||||
|
||||
@@ -535,6 +536,23 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
var sealed string
|
||||
var err error
|
||||
if choosing == Allocating {
|
||||
// **The broker credential is never invented here** (novox/hq issue 203). Every other
|
||||
// own secret is the mesh's to make — a password nobody else knows — but this one
|
||||
// is an account on the bus, minted by `module issue` and sealed by it; a push that
|
||||
// made a random one would deliver a file the process cannot read and report the
|
||||
// machine applied. Refused by name, with the verb.
|
||||
if name == "broker" {
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
} else if !minted {
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
||||
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
||||
"`module issue %s --node %s`, then push again",
|
||||
m.Module, node, user, m.Module, node)
|
||||
}
|
||||
}
|
||||
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
|
||||
} else {
|
||||
var held bool
|
||||
@@ -630,43 +648,31 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
}
|
||||
|
||||
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
||||
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
|
||||
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
|
||||
// to serve and knows nothing about what they mean.
|
||||
// Kept apart from the machines, because a fact about the machines must not be handed the names
|
||||
// the mesh merely serves (novox/hq 04-ISSUES/111).
|
||||
// **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal
|
||||
// domain, `<node>.internal`, and every route on it is a name under that domain (ADR 0151), which
|
||||
// the resolver answers with one wildcard per machine — so no route needs a line of its own. A
|
||||
// node's public domains are the operator's and public DNS answers them; the mesh gives no private
|
||||
// answer for any of them. The roster once carried every routed name, public ones included, and a
|
||||
// resolver that also serves a LAN handed a phone a tunnel address for the mail server.
|
||||
// `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111).
|
||||
machines := make(map[string]string, len(names))
|
||||
for name, at := range names {
|
||||
machines[name] = at
|
||||
}
|
||||
routes, err := routeNamesInTheMesh(ctx, open)
|
||||
|
||||
// And every zone a module in the mesh answers itself (novox/hq ADR 0199), for the mesh's resolver
|
||||
// to forward.
|
||||
zones, err := zonesInTheMesh(ctx, open)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
for name, at := range routes {
|
||||
names[name] = at
|
||||
}
|
||||
|
||||
// The ports the mesh itself needs open, which no module declares. Read from the broker this
|
||||
// control plane was told about rather than written down twice: the address a node is handed in
|
||||
// its token and the port its machine must accept on are the same fact.
|
||||
//
|
||||
// **Only on the node that listens on it** (novox/hq issue: the broker opening leaked onto
|
||||
// every node). The opening exists to WIDEN the broker's port to from-anywhere — a machine
|
||||
// enrolling is not on the mesh yet, so the broker's own `from: mesh` listen would refuse its
|
||||
// first dial. That widening belongs on the broker's host and nowhere else: a node that only
|
||||
// dials out needs no incoming rule, and an opening for a port nothing here listens on is a
|
||||
// from-anywhere hole for a dead port. So the foundation port is kept only when a module
|
||||
// resolved onto THIS node actually listens on it.
|
||||
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
||||
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
||||
// before it had an address on the private network. A machine joins through the tunnel now, and
|
||||
// every link to the bus crosses it, so its reach is what the `nats` module declares: the mesh.
|
||||
// Nothing the mesh itself needs is opened beyond what a module declares.
|
||||
var foundation []int
|
||||
if b, err := broker.FromEnvironment(); err == nil {
|
||||
if _, port, err := net.SplitHostPort(b.Address); err == nil {
|
||||
if n, err := strconv.Atoi(port); err == nil {
|
||||
foundation = foundationPortsFor(n, plan.Modules)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
|
||||
// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The
|
||||
@@ -730,34 +736,25 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Machines: machines, Zones: zones,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
BusUsers: busUsers,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
|
||||
// ADR 0066).
|
||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||
// answering listen is published on there.
|
||||
//
|
||||
// **Mesh-wide, so any container resolves any routed name to its proxy** — including an internal
|
||||
// ACME validator, which cannot complete a challenge for a name it cannot reach. A routed name is
|
||||
// composed on the consumer's node (from its label and that node's public domain) and served by the
|
||||
// node answering the consumer's route requirement; this gathers both.
|
||||
//
|
||||
// It reads route names off resolutions rather than a table because there is no table: a route is a
|
||||
// contribution, computed from what each node runs. Name-agnostic — a contribution counts as a
|
||||
// routed name only because it carried a label the mesh composed, never because the mesh knows what
|
||||
// "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
|
||||
// the rest their names.
|
||||
//
|
||||
// **A node that could not be READ is a different matter and is raised.** Skipping one states, to
|
||||
// every machine at once, that its names do not exist — and since the roster is part of every
|
||||
// container's identity, that withdraws them and replaces every container (novox/hq 04-ISSUES/152,
|
||||
// 151). So every failure here says which machine and which read, because the alternative is a
|
||||
// mesh-wide refusal with nothing named in it.
|
||||
func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
|
||||
// Read across every machine's resolution, as the roster once read routed names: a node whose set does
|
||||
// not compose declares nothing and is passed over, so one broken machine does not cost the rest their
|
||||
// zones; a store that cannot be read is raised, naming the machine, because returning the zones
|
||||
// without it would withdraw them from the resolver as if the operator had (novox/hq 04-ISSUES/152).
|
||||
// What the mesh refuses about the zones together — one declared twice, one shadowing the mesh's
|
||||
// suffix or a node's public domain — is refused here, by name.
|
||||
func zonesInTheMesh(ctx context.Context, open *stores) ([]catalogue.ZoneAt, error) {
|
||||
inv := open.inventory
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
@@ -769,43 +766,47 @@ func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string,
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
|
||||
}
|
||||
|
||||
// Every machine's resolution first, then the names across them at once: which node serves a
|
||||
// name is a question about the graph — the consumer on one machine, the provider on another —
|
||||
// and answered wrongly by looking at one contribution at a time (novox/hq issue 178).
|
||||
plans := map[string]catalogue.Resolution{}
|
||||
settings := map[string]catalogue.SettingsBy{}
|
||||
var zones []catalogue.ZoneAt
|
||||
var public []string
|
||||
for _, n := range nodes {
|
||||
plan, layers, err := planFor(ctx, open, n.Name)
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
switch {
|
||||
case unresolvable(err):
|
||||
// Their set does not compose, so they serve no names. Passed over, so one machine's
|
||||
// broken set does not cost the rest theirs.
|
||||
continue
|
||||
case err != nil:
|
||||
// The mesh could not be asked. Returning the roster without this machine's names would
|
||||
// state that they do not exist — to every machine, and indistinguishably from the
|
||||
// operator having withdrawn them (novox/hq 04-ISSUES/152).
|
||||
return nil, fmt.Errorf("the names %s serves cannot be read: %w", n.Name, err)
|
||||
return nil, fmt.Errorf("the zones %s answers cannot be read: %w", n.Name, err)
|
||||
}
|
||||
plans[n.Name], settings[n.Name] = plan, layers
|
||||
}
|
||||
served, err := catalogue.NamesServed(plans, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for name, node := range served {
|
||||
if at := address[node]; at != "" {
|
||||
out[name] = at
|
||||
if plan.PublicDomain != "" {
|
||||
public = append(public, plan.PublicDomain)
|
||||
}
|
||||
for _, m := range plan.Modules {
|
||||
if m.Zone == nil {
|
||||
continue
|
||||
}
|
||||
at := address[n.Name]
|
||||
if at == "" {
|
||||
// Not on the private network yet: nothing could reach its answerer.
|
||||
continue
|
||||
}
|
||||
published, layers, err := portsGivenOn(ctx, inv, n.Name, m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the zone %s declares on %s cannot be read: %w", m.Module, n.Name, err)
|
||||
}
|
||||
z, err := catalogue.ZoneOn(m, layers, published, n.Name, at)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zones = append(zones, *z)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
if problems := catalogue.ZonesProblems(zones, overlay.Suffix(), public); len(problems) > 0 {
|
||||
return nil, fmt.Errorf("the mesh's zones cannot be forwarded:\n - %s", strings.Join(problems, "\n - "))
|
||||
}
|
||||
return zones, nil
|
||||
}
|
||||
|
||||
// certificateFor is what the mesh certifies about one machine's internal name.
|
||||
@@ -1340,6 +1341,20 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||
//
|
||||
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
|
||||
// resource of that module, so the question is whether it is here.
|
||||
list, missing, err := busUserList(ctx, inv, onThisNode)
|
||||
if len(missing) > 0 {
|
||||
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
return list, err
|
||||
}
|
||||
|
||||
// busUserList is composeBusUsers without saying anything: the list, and the users left out of it
|
||||
// for want of a credential. Asked on every send to decide whether the machine holding the bus must
|
||||
// go first (novox/hq issue 249), where saying the same missing users each time would bury them.
|
||||
func busUserList(ctx context.Context, inv *inventory.Inventory,
|
||||
onThisNode []catalogue.Manifest) (string, []string, error) {
|
||||
holdsTheBus := false
|
||||
for _, m := range onThisNode {
|
||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||
@@ -1347,54 +1362,33 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||
}
|
||||
}
|
||||
if !holdsTheBus {
|
||||
return "", nil
|
||||
return "", nil, nil
|
||||
}
|
||||
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
hashes := make(map[string]string, len(kept))
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
filled, missing := broker.WithPasswords(users, hashes)
|
||||
if len(missing) > 0 {
|
||||
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
if len(filled) == 0 {
|
||||
return "", fmt.Errorf(
|
||||
return "", missing, fmt.Errorf(
|
||||
"this machine runs the bus and not one user has a credential, so the composed list " +
|
||||
"would refuse every connection in the mesh")
|
||||
}
|
||||
return broker.ComposeAccounts(filled)
|
||||
}
|
||||
|
||||
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
|
||||
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
|
||||
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
|
||||
// not on the mesh yet and its first dial would be refused. That widening belongs on the broker's
|
||||
// host alone: a node that only dials out needs no incoming rule, and an opening for a port
|
||||
// nothing here listens on is a from-anywhere hole for a dead port.
|
||||
func foundationPortsFor(brokerPort int, modules []catalogue.Manifest) []int {
|
||||
for _, m := range modules {
|
||||
for _, l := range m.Listens {
|
||||
if l.Port == brokerPort {
|
||||
return []int{brokerPort}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
list, err := broker.ComposeAccounts(filled)
|
||||
return list, missing, err
|
||||
}
|
||||
|
||||
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
||||
@@ -1416,3 +1410,90 @@ func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.C
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
|
||||
// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
||||
//
|
||||
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
||||
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
||||
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
||||
// burying the line about the node it acted on. A command says what concerns its own act instead
|
||||
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
||||
var (
|
||||
unheldLogged = map[string]string{}
|
||||
unheldLoggedMu sync.Mutex
|
||||
logUnheldChanges bool
|
||||
)
|
||||
|
||||
// logUnheld logs a node's unmet seat dependencies when they differ from what was last logged for
|
||||
// it, including when they become none — in the serving controller, and nowhere else.
|
||||
func logUnheld(node string, unheld []catalogue.Unheld) {
|
||||
if !logUnheldChanges {
|
||||
return
|
||||
}
|
||||
lines := make([]string, 0, len(unheld))
|
||||
for _, u := range unheld {
|
||||
lines = append(lines, u.String())
|
||||
}
|
||||
now := strings.Join(lines, "\n")
|
||||
unheldLoggedMu.Lock()
|
||||
before, seen := unheldLogged[node]
|
||||
unheldLogged[node] = now
|
||||
unheldLoggedMu.Unlock()
|
||||
if (seen && before == now) || (!seen && now == "") {
|
||||
return
|
||||
}
|
||||
if now == "" {
|
||||
fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
||||
node, len(lines), strings.Join(lines, "\n "))
|
||||
}
|
||||
|
||||
// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over
|
||||
// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not —
|
||||
// which includes every one of a module just assigned — and each now met that was not. Nothing about
|
||||
// any other node, and nothing that was already true before the act.
|
||||
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
|
||||
judge := func(names []string) map[string]catalogue.Unheld {
|
||||
var set []catalogue.Manifest
|
||||
for _, n := range names {
|
||||
if m, known := shelf[n]; known {
|
||||
set = append(set, m)
|
||||
}
|
||||
}
|
||||
out := map[string]catalogue.Unheld{}
|
||||
for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
|
||||
out[u.Module+" "+u.Seat] = u
|
||||
}
|
||||
return out
|
||||
}
|
||||
was, now := judge(before), judge(after)
|
||||
var lines []string
|
||||
for _, k := range sortedNames(now) {
|
||||
if _, already := was[k]; !already {
|
||||
lines = append(lines, "but "+now[k].String())
|
||||
}
|
||||
}
|
||||
for _, k := range sortedNames(was) {
|
||||
if _, still := now[k]; still {
|
||||
continue
|
||||
}
|
||||
u := was[k]
|
||||
if !slices.Contains(after, u.Module) {
|
||||
continue // went with its module, which says nothing about the seat
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("and %s on %s now has %s held", u.Module, node, u.Seat))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func sortedNames[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,401 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A plan follows what is done to the build queue (novox/hq ADR 0219).
|
||||
//
|
||||
// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
|
||||
//
|
||||
// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
|
||||
// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
|
||||
// sends a reader looking for a fault that is a decision. It says what it waits on instead.
|
||||
// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
|
||||
// failed plan's failed modules and the plan goes on from that tier as if they had built the
|
||||
// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
|
||||
// beside it — beside it, the plan would either ask it again or stay failed on an outcome the
|
||||
// rebuild has already replaced.
|
||||
|
||||
// pauseView is whether the build seat takes work: the holders that said they are paused, and
|
||||
// whether that is every holder.
|
||||
type pauseView struct {
|
||||
Nodes []string
|
||||
All bool
|
||||
}
|
||||
|
||||
// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
|
||||
// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
|
||||
func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
|
||||
if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
|
||||
return "", false
|
||||
}
|
||||
var asked *time.Time
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
|
||||
if asked == nil || s.AskedAt.Before(*asked) {
|
||||
asked = s.AskedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
if asked == nil {
|
||||
return "", false
|
||||
}
|
||||
waited := now.Sub(*asked)
|
||||
said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
|
||||
strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
|
||||
// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
|
||||
// longer than a day is named.
|
||||
if waited > pausedTooLong {
|
||||
said += " — PAUSED OVER A DAY: `resume` takes builds again"
|
||||
}
|
||||
return said, true
|
||||
}
|
||||
|
||||
// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
|
||||
const pausedTooLong = 24 * time.Hour
|
||||
|
||||
// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
|
||||
// where the seat being paused changes what a plan says.
|
||||
func awaitsABuild(plans []inventory.Plan) bool {
|
||||
for _, p := range plans {
|
||||
if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
|
||||
// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
|
||||
// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
|
||||
// reads as late, which is what it said before this existed.
|
||||
func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
|
||||
if !awaitsABuild(plans) {
|
||||
return pauseView{}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
seat := buildSeatAmong(entries)
|
||||
holders := holdersAmong(entries, seat)
|
||||
if len(holders) == 0 {
|
||||
return pauseView{}
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
}
|
||||
defer js.Close()
|
||||
said, err := link.PausedSaid(js, seat, holders)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
}
|
||||
return pauseOf(holders, said)
|
||||
}
|
||||
|
||||
// pauseOf is the view from what each holder said.
|
||||
func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
|
||||
var v pauseView
|
||||
for _, n := range holders {
|
||||
if said[n].Paused {
|
||||
v.Nodes = append(v.Nodes, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(v.Nodes)
|
||||
v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
|
||||
return v
|
||||
}
|
||||
|
||||
// failedIn is the modules of a plan's current tier that failed to build, sorted.
|
||||
func failedIn(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
|
||||
// that holds what this one held now (issue 254, ADR 0218).
|
||||
func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
return q, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
switch {
|
||||
case p.State == inventory.PlanDone:
|
||||
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||
case p.State == inventory.PlanSuperseded:
|
||||
return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
|
||||
case p.Open():
|
||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||
}
|
||||
if len(failedIn(p)) > 0 {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
stopped := stoppedRollouts(p)
|
||||
if len(stopped) == 0 {
|
||||
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
|
||||
p.Tier, p.ID, p.Note)
|
||||
}
|
||||
// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
|
||||
// sent — a newer build, and sending this one again would put the older build back on its machines.
|
||||
for _, m := range stopped {
|
||||
if q, found := newerPlanFor(m, p, plans); found {
|
||||
return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
|
||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
|
||||
// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
|
||||
func stoppedRollouts(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
|
||||
len(s.First) > 0 && s.Why != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
|
||||
func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
for _, tier := range q.Tiers {
|
||||
for _, m := range tier {
|
||||
if m == module {
|
||||
return q, true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
|
||||
// variable so a test of a retried rollout needs no machine.
|
||||
var sendRollout = sendToEach
|
||||
|
||||
// resumed sets a failed plan building again once nothing in its tier is failed.
|
||||
func resumed(p *inventory.Plan, why string) {
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = why
|
||||
}
|
||||
}
|
||||
|
||||
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||
// that tier: what follows is the plan going on as if they had built the first time.
|
||||
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans = append(plans, recent...)
|
||||
if err := retryRefusal(p, plans); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(failedIn(p)) == 0 {
|
||||
return retryRollouts(ctx, open, &p)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
failed := failedIn(p)
|
||||
var asked []string
|
||||
for _, m := range failed {
|
||||
askModule(ctx, &p, m, byName)
|
||||
if s := p.Modules[m]; s.State == "asked" {
|
||||
asked = append(asked, m+" as "+s.Build)
|
||||
}
|
||||
}
|
||||
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.State != inventory.PlanBuilding {
|
||||
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
|
||||
p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
|
||||
}
|
||||
|
||||
// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
|
||||
// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
|
||||
func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
defer release()
|
||||
openPlans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 20)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
p, found := planHolding(module, openPlans, recent)
|
||||
if !found {
|
||||
return false, "", nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
was := p.State
|
||||
askModule(ctx, &p, module, byName)
|
||||
s := p.Modules[module]
|
||||
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if s.State != "asked" {
|
||||
return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
|
||||
}
|
||||
said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
|
||||
s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
|
||||
switch {
|
||||
case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
|
||||
said += "; the plan had failed and builds again from this tier"
|
||||
case was == inventory.PlanFailed:
|
||||
said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
|
||||
}
|
||||
return true, said, nil
|
||||
}
|
||||
|
||||
// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
|
||||
// yet built, else the newest failed plan whose current tier does, and that no open plan of its
|
||||
// repository supersedes.
|
||||
func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
|
||||
holds := func(p inventory.Plan) bool {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if m == module {
|
||||
s := p.Modules[m]
|
||||
return s == nil || s.State != "built"
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, p := range openPlans {
|
||||
if holds(p) {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
sorted := append([]inventory.Plan(nil), recent...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
|
||||
for _, p := range sorted {
|
||||
if p.State != inventory.PlanFailed || !holds(p) {
|
||||
continue
|
||||
}
|
||||
if _, superseded := newerOpenPlan(p, openPlans); superseded {
|
||||
continue
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
|
||||
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
|
||||
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
|
||||
func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
|
||||
var said []string
|
||||
for _, m := range stoppedRollouts(*p) {
|
||||
s := p.Modules[m]
|
||||
sent, err := sendRollout(ctx, open, s.First)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
|
||||
m, strings.Join(s.First, ", "), p.ID, err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
s.First, s.FirstAt, s.Why = sent, &now, ""
|
||||
said = append(said, m+" to "+strings.Join(sent, ", "))
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||
if err := open.inventory.SavePlan(ctx, *p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
|
||||
"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 213: for the moment a machine hands its controller over, the container and the
|
||||
// process both run the plan timer on one store. Only the one holding the plans moves them; the other
|
||||
// leaves them alone, and moves them once they are let go.
|
||||
func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now().UTC()
|
||||
// Every tier done: the next step is the plan's last, and needs nothing but the store.
|
||||
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
|
||||
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The other controller: its own connections to the same store, holding the plans.
|
||||
other, err := inventory.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(other.Close)
|
||||
release, err := other.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(release) // before the close above: a pool waits for a connection still held
|
||||
|
||||
advancePlans(ctx, open)
|
||||
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() {
|
||||
t.Fatalf("a controller moved a plan another held: %+v %v", p, err)
|
||||
}
|
||||
|
||||
release()
|
||||
advancePlans(ctx, open)
|
||||
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone {
|
||||
t.Fatalf("the plan did not move once it was let go: %+v %v", p, err)
|
||||
}
|
||||
}
|
||||
+440
-118
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
@@ -62,6 +63,9 @@ func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.En
|
||||
}
|
||||
|
||||
func serve(ctx context.Context) error {
|
||||
// The one process whose log is read over time, so the one that says each change to a node's
|
||||
// unmet seat dependencies once (novox/hq ADR 0207).
|
||||
logUnheldChanges = true
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -131,10 +135,17 @@ func serve(ctx context.Context) error {
|
||||
|
||||
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
|
||||
// from the store's row, so what the seat declares is what is answered.
|
||||
handlers, err := seatToolHandlers()
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
// Said once, loudly, and then served anyway (novox/hq ADR 0185): the mesh keeps answering
|
||||
// while whatever put an older control plane here is undone.
|
||||
fmt.Printf("this control plane is behind the %s row: it cannot run %s. "+
|
||||
"Those answer the reason when called; everything else is served as usual\n",
|
||||
catalogue.ControllerSeatName, strings.Join(behind, ", "))
|
||||
}
|
||||
bus, isNATS := server.Bus().(link.OverNATS)
|
||||
if !isNATS {
|
||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||
@@ -144,6 +155,12 @@ func serve(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
defer stopServing()
|
||||
// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
|
||||
stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopAnnouncing()
|
||||
|
||||
return server.Serve(ctx)
|
||||
}
|
||||
@@ -198,6 +215,12 @@ func declare(ctx context.Context, args []string) error {
|
||||
if err := link.Declare(ctx, server.Bus(), ident, node, raw, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
|
||||
// is still what the machine was last told, and status must not read it as current for the one
|
||||
// the mesh would compose.
|
||||
if _, err := recordSent(ctx, inv, node, raw); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||
return nil
|
||||
}
|
||||
@@ -335,13 +358,27 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
asked = append(asked, n.Name)
|
||||
}
|
||||
|
||||
// **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's
|
||||
// user list, and a module's new grants are refused by the bus until that list says them. Among
|
||||
// the machines asked it goes first; not among them and behind, it is added — a named push whose
|
||||
// module gained a state would otherwise send the code and leave the right to use it for the
|
||||
// cascade below, after.
|
||||
holder, holderBehind, err := brokerBehind(ctx, open, asked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
asked = brokerFirst(asked, holder, holderBehind)
|
||||
|
||||
// Held from composing to sending, so a converge on one of them cannot send between the two
|
||||
// and be overtaken by what was composed before it (novox/hq ADR 0100).
|
||||
held, release, err := holdNodes(ctx, open, asked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sending, refusals := composeEach(asked, func(node string) (sendable, error) {
|
||||
// Each machine's unmet seat dependencies (novox/hq ADR 0207), said after the sends: in full
|
||||
// for a machine named, as a count for each of many — the full list is `status`'s.
|
||||
unheld := map[string][]catalogue.Unheld{}
|
||||
sending, refusals := composeEach(asked, allotting(held, inv), func(node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
@@ -350,6 +387,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
// healthy modules beside it are still resolved and sent. Reported so it is not silently
|
||||
// dropped — the remedy is to move it, and until then the rest of the node converges.
|
||||
reportUnhostable(node, plan)
|
||||
unheld[node] = plan.Unheld
|
||||
// The private network is in here with everything else. It used to be composed separately
|
||||
// and prepended, which meant every machine with an address was on it and no machine could
|
||||
// be kept off. It is a module now, so it arrives the way a module does.
|
||||
@@ -360,42 +398,17 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return declared, err
|
||||
})
|
||||
|
||||
sentDigest := map[string]string{}
|
||||
defer release()
|
||||
for _, s := range sending {
|
||||
// Numbered under the hold, one higher than the last, before the body exists — the number is
|
||||
// inside the signed bytes, so a replayed older declaration cannot borrow a newer one's
|
||||
// (novox/hq 04-ISSUES/107).
|
||||
if err := number(ctx, inv, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would
|
||||
// make the machine look current for a declaration it never received.
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := inv.RecordSent(ctx, record.ID, digest); err != nil {
|
||||
return err
|
||||
}
|
||||
sentDigest[s.node] = digest
|
||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
// Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push
|
||||
// is the one most operators run, and on 2026-10-01 it was the one path that issued none.
|
||||
bus := overTheBus{open: open, server: server, signer: ident}
|
||||
sentDigest, err := deliver(ctx, bus, holder, sending)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
release()
|
||||
fmt.Printf("\n%d node(s) told\n", len(sending))
|
||||
// And each machine's memberships, as every other send does (ADR 0160): a push is the one most
|
||||
// operators run, and on 2026-10-01 it was the one path that issued none.
|
||||
if err := issueMemberships(ctx, open, server, sending); err != nil {
|
||||
return err
|
||||
}
|
||||
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
|
||||
|
||||
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
||||
// issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's
|
||||
@@ -464,21 +477,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
return declared, err
|
||||
},
|
||||
func(s readyNode, body []byte) error {
|
||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body,
|
||||
15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
return nil
|
||||
})
|
||||
bus, holder)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -562,17 +561,31 @@ type readyNode struct {
|
||||
//
|
||||
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
||||
// across two machines that must agree — and dropped here, where it never did.
|
||||
func composeEach(names []string,
|
||||
func composeEach(names []string, allot func(node string) (int64, error),
|
||||
compose func(node string) (sendable, error)) ([]readyNode, []string) {
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// **Numbered before it is composed, not before it is sent** (novox/hq issue 204). The
|
||||
// number says where this declaration stands against every other the mesh composed for the
|
||||
// machine, and the host refuses one lower than the last it applied. Taken at send time, as
|
||||
// it was, a declaration composed a minute ago — before an assignment changed — went out with
|
||||
// a number higher than one composed after the change and sent before it, and the machine
|
||||
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
|
||||
// two machines was undone two seconds later by exactly that. Taken here, before the first
|
||||
// read, what was composed earlier is numbered lower whatever order the sends happen in.
|
||||
seq, err := allot(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared, err := compose(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence = seq
|
||||
if len(declared.Resources) == 0 {
|
||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||
@@ -591,33 +604,258 @@ func composeEach(names []string,
|
||||
// sendRound holds the named nodes, composes each and sends each that composed, and gives the hold
|
||||
// back on every way out — a body that cannot be marshalled and a send that fails included
|
||||
// (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are
|
||||
// still sent.
|
||||
// still sent. Their memberships go before their declarations, as every send's do (issue 249).
|
||||
func sendRound(ctx context.Context, open *stores, names []string,
|
||||
compose func(held context.Context, node string) (sendable, error),
|
||||
send func(s readyNode, body []byte) error) ([]string, error) {
|
||||
d delivery, holder string) ([]string, error) {
|
||||
held, release, err := holdNodes(ctx, open, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
sending, refused := composeEach(names, func(node string) (sendable, error) {
|
||||
sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
|
||||
return compose(held, node)
|
||||
})
|
||||
for _, s := range sending {
|
||||
if err := number(ctx, open.inventory, &s); err != nil {
|
||||
return refused, err
|
||||
}
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return refused, err
|
||||
}
|
||||
if err := send(s, body); err != nil {
|
||||
return refused, err
|
||||
}
|
||||
if _, err := deliver(held, d, holder, sending); err != nil {
|
||||
return refused, err
|
||||
}
|
||||
return refused, nil
|
||||
}
|
||||
|
||||
// delivery is the two acts of sending machines what they should be, apart, so the order between
|
||||
// them is one function's and can be read and tested there (novox/hq issue 249).
|
||||
type delivery interface {
|
||||
// grant issues what the machines' modules may do — each module's state raised and its
|
||||
// membership issued — for every machine about to be sent.
|
||||
grant(ctx context.Context, sending []readyNode) error
|
||||
// declare sends one machine its declaration and records it sent, answering the digest.
|
||||
declare(ctx context.Context, s readyNode, body []byte) (string, error)
|
||||
}
|
||||
|
||||
// errGrants marks a send that stopped because what the machines' modules may do could not be issued
|
||||
// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an
|
||||
// announcement that hits it is held and asked again.
|
||||
var errGrants = errors.New("what the machines' modules may do on the bus could not be issued, and code " +
|
||||
"sent before its grants is refused there")
|
||||
|
||||
// grantsRefused is a grant that failed for some machines and not others: their memberships could not
|
||||
// be issued, by machine, and only those machines are held back.
|
||||
type grantsRefused struct{ nodes map[string]error }
|
||||
|
||||
func (g *grantsRefused) Error() string {
|
||||
names := make([]string, 0, len(g.nodes))
|
||||
for n := range g.nodes {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return fmt.Sprintf("the memberships of %s could not be issued; the first: %v",
|
||||
strings.Join(names, ", "), g.nodes[names[0]])
|
||||
}
|
||||
|
||||
// deliver sends the machines their declarations: **the machine holding the bus, then the grants,
|
||||
// then the rest** (novox/hq issue 249).
|
||||
//
|
||||
// A merge gave a module a new state; its bundle reached every machine within a minute, and the
|
||||
// machines' permissions on the bus did not include the state until somebody pushed by hand: the code
|
||||
// arrived before the right to use it. A module that read its new state on start failed its start; the
|
||||
// one that was there retried for two minutes. The memberships were issued after the declarations —
|
||||
// "because the runtime it is for arrives with it" — and a membership is retained last-per-subject on
|
||||
// the bus (internal/link/bus.go), so issued first it waits for the runtime that arrives after it. A
|
||||
// runtime still on the old code merely holds a grant it does not use yet.
|
||||
//
|
||||
// **The holder's declaration before the grants, though.** The bus's user list travels in it, and the
|
||||
// controller's own right to publish memberships and raise buckets is in that list (the precedent of
|
||||
// issue 183): grants first, and a grant the controller is not yet allowed to make would hold the very
|
||||
// declaration that allows it — a lock only a hand on the broker could open. A runtime already running
|
||||
// on that machine follows a membership issued after its declaration, as it always has.
|
||||
//
|
||||
// **A grant that cannot be issued holds back what it concerns, and says so as an error.** It used to
|
||||
// be said and passed over — "the machines keep what they derive until the next push" — which reported
|
||||
// a rollout done that had delivered code its machines could not run. A membership that failed holds
|
||||
// back its own machine; a failure that names no machine (the buckets) holds back every machine but the
|
||||
// holder, already sent. The error carries errGrants, so the caller's rollout is not marked sent and is
|
||||
// tried again.
|
||||
//
|
||||
// The grants are issued, not waited on: a membership is a retained message the runtime reads when it
|
||||
// comes, and the bus answers its publication; nothing here waits for a runtime to have read one.
|
||||
func deliver(ctx context.Context, d delivery, holder string, sending []readyNode) (map[string]string, error) {
|
||||
digests := map[string]string{}
|
||||
if len(sending) == 0 {
|
||||
return digests, nil
|
||||
}
|
||||
send := func(s readyNode) error {
|
||||
// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
|
||||
// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err := d.declare(ctx, s, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digests[s.node] = digest
|
||||
return nil
|
||||
}
|
||||
var rest []readyNode
|
||||
for _, s := range sending {
|
||||
if holder != "" && s.node == holder {
|
||||
if err := send(s); err != nil {
|
||||
return digests, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
rest = append(rest, s)
|
||||
}
|
||||
held := map[string]error{}
|
||||
if err := d.grant(ctx, sending); err != nil {
|
||||
var some *grantsRefused
|
||||
if !errors.As(err, &some) {
|
||||
var names []string
|
||||
for _, s := range rest {
|
||||
names = append(names, s.node)
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return digests, fmt.Errorf("%w: %w", errGrants, err)
|
||||
}
|
||||
return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(names, ", "), err)
|
||||
}
|
||||
held = some.nodes
|
||||
}
|
||||
var notSent []string
|
||||
for _, s := range rest {
|
||||
if _, refused := held[s.node]; refused {
|
||||
notSent = append(notSent, s.node)
|
||||
continue
|
||||
}
|
||||
if err := send(s); err != nil {
|
||||
return digests, err
|
||||
}
|
||||
}
|
||||
if len(notSent) > 0 {
|
||||
return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(notSent, ", "),
|
||||
&grantsRefused{nodes: held})
|
||||
}
|
||||
if len(held) > 0 {
|
||||
// Only the holder's own memberships failed, and it was sent before them.
|
||||
return digests, fmt.Errorf("%w: %w", errGrants, &grantsRefused{nodes: held})
|
||||
}
|
||||
return digests, nil
|
||||
}
|
||||
|
||||
// overTheBus is delivery as the mesh does it: memberships on the bus, declarations signed.
|
||||
type overTheBus struct {
|
||||
open *stores
|
||||
server *link.Server
|
||||
signer link.Signer
|
||||
// indent is put before each "sent" line, for the callers whose output is nested.
|
||||
indent string
|
||||
}
|
||||
|
||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
return issueMemberships(ctx, b.open, b.server, sending)
|
||||
}
|
||||
|
||||
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if s.declared.BusUsers != "" {
|
||||
// And the user list it carried, so the next send reads whether it must go first from the
|
||||
// list alone (novox/hq issue 249). On the same outliving context as the send's record.
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers)))
|
||||
cancel()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources))
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine
|
||||
// holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a
|
||||
// module's new permissions are refused by the bus until that list says them. Among the machines it
|
||||
// is moved to the front; not among them, it is added only when it is behind.
|
||||
func brokerFirst(names []string, holder string, behind bool) []string {
|
||||
if holder == "" {
|
||||
return names
|
||||
}
|
||||
present := false
|
||||
rest := make([]string, 0, len(names))
|
||||
for _, n := range names {
|
||||
if n == holder {
|
||||
present = true
|
||||
continue
|
||||
}
|
||||
rest = append(rest, n)
|
||||
}
|
||||
if !present && !behind {
|
||||
return names
|
||||
}
|
||||
return append([]string{holder}, rest...)
|
||||
}
|
||||
|
||||
// brokerBehind is the machine holding the bus — the one whose declaration carries the user list —
|
||||
// and, when it is not among the machines named, whether the user list it would be sent now differs
|
||||
// from the one it was last sent (novox/hq issue 249).
|
||||
//
|
||||
// **The user list alone, not the whole declaration.** Read from the whole declaration, any change
|
||||
// pending on that machine — an upgrade its policy records rather than rolls out — went with every
|
||||
// send anywhere, and a module running there always put it in its first wave. A digest of the list
|
||||
// last sent is kept for this (ADR 0043: the list is composed on each push, never kept itself).
|
||||
func brokerBehind(ctx context.Context, open *stores, names []string) (string, bool, error) {
|
||||
inv := open.inventory
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
h, held := holders[theBrokerSeat]
|
||||
if !held || h.Node == "" {
|
||||
return "", false, nil
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if m, known := shelf[h.Module]; !known || m.BusUsers == "" {
|
||||
// A holder that is sent no user list carries no grant: nothing to send first.
|
||||
return "", false, nil
|
||||
}
|
||||
for _, n := range names {
|
||||
if n == h.Node {
|
||||
return h.Node, false, nil
|
||||
}
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, h.Node)
|
||||
if err != nil {
|
||||
// It cannot be worked out: sending it would refuse the whole send, and `plan` says why.
|
||||
return h.Node, false, nil
|
||||
}
|
||||
list, _, err := busUserList(ctx, inv, plan.Modules)
|
||||
if err != nil {
|
||||
return h.Node, false, nil
|
||||
}
|
||||
sent, err := inv.SentBusUsers(ctx, h.Node)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return h.Node, userListBehind(list, sent), nil
|
||||
}
|
||||
|
||||
// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
|
||||
// empty list composed is never behind: there is nothing for it to carry.
|
||||
func userListBehind(now, sentDigest string) bool {
|
||||
return now != "" && digestOf([]byte(now)) != sentDigest
|
||||
}
|
||||
|
||||
// couldNotBeResolved is what a push ends with when some machines could not be worked out.
|
||||
//
|
||||
// **After the rest have been sent, never instead of sending them.** It is still an error, because
|
||||
@@ -640,29 +878,48 @@ func couldNotBeResolved(refusals []string, sent int) error {
|
||||
// consumer and refused on the provider would leave one end holding a credential the other has
|
||||
// never heard of — which is the state this whole mechanism exists to make impossible.
|
||||
func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
_, err := sendToEach(ctx, open, names)
|
||||
return err
|
||||
}
|
||||
|
||||
// sendToEach is sendTo, answering the machines it sent: those named, and before them the machine
|
||||
// holding the bus when its user list must go first (novox/hq issue 249) — so a caller that waits for
|
||||
// the machines it sent waits for that one too.
|
||||
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
inv := open.inventory
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
holder, behind, err := brokerBehind(ctx, open, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = brokerFirst(names, holder, behind)
|
||||
|
||||
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
||||
// converge, which flips the node and then sends it — is not made to wait on itself.
|
||||
ctx, release, err := holdNodes(ctx, open, names)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
|
||||
seq, err := allot(ctx, inv, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
plan, settings, err := planFor(ctx, open, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
@@ -674,44 +931,34 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence = seq
|
||||
reportLeftOut(name, declared)
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
if len(refusals) > 0 {
|
||||
return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
||||
return nil, fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
||||
len(refusals), strings.Join(refusals, "\n\n"))
|
||||
}
|
||||
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
for _, s := range sending {
|
||||
if err := number(ctx, inv, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
}
|
||||
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
|
||||
// same records the bus's accounts are, so what a runtime serves and what its account may are one
|
||||
// composition. Issued after the declaration, because the runtime it is for arrives with it.
|
||||
return issueMemberships(ctx, open, server, sending)
|
||||
// composition. **Issued before the declarations** (novox/hq issue 249): the runtime the
|
||||
// membership is for arrives with the declaration, and a membership waits for it on the bus; the
|
||||
// code arriving first was refused its own state until somebody pushed.
|
||||
if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: " "}, holder, sending); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent := make([]string, 0, len(sending))
|
||||
for _, s := range sending {
|
||||
sent = append(sent, s.node)
|
||||
}
|
||||
return sent, nil
|
||||
}
|
||||
|
||||
// issueMemberships publishes the membership of every module on the machines just sent.
|
||||
@@ -729,11 +976,26 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
// The declarations are sent and recorded by now; a membership that cannot be issued is said
|
||||
// and does not unsay them. Every runtime without one serves the shape it derives (ADR 0160), so
|
||||
// the push stands, the first failure is named once, and the next push tries again.
|
||||
issued, failed := 0, 0
|
||||
var first error
|
||||
// **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The
|
||||
// raise at start asserts them too, but a module registered and assigned since would otherwise have
|
||||
// its bucket only after the control plane next restarts — found the first time a module declared
|
||||
// state: its bundle asked for a bucket that did not exist. Idempotent and cheap.
|
||||
//
|
||||
// **A failure here is the send's failure** (novox/hq issue 249). It was said and the push stood,
|
||||
// because the declarations were already away; they are sent after this now — all but the bus's
|
||||
// own machine, sent before it (deliver) — and a module whose state does not exist is a module
|
||||
// that fails its start, so they are not sent and the caller tries again rather than reporting the
|
||||
// rollout done.
|
||||
buckets, err := open.inventory.DeclaredBuckets(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the modules' state could not be read, so no bucket was asserted: %w", err)
|
||||
}
|
||||
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// Every membership is tried, and the first failure named once.
|
||||
issued := 0
|
||||
refused := map[string]error{}
|
||||
for _, s := range sent {
|
||||
node := s.node
|
||||
for _, d := range records.Assigned[node] {
|
||||
@@ -754,10 +1016,9 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
||||
return err
|
||||
}
|
||||
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
||||
if first == nil {
|
||||
first = err
|
||||
if refused[node] == nil {
|
||||
refused[node] = fmt.Errorf("%s: %w", d.Module, err)
|
||||
}
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
issued++
|
||||
@@ -766,9 +1027,10 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
||||
if issued > 0 {
|
||||
fmt.Printf(" issued %d membership(s)\n", issued)
|
||||
}
|
||||
if failed > 0 {
|
||||
fmt.Printf(" %d membership(s) could not be issued; the first: %v — the machines keep what "+
|
||||
"they derive until the next push\n", failed, first)
|
||||
if len(refused) > 0 {
|
||||
// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
|
||||
// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
|
||||
return &grantsRefused{nodes: refused}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -880,6 +1142,26 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
|
||||
return err
|
||||
}
|
||||
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
|
||||
// whether it was cancelled the moment it took it.
|
||||
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
||||
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
||||
// nothing declares any more is said and kept — what it holds is data.
|
||||
buckets, err := inv.DeclaredBuckets(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
undeclared, err := broker.RaiseBuckets(js, buckets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(undeclared) > 0 {
|
||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||
}
|
||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
@@ -893,18 +1175,14 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
return err
|
||||
}
|
||||
hearing := 0
|
||||
for _, p := range users {
|
||||
consumer, needed := broker.ConsumerFor(p)
|
||||
if !needed {
|
||||
continue
|
||||
}
|
||||
if err := js.EnsureConsumer(consumer); err != nil {
|
||||
return fmt.Errorf("how %s on %s hears what it consumes: %w", p.Module, p.Node, err)
|
||||
for _, c := range broker.ConsumersOf(users) {
|
||||
if err := js.EnsureConsumer(c.Consumer); err != nil {
|
||||
return fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
|
||||
}
|
||||
hearing++
|
||||
}
|
||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, and %d module(s) "+
|
||||
"can hear what they consume\n", broker.BareAddress(address), len(names), hearing)
|
||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -944,15 +1222,59 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
|
||||
}
|
||||
|
||||
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
|
||||
func number(ctx context.Context, inv *inventory.Inventory, s *readyNode) error {
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
// allotting is allot over one inventory, in the shape composeEach takes.
|
||||
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (int64, error) {
|
||||
return func(node string) (int64, error) { return allot(ctx, inv, node) }
|
||||
}
|
||||
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries.
|
||||
func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, error) {
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
seq, err := inv.NextSequence(ctx, record.ID)
|
||||
return inv.NextSequence(ctx, record.ID)
|
||||
}
|
||||
|
||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||
//
|
||||
// **On a context that outlives the caller's** (novox/hq issue 204). The record is written after the
|
||||
// declaration is away, so a send that failed is never recorded as current — and a controller being
|
||||
// replaced mid-send had its context cancelled between the two, so the machine was told and the mesh
|
||||
// never wrote it down: status read "applied, current" over a machine that had just been sent
|
||||
// something else. What was sent was sent; the record of it must not depend on the sender living
|
||||
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte) (string, error) {
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
record, err := inv.NodeByName(kept, node)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
s.declared.Sequence = seq
|
||||
return nil
|
||||
digest := digestOf(body)
|
||||
if err := inv.RecordSent(kept, record.ID, digest); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// reportUnheldPushed says what a push's machines lack of the seats their modules depend on
|
||||
// (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody
|
||||
// is looking at, and one line per machine otherwise — a list per machine across the mesh is the
|
||||
// hundred lines that buried the one that mattered. Nothing for a machine that lacks nothing.
|
||||
func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[string][]catalogue.Unheld) {
|
||||
for _, node := range asked {
|
||||
lines := unheld[node]
|
||||
if len(lines) == 0 {
|
||||
continue
|
||||
}
|
||||
if named {
|
||||
fmt.Fprintf(w, "\n%s has %d unmet seat dependenc(ies) (novox/hq ADR 0207):\n", node, len(lines))
|
||||
for _, u := range lines {
|
||||
fmt.Fprintf(w, " %s\n", u)
|
||||
}
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
// the wrong machine no longer refuses the whole node), applied one level up.
|
||||
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
sending, refusals := composeEach(
|
||||
[]string{"anchor", "home-server", "laptop"},
|
||||
[]string{"anchor", "home-server", "laptop"}, numbered(),
|
||||
func(node string) (sendable, error) {
|
||||
if node == "anchor" {
|
||||
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||
@@ -43,7 +43,7 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||
sending, refusals := composeEach([]string{"spare"},
|
||||
sending, refusals := composeEach([]string{"spare"}, numbered(),
|
||||
func(string) (sendable, error) { return sendable{}, nil })
|
||||
if len(sending) != 1 || len(refusals) != 0 {
|
||||
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||
@@ -74,3 +74,9 @@ func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||
func numbered() func(string) (int64, error) {
|
||||
var n int64
|
||||
return func(string) (int64, error) { n++; return n, nil }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,709 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
//
|
||||
// Seen whole — what waits, what runs where and for how long, what was handed out as often as it
|
||||
// may be and never settled — and changed through the controller: an ask cancelled, the queue
|
||||
// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's
|
||||
// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`).
|
||||
//
|
||||
// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that
|
||||
// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
|
||||
// rather than waiting for ever on an answer nobody will give.
|
||||
|
||||
// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
|
||||
// than the holder's worst case (its two passes removing containers and its wait between, 50s).
|
||||
const (
|
||||
holderAsks = 15 * time.Second
|
||||
killAnswer = 75 * time.Second
|
||||
)
|
||||
|
||||
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
|
||||
func dialTheBus() (*broker.JetStream, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
return js, nil
|
||||
}
|
||||
|
||||
// queueCommand prints every ask in the build seat's work queue.
|
||||
func queueCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("queue", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "the queue as JSON")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(q, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
fmt.Print(queueText(q, time.Now()))
|
||||
return nil
|
||||
}
|
||||
|
||||
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
|
||||
// Never what an ask carries as `held` — that is every artifact the mesh has built.
|
||||
func queueText(q link.Queue, now time.Time) string {
|
||||
var b strings.Builder
|
||||
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||
fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead))
|
||||
ago := func(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "asked at an unknown time"
|
||||
}
|
||||
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
|
||||
}
|
||||
what := func(a link.QueuedAsk) string {
|
||||
s := a.Repository
|
||||
if a.Path != "" {
|
||||
s += " at " + a.Path
|
||||
}
|
||||
if a.Ref != "" {
|
||||
s += " on " + a.Ref
|
||||
}
|
||||
return s
|
||||
}
|
||||
if len(running) > 0 {
|
||||
fmt.Fprintln(&b, "\nin flight:")
|
||||
for _, a := range running {
|
||||
where := "taken, not yet said where"
|
||||
switch {
|
||||
case a.On != "":
|
||||
where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
fmt.Fprintln(&b, "\nwaiting:")
|
||||
for _, a := range waiting {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(dead) > 0 {
|
||||
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
|
||||
for _, a := range dead {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(q.Asks) == 0:
|
||||
fmt.Fprintln(&b, "nothing is asked of it")
|
||||
default:
|
||||
fmt.Fprintln(&b, "\n`cancel <id>` drops a waiting or dead ask, `clear` every waiting one, `kill <id>` ends one in flight")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// cancelCommand drops one waiting or dead ask.
|
||||
func cancelCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("cancel <build id>")
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ask, found := q.Find(args[0])
|
||||
if !found {
|
||||
return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+
|
||||
"what came of it", args[0], seat)
|
||||
}
|
||||
said, err := cancelAsk(ctx, js, open, seat, ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
|
||||
//
|
||||
// **In this order, and each step for a reason** (novox/hq ADR 0219):
|
||||
// 1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
|
||||
// is running — that is `kill`, on the machine running it;
|
||||
// 2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
|
||||
// 3. for a waiting ask, the worker is read again: one handed out since the queue was read was
|
||||
// taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
|
||||
// read the mark first and ends it as cancelled, or is building it;
|
||||
// 4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
|
||||
// handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
|
||||
// at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
|
||||
// that took it before the mark is building it, and only `kill` ends that;
|
||||
// 5. the message is deleted by its sequence;
|
||||
// 6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
|
||||
func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
|
||||
if ask.State == link.AskInFlight && ask.On != "" {
|
||||
return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
|
||||
"ends the build where it runs", ask.ID, ask.On, ask.ID)
|
||||
}
|
||||
if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
withdraw := func() {
|
||||
if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
|
||||
}
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
switch ask.State {
|
||||
case link.AskWaiting:
|
||||
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||
withdraw()
|
||||
return "", err
|
||||
} else if taken {
|
||||
withdraw()
|
||||
return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
|
||||
"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
|
||||
"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
|
||||
}
|
||||
case link.AskInFlight:
|
||||
if started, err := startedSince(ctx, js, seat, ask); err != nil {
|
||||
withdraw()
|
||||
return "", err
|
||||
} else if started != "" {
|
||||
withdraw()
|
||||
return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
|
||||
"ends it there", ask.ID, started, ask.ID)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
|
||||
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
|
||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||
}
|
||||
recordCancelled(ctx, open, ask)
|
||||
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
|
||||
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
|
||||
}
|
||||
|
||||
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
|
||||
// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
|
||||
var holderLooks = 5 * time.Second
|
||||
|
||||
// startedSince waits out a holder's look at the cancelled set and says the machine that started the
|
||||
// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
|
||||
// a start of a build.
|
||||
func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(holderLooks):
|
||||
}
|
||||
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||
if !ask.AskedAt.IsZero() {
|
||||
since = ask.AskedAt.Add(-time.Minute)
|
||||
}
|
||||
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
s, ok := heard.Started[ask.ID]
|
||||
if !ok || heard.Outcomes[ask.ID] {
|
||||
return "", nil
|
||||
}
|
||||
at, _ := time.Parse(time.RFC3339Nano, s.At)
|
||||
if ask.Started.IsZero() || at.After(ask.Started) {
|
||||
return s.On, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// takenSince is whether the worker has handed out the ask at this sequence.
|
||||
func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
|
||||
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
|
||||
if errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot read the worker of the queue again: %w", err)
|
||||
}
|
||||
return info.Delivered.Stream >= seq, nil
|
||||
}
|
||||
|
||||
// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded,
|
||||
// then the plan that asked for it — by the id it asked with, or by repository and path.
|
||||
func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) {
|
||||
r := ask.Request
|
||||
result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref,
|
||||
Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand}
|
||||
_ = builds{open.inventory, open}.Built(ctx, result)
|
||||
}
|
||||
|
||||
// clearCommand cancels every waiting ask, and with --dead every dead one too.
|
||||
func clearCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("clear", flag.ContinueOnError)
|
||||
dead := set.Bool("dead", false, "the dead asks too")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
said, err := clearQueue(ctx, js, open, seat, *dead)
|
||||
fmt.Print(said)
|
||||
return err
|
||||
}
|
||||
|
||||
// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight.
|
||||
func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) {
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var b strings.Builder
|
||||
cancelled, refused := 0, 0
|
||||
for _, a := range q.Asks {
|
||||
if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) {
|
||||
continue
|
||||
}
|
||||
said, err := cancelAsk(ctx, js, open, seat, a)
|
||||
if err != nil {
|
||||
refused++
|
||||
fmt.Fprintf(&b, " %s: %v\n", a.ID, err)
|
||||
continue
|
||||
}
|
||||
cancelled++
|
||||
fmt.Fprintf(&b, " %s\n", said)
|
||||
}
|
||||
what := "waiting"
|
||||
if dead {
|
||||
what = "waiting and dead"
|
||||
}
|
||||
fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what)
|
||||
if refused > 0 {
|
||||
fmt.Fprintf(&b, ", %d could not be", refused)
|
||||
}
|
||||
fmt.Fprintln(&b)
|
||||
if n := len(q.Of(link.AskInFlight)); n > 0 {
|
||||
fmt.Fprintf(&b, "%d in flight, left running: `kill <id>` ends one where it runs\n", n)
|
||||
}
|
||||
if n := len(q.Of(link.AskDead)); n > 0 && !dead {
|
||||
fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n)
|
||||
}
|
||||
if refused > 0 {
|
||||
return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused)
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// rebuildCommand asks the module's current source again — or, given a build's id, that build's
|
||||
// repository, path and ref — under a new id.
|
||||
func rebuildCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("rebuild <module | build id>")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var source buildSource
|
||||
var path, ref, module string
|
||||
if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
source, module = sourceOfBuild(b, entries)
|
||||
path, ref = b.Path, b.Ref
|
||||
// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
|
||||
// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
|
||||
// that commit out over everything since. Building that commit again is `replay`, which says
|
||||
// what it would do and refuses to register it while anything newer is asked or registered.
|
||||
if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
|
||||
ref = followedBranch(e.Source.Ref)
|
||||
follows := ref
|
||||
if follows == "" {
|
||||
follows = "the repository's default branch"
|
||||
}
|
||||
fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
|
||||
"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
|
||||
}
|
||||
} else if e, known := entryNamed(entries, args[0]); known {
|
||||
// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
|
||||
source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
|
||||
} else {
|
||||
return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
|
||||
}
|
||||
|
||||
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
|
||||
// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
|
||||
if module != "" {
|
||||
joined, said, err := joinAPlan(ctx, open, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if joined {
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
id, err := askABuild(ctx, source, path, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("rebuild asked as %s\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// entryNamed is the catalogue's entry for a module.
|
||||
func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == name {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Entry{}, false
|
||||
}
|
||||
|
||||
// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
|
||||
// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
|
||||
// (ADR 0111) — else the repository as it was cloned.
|
||||
func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) {
|
||||
for _, e := range entries {
|
||||
if (b.Module != "" && e.Manifest.Module == b.Module) ||
|
||||
(b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) {
|
||||
return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module
|
||||
}
|
||||
}
|
||||
return buildSource{Repository: b.Repository}, b.Module
|
||||
}
|
||||
|
||||
// replayCommand asks a recorded build's repository and path again at the commit it built.
|
||||
func replayCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("replay", flag.ContinueOnError)
|
||||
register := set.Bool("register", false, "register what it builds, as any build is")
|
||||
older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("replay <build id> [--register [--older]]")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
b, found, err := inv.BuildByID(ctx, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no build %s is recorded", positionals[0])
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, module := sourceOfBuild(b, entries)
|
||||
var history []inventory.Build
|
||||
if module != "" {
|
||||
if history, err = inv.Builds(ctx, module, 100); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// What is asked of the module and not yet answered, when the replay would be registered.
|
||||
var outstanding []string
|
||||
if *register {
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
|
||||
js.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
|
||||
}
|
||||
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(replaySaid(b, module, id, *register))
|
||||
return nil
|
||||
}
|
||||
|
||||
// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
|
||||
//
|
||||
// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
|
||||
// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
|
||||
// older commit is newer than everything since, and would roll that older commit out as the module's
|
||||
// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
|
||||
// --register says otherwise, and --register is refused when a newer build of a different commit is
|
||||
// registered, unless --older says that is the point.
|
||||
//
|
||||
// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
|
||||
// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
|
||||
// and their builds — made from newer source — would be recorded and never registered (issue 219
|
||||
// orders by ask), the plan waiting on them sending the older commit instead.
|
||||
func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
|
||||
outstanding []string) error {
|
||||
if b.Commit == "" {
|
||||
return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
|
||||
"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
|
||||
}
|
||||
if older && !register {
|
||||
return errors.New("--older only says what --register may do; a dry run registers nothing")
|
||||
}
|
||||
if register && len(outstanding) > 0 {
|
||||
return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
|
||||
"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
|
||||
orNone(module), strings.Join(outstanding, "; "), b.ID)
|
||||
}
|
||||
if !register || older {
|
||||
return nil
|
||||
}
|
||||
for _, h := range history {
|
||||
if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
|
||||
continue
|
||||
}
|
||||
if h.AskedOrAt().After(b.AskedOrAt()) {
|
||||
return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
|
||||
"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
|
||||
"without --register looks at it; --register --older registers it anyway",
|
||||
module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome.
|
||||
func replaySaid(b inventory.Build, module, id string, register bool) string {
|
||||
what := b.Repository
|
||||
if module != "" {
|
||||
what = module
|
||||
}
|
||||
said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id)
|
||||
if !register {
|
||||
return said + "\n a dry run: the outcome is looked at and not taken in — nothing is recorded or " +
|
||||
"registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it"
|
||||
}
|
||||
return said + "\n registered when it is built, as the module's current version — newer than every build " +
|
||||
"asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it"
|
||||
}
|
||||
|
||||
// killCommand finds the machine running a build and asks its holder to end it.
|
||||
func killCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("kill <build id>")
|
||||
}
|
||||
id := args[0]
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||
if at, ok := link.BuildAskedAt(id); ok {
|
||||
since = at.Add(-time.Minute)
|
||||
}
|
||||
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started, ok := heard.Started[id]
|
||||
if !ok {
|
||||
return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id)
|
||||
}
|
||||
if heard.Outcomes[id] {
|
||||
return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
|
||||
}
|
||||
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return printHolderAnswer(started.On, answer)
|
||||
}
|
||||
|
||||
// pauseCommand asks one machine's holder, or every holder's, to pause or resume.
|
||||
func pauseCommand(ctx context.Context, verb string, args []string) error {
|
||||
if len(args) > 1 {
|
||||
return fmt.Errorf("%s [node]", verb)
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
nodes := args
|
||||
if len(nodes) == 0 {
|
||||
if nodes, err = buildSeatHolders(ctx, seat); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb)
|
||||
}
|
||||
}
|
||||
failed := 0
|
||||
for _, node := range nodes {
|
||||
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks)
|
||||
if err != nil {
|
||||
fmt.Printf("%s: %v\n", node, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
if err := printHolderAnswer(node, answer); err != nil {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printHolderAnswer prints what a holder said, or its refusal as the command's failure.
|
||||
func printHolderAnswer(node string, answer link.Answer) error {
|
||||
if answer.Error != "" {
|
||||
fmt.Printf("%s: %s\n", node, answer.Error)
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
var said struct {
|
||||
Said string `json:"said"`
|
||||
}
|
||||
if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" {
|
||||
fmt.Printf("%s: %s\n", node, said.Said)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s: %s\n", node, string(answer.Result))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildSeatHolders is every machine an assigned module holding the build seat runs on.
|
||||
func buildSeatHolders(ctx context.Context, seat string) ([]string, error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return holdersAmong(entries, seat), nil
|
||||
}
|
||||
|
||||
// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each.
|
||||
func holdersAmong(entries []inventory.Entry, seat string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if !e.Manifest.ClaimsSeat(seat) {
|
||||
continue
|
||||
}
|
||||
for _, n := range e.On {
|
||||
if !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
|
||||
// by repository and path, and every open plan holding it not yet built.
|
||||
func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
|
||||
var out []string
|
||||
for _, a := range q.Asks {
|
||||
if repositoryMatches(a.Repository, repository) && a.Path == path {
|
||||
out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
|
||||
}
|
||||
}
|
||||
if module == "" {
|
||||
return out
|
||||
}
|
||||
for _, p := range plans {
|
||||
if !p.Open() {
|
||||
continue
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
if m == module {
|
||||
if st := p.Modules[m]; st == nil || st.State != "built" {
|
||||
out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,772 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
|
||||
//
|
||||
// docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js
|
||||
// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14294 \
|
||||
// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
|
||||
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused'
|
||||
|
||||
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
|
||||
func asksRecorded(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond))
|
||||
asked = append(asked, source.Repository+"#"+id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
return &asked
|
||||
}
|
||||
|
||||
// twoTiers registers two modules, b standing on a, each with a source a plan asks.
|
||||
func twoTiers(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"a", "b"} {
|
||||
if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A failed plan is retried: its failed module asked again under a new id, the plan building at that
|
||||
// tier, and when that build comes in the plan goes on and asks its next tier.
|
||||
func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.KilledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said, err := retryPlan(ctx, open, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := p.Modules["a"]
|
||||
if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
|
||||
t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
|
||||
}
|
||||
if !strings.Contains(said, a.Build) {
|
||||
t.Errorf("retry does not say the new id: %q", said)
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
|
||||
// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
|
||||
planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
|
||||
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
|
||||
t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
|
||||
}
|
||||
asking, _ := link.BuildAskedAt(a.Build)
|
||||
planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
|
||||
p, err = open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
|
||||
t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// What retry refuses, and says why.
|
||||
func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
plan := func(id, state string, created time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee",
|
||||
Created: created, State: state, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||
}
|
||||
failed := plan("plan-1", inventory.PlanFailed, at)
|
||||
for _, c := range []struct {
|
||||
p inventory.Plan
|
||||
others []inventory.Plan
|
||||
says string
|
||||
}{
|
||||
{plan("plan-d", inventory.PlanDone, at), nil, "is done"},
|
||||
{plan("plan-s", inventory.PlanSuperseded, at), nil, "was"},
|
||||
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
||||
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
||||
} {
|
||||
err := retryRefusal(c.p, c.others)
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
||||
}
|
||||
}
|
||||
stopped := failed
|
||||
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
stopped.Note = "a stopped at its first machine"
|
||||
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||
}
|
||||
// Another branch's newer plan, an older one, and a failed one do not supersede it.
|
||||
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||
other.Branch = "release"
|
||||
if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
|
||||
plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
|
||||
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
|
||||
func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
|
||||
t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
|
||||
if err := rebuildCommand(ctx, []string{"b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
|
||||
t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
|
||||
func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
|
||||
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
|
||||
t.Fatal("joined a failed plan a newer open one supersedes")
|
||||
}
|
||||
if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
|
||||
t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
|
||||
}
|
||||
built := failed
|
||||
built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
|
||||
t.Fatal("joined a plan that has the module built")
|
||||
}
|
||||
}
|
||||
|
||||
// replay is a dry run unless registered, and registering an older commit than one registered since
|
||||
// is refused unless --older says it is meant (novox/hq issue 207).
|
||||
func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
|
||||
newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
|
||||
history := []inventory.Build{newer, old}
|
||||
|
||||
if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
|
||||
t.Errorf("a dry run was refused: %v", err)
|
||||
}
|
||||
err := replayRefusal(old, "a", history, true, false, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
|
||||
t.Errorf("registering an older commit than the one registered was not refused: %v", err)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
|
||||
t.Errorf("--register --older was refused: %v", err)
|
||||
}
|
||||
if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
|
||||
t.Errorf("registering the newest build again was refused: %v", err)
|
||||
}
|
||||
// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
|
||||
same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
|
||||
broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
|
||||
if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
|
||||
t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
|
||||
}
|
||||
if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
|
||||
t.Error("a build that recorded no commit was replayed")
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
|
||||
t.Error("--older without --register was taken")
|
||||
}
|
||||
// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
|
||||
// holding it unbuilt, would be replaced by a replay asked now (issue 219).
|
||||
q := link.Queue{Asks: []link.QueuedAsk{
|
||||
{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
|
||||
{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
|
||||
{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
|
||||
}}
|
||||
plans := []inventory.Plan{
|
||||
{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
|
||||
{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
}
|
||||
outstanding := outstandingFor("a", "novox/a", "", q, plans)
|
||||
if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
|
||||
t.Fatalf("outstanding: %v", outstanding)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
|
||||
t.Errorf("registered over an outstanding ask: %v", err)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
|
||||
t.Errorf("a dry run was refused for what is outstanding: %v", err)
|
||||
}
|
||||
if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
|
||||
t.Errorf("a dry replay does not say what it is: %q", said)
|
||||
}
|
||||
if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
|
||||
t.Errorf("a registered replay does not say what it does: %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
|
||||
// paused on some holders only is not a reason the plan is waiting.
|
||||
func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
||||
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
asked := now.Add(-12 * time.Minute)
|
||||
p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding,
|
||||
Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||
|
||||
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
|
||||
line := planLineWith(p, now, all)
|
||||
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan on a paused seat reads %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
||||
t.Errorf("status --json says %+v", st)
|
||||
}
|
||||
if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
|
||||
t.Errorf("a plan waiting on a paused seat counted late")
|
||||
}
|
||||
|
||||
longAgo := now.Add(-25 * time.Hour)
|
||||
forgotten := p
|
||||
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
|
||||
if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan paused over a day reads %q", line)
|
||||
}
|
||||
|
||||
some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
|
||||
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
|
||||
t.Fatalf("%+v", some)
|
||||
}
|
||||
if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
|
||||
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
|
||||
t.Errorf("status --json: %+v", st)
|
||||
}
|
||||
// A plan rolling out, or with nothing asked, is not waiting on the seat.
|
||||
rolling := p
|
||||
rolling.State = inventory.PlanRolling
|
||||
if _, paused := pausedWaiting(rolling, all, now); paused {
|
||||
t.Error("a rolling plan reads as waiting on the build seat")
|
||||
}
|
||||
}
|
||||
|
||||
// The queue's verbs are the controller seat's, each to the command it names.
|
||||
func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{"queue", nil, []string{"queue"}},
|
||||
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
||||
{"clear", nil, []string{"clear"}},
|
||||
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
||||
{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
|
||||
{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
|
||||
{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
|
||||
{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
|
||||
{"pause", nil, []string{"pause"}},
|
||||
{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}},
|
||||
} {
|
||||
got, err := argvFor(c.verb, c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("cancel", nil); err == nil {
|
||||
t.Error("cancel without an id was taken")
|
||||
}
|
||||
declared := map[string]bool{}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
declared[v.Name] = true
|
||||
}
|
||||
for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} {
|
||||
if !declared[v] {
|
||||
t.Errorf("%s is not a verb of the controller seat", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- against a real bus -----------------------------------------------------------------------
|
||||
|
||||
// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller
|
||||
// pointed at it, and a function that asks the seat one build.
|
||||
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
|
||||
Emits: []string{"started", "built", "log.*", "paused.*"}}
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||
if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
|
||||
link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||
_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
|
||||
_ = js.Context().PurgeStream(broker.EventsStream)
|
||||
})
|
||||
ask := func(id, repository string) link.BuildRequest {
|
||||
r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
|
||||
body, _ := json.Marshal(r)
|
||||
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
return js, ask
|
||||
}
|
||||
|
||||
// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows.
|
||||
func deadOne(t *testing.T, js *broker.JetStream) {
|
||||
t.Helper()
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
for i := 0; i < worker.MaxDeliver; i++ {
|
||||
msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("delivery %d: %v", i+1, err)
|
||||
}
|
||||
_ = msgs[0].Nak()
|
||||
}
|
||||
// One more pull, as a holder always has one waiting: the server finds the ask past its deliveries
|
||||
// then, and stops counting it pending.
|
||||
if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 {
|
||||
t.Fatalf("an ask past its deliveries was delivered again")
|
||||
}
|
||||
}
|
||||
|
||||
// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan
|
||||
// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for
|
||||
// is still found.
|
||||
func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
twoTiers(t, open)
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
asked, _ := link.BuildAskedAt(id)
|
||||
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
|
||||
t.Fatalf("the queue reads %+v", q)
|
||||
}
|
||||
if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
|
||||
strings.Contains(text, "secret-ish") {
|
||||
t.Errorf("the queue says:\n%s", text)
|
||||
}
|
||||
|
||||
if err := cancelCommand(ctx, []string{id}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||
t.Fatalf("the ask is still queued: %+v", q.Asks)
|
||||
}
|
||||
if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
|
||||
t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
|
||||
}
|
||||
b, found, err := open.inventory.BuildByID(ctx, id)
|
||||
if err != nil || !found || b.Failed != link.CancelledByHand {
|
||||
t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, plan.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
|
||||
t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
|
||||
}
|
||||
if err := cancelCommand(ctx, []string{id}); err == nil {
|
||||
t.Error("an ask cancelled already was cancelled again")
|
||||
}
|
||||
}
|
||||
|
||||
// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight.
|
||||
func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
start := time.Now()
|
||||
dead := link.NewBuildID(start)
|
||||
ask(dead, "https://forge.example/novox/dead.git")
|
||||
deadOne(t, js)
|
||||
var waiting []string
|
||||
for i := 1; i <= 2; i++ {
|
||||
id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
|
||||
waiting = append(waiting, id)
|
||||
ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
|
||||
t.Fatalf("the queue reads %+v", q)
|
||||
}
|
||||
said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
|
||||
t.Errorf("clear said:\n%s", said)
|
||||
}
|
||||
q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
|
||||
t.Fatalf("after clear the queue is %+v", q.Asks)
|
||||
}
|
||||
if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||
t.Fatalf("clear --dead left %+v", q.Asks)
|
||||
}
|
||||
for _, id := range append(waiting, dead) {
|
||||
if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
|
||||
t.Errorf("%s is recorded as %+v", id, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An ask in flight is not cancelled: kill ends it where it runs.
|
||||
func TestCancelRefusesAnAskInFlight(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := q.Find(id)
|
||||
if a.State != link.AskInFlight || a.On != "ace" {
|
||||
t.Fatalf("the taken ask reads %+v", a)
|
||||
}
|
||||
_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
|
||||
if err == nil || !strings.Contains(err.Error(), "kill "+id) {
|
||||
t.Fatalf("an ask in flight was cancelled: %v", err)
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||
t.Error("a refused cancel recorded an outcome")
|
||||
}
|
||||
}
|
||||
|
||||
// kill finds the machine from the build's start and asks that machine's holder; pause asks the
|
||||
// machine named. Each prints what the holder answered, and a refusal is the command's failure.
|
||||
func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
||||
js, _ := aBuildQueue(t)
|
||||
ctx := t.Context()
|
||||
asked := map[string]string{}
|
||||
handlers := map[string]link.ToolHandler{
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct{ ID string }
|
||||
_ = json.Unmarshal(raw, &args)
|
||||
asked["kill"] = args.ID
|
||||
if args.ID != "build-running" {
|
||||
return nil, fmt.Errorf("ace is not building %s", args.ID)
|
||||
}
|
||||
return map[string]any{"said": "killed " + args.ID}, nil
|
||||
},
|
||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||
asked["pause"] = "ace"
|
||||
return map[string]any{"said": "ace is paused"}, nil
|
||||
},
|
||||
}
|
||||
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
for _, id := range []string{"build-running", "build-other"} {
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-running"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked["kill"] != "build-running" {
|
||||
t.Fatalf("the holder was asked %v", asked)
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-other"}); err == nil {
|
||||
t.Error("the holder's refusal was not the command's")
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
|
||||
t.Errorf("a build nobody started: %v", err)
|
||||
}
|
||||
if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
|
||||
t.Fatalf("pause: %v %v", err, asked)
|
||||
}
|
||||
if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
|
||||
t.Error("a machine nothing answers on was resumed")
|
||||
}
|
||||
}
|
||||
|
||||
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
var sentTo [][]string
|
||||
was := sendRollout
|
||||
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||
sentTo = append(sentTo, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
|
||||
long := time.Now().UTC().Add(-2 * time.Hour)
|
||||
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
if err := open.inventory.SavePlan(ctx, stopped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A newer plan holding a refuses it: sending the older build would put it back.
|
||||
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
|
||||
t.Fatalf("retried under a newer plan: %v", err)
|
||||
}
|
||||
newer.State = inventory.PlanSuperseded
|
||||
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said, err := retryPlan(ctx, open, stopped.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
|
||||
t.Fatalf("sent %v; said %q", sentTo, said)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, stopped.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := p.Modules["a"]
|
||||
if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
|
||||
t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
|
||||
}
|
||||
// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
|
||||
advancePlans(ctx, open)
|
||||
if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
|
||||
t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
|
||||
// the plan, asked later, never answers it (novox/hq ADR 0219).
|
||||
func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := time.Now().UTC().Add(-time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
|
||||
p, _ := open.inventory.PlanByID(ctx, plan.ID)
|
||||
if p.Modules["a"].State != "asked" {
|
||||
t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
|
||||
}
|
||||
// From the records too: a later build of the module recorded is not the plan's.
|
||||
recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
|
||||
if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
|
||||
t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
|
||||
}
|
||||
planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
|
||||
if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
|
||||
t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
|
||||
}
|
||||
}
|
||||
|
||||
// rebuild of a build made at a commit asks what the module follows now, never the commit.
|
||||
func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
var refs []string
|
||||
was := askABuild
|
||||
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||
refs = append(refs, ref)
|
||||
return was(c, source, path, ref)
|
||||
}
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
|
||||
Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
|
||||
t.Fatalf("asked %v at %v", *asked, refs)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask the worker counts out that no machine said it started is cancelled only if no start comes
|
||||
// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
|
||||
func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
was := holderLooks
|
||||
holderLooks = 300 * time.Millisecond
|
||||
t.Cleanup(func() { holderLooks = was })
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := q.Find(id)
|
||||
if a.State != link.AskInFlight || a.On != "" {
|
||||
t.Fatalf("the taken ask reads %+v", a)
|
||||
}
|
||||
// The holder that took it says it started, while the cancel waits.
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
_, _ = js.Context().Publish(link.BuildStarted(), started)
|
||||
}()
|
||||
if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
|
||||
t.Fatalf("a build that started was cancelled: %v", err)
|
||||
}
|
||||
if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
|
||||
t.Error("the withdrawn cancel is still marked")
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||
t.Error("a withdrawn cancel recorded an outcome")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -73,6 +74,10 @@ type meshStatus struct {
|
||||
// alone. A document without this called a machine well while a predecessor's chain refused
|
||||
// what the mesh declared open.
|
||||
Filtered []machineFiltered `json:"filtered,omitempty"`
|
||||
// Unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||
// dependency is met. Reported, not refused, until the switch.
|
||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
@@ -171,7 +176,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now())}
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
|
||||
// In a stated order, so two readings of an unchanged mesh are the same document.
|
||||
untakenNodes := make([]string, 0, len(asked.untaken))
|
||||
for name := range asked.untaken {
|
||||
@@ -204,6 +209,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
@@ -36,17 +37,29 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string {
|
||||
for _, m := range set {
|
||||
deps[m] = map[string]bool{}
|
||||
}
|
||||
// The build seat's holders follow the controller that defines their worker (EdgeWorkerOf,
|
||||
// novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the
|
||||
// controller is built by whichever build machine is running, as the runtime image always was.
|
||||
worker := map[string]map[string]bool{}
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] {
|
||||
if worker[e.To] == nil {
|
||||
worker[e.To] = map[string]bool{}
|
||||
}
|
||||
worker[e.To][e.From] = true
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
// A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's
|
||||
// build needs nothing of A's first. The other kinds order: stands-on and declared after
|
||||
// the base is built, built-by after the build machine is built and running — except for
|
||||
// what the build machine itself stands on. The runtime image is built by the builder and
|
||||
// the builder is built on the runtime image; the image comes first, built by the builder
|
||||
// that is running, which is the only one there could be.
|
||||
// what the build machine itself stands on, and for the controller whose worker the build
|
||||
// machine binds. The runtime image is built by the builder and the builder is built on the
|
||||
// runtime image; the image comes first, built by the builder that is running.
|
||||
if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages {
|
||||
continue
|
||||
}
|
||||
if e.Kind == inventory.EdgeBuiltBy && isBaseOf(e.From, e.To, edges, in) {
|
||||
if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) {
|
||||
continue
|
||||
}
|
||||
deps[e.From][e.To] = true
|
||||
@@ -122,7 +135,10 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgeBuiltBy {
|
||||
// Built-by and worker-of order a plan; neither widens it. A new build machine changes
|
||||
// nothing it builds, and a new controller changes nothing about the holder it orders —
|
||||
// what packages the controller's source is already a code edge.
|
||||
if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf {
|
||||
continue
|
||||
}
|
||||
if in[e.To] && !in[e.From] {
|
||||
@@ -168,6 +184,7 @@ func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inv
|
||||
return inventory.Plan{
|
||||
ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()),
|
||||
Repository: m.Owner + "/" + m.Repo,
|
||||
Branch: m.Base,
|
||||
Commit: m.Commit,
|
||||
Created: time.Now().UTC(),
|
||||
State: inventory.PlanBuilding,
|
||||
@@ -176,6 +193,57 @@ func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inv
|
||||
}
|
||||
}
|
||||
|
||||
// supersededBy is what a newer plan takes over from the open plans it supersedes (novox/hq issue
|
||||
// 254, ADR 0218): the modules they had not finished, and those plans closed as superseded.
|
||||
//
|
||||
// **A merge looked at no plan but its own.** Two merges of one repository a few minutes apart were
|
||||
// two open plans asking for the same modules, each sending machines what it built; and a plan that
|
||||
// would never move again — waiting on a report that could not come, at 97b1b2b — stayed open for
|
||||
// ever beside the newer ones, read as work in progress by everyone who looked. The newer merge is the
|
||||
// newer intent for that repository and branch, so its plan takes over: every open plan of the same
|
||||
// repository and branch **created before it** — by the time the plans were made, never by comparing
|
||||
// commits, which have no order of their own — gives up the modules it had not built, and those are
|
||||
// planned again in the newer plan beside what the newer merge moved.
|
||||
//
|
||||
// "Not built" is a module not yet asked, or asked and not answered; **and a module built and not
|
||||
// yet sent to its machines**, where its policy rolls it out: closed, the older plan would never send
|
||||
// it, and the catalogue announces no move for a rebuild (issue 189), so the newer plan builds and
|
||||
// sends it. A build the older plan asked still finishes and registers as any build does — ordered by
|
||||
// when it was asked (issue 219), so the newer plan's ask, made later, is the one that stands.
|
||||
//
|
||||
// A plan with no branch recorded is from before branches were kept, and is superseded by the next
|
||||
// plan of its repository: what it had not built is folded in, so nothing is lost by it.
|
||||
func supersededBy(newer inventory.Plan, open []inventory.Plan, rollsOut func(string) bool) ([]string, []inventory.Plan) {
|
||||
folded := map[string]bool{}
|
||||
var closed []inventory.Plan
|
||||
for _, old := range open {
|
||||
if old.ID == newer.ID || !old.Open() || !strings.EqualFold(old.Repository, newer.Repository) ||
|
||||
(old.Branch != "" && old.Branch != newer.Branch) || !old.Created.Before(newer.Created) {
|
||||
continue
|
||||
}
|
||||
var took []string
|
||||
for name, s := range old.Modules {
|
||||
if s == nil || s.State != "built" || (s.SentAt == nil && rollsOut(name)) {
|
||||
folded[name] = true
|
||||
took = append(took, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(took)
|
||||
old.State = inventory.PlanSuperseded
|
||||
old.Note = fmt.Sprintf("superseded at tier %d by %s (%s at %s)", old.Tier, newer.ID, newer.Repository, short(newer.Commit))
|
||||
if len(took) > 0 {
|
||||
old.Note += "; " + strings.Join(took, ", ") + " planned there again"
|
||||
}
|
||||
closed = append(closed, old)
|
||||
}
|
||||
out := make([]string, 0, len(folded))
|
||||
for name := range folded {
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, closed
|
||||
}
|
||||
|
||||
// gates is what the next tier needs running from this one: a module of the tier that a later
|
||||
// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by
|
||||
// the machines running it before the next tier is asked. A base an image stands on need only be
|
||||
@@ -214,6 +282,22 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
||||
}
|
||||
|
||||
// applied says whether every machine running the module has reported since the module was built.
|
||||
// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
|
||||
// after that machine was sent the build (novox/hq issue 256).
|
||||
func appliedEach(module string, since func(node string) time.Time, running []string, reports []inventory.Reported) (bool, []string) {
|
||||
at := map[string]*time.Time{}
|
||||
for _, r := range reports {
|
||||
at[r.Node] = r.At
|
||||
}
|
||||
var waiting []string
|
||||
for _, n := range running {
|
||||
if t := at[n]; t == nil || t.Before(since(n)) {
|
||||
waiting = append(waiting, n)
|
||||
}
|
||||
}
|
||||
return len(waiting) == 0, waiting
|
||||
}
|
||||
|
||||
func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
|
||||
at := map[string]*time.Time{}
|
||||
for _, r := range reports {
|
||||
@@ -240,40 +324,71 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, name := range p.Tiers[p.Tier] {
|
||||
state := p.Modules[name]
|
||||
if state == nil {
|
||||
state = &inventory.PlanModule{}
|
||||
p.Modules[name] = state
|
||||
}
|
||||
e, known := byName[name]
|
||||
if !known {
|
||||
state.State = "failed"
|
||||
state.Why = "no longer in the catalogue"
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = name + " is no longer in the catalogue"
|
||||
continue
|
||||
}
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
fmt.Printf(" tier %d: ", p.Tier)
|
||||
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, 0); err != nil {
|
||||
state.State = "failed"
|
||||
state.Why = err.Error()
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
|
||||
continue
|
||||
}
|
||||
state.State = "asked"
|
||||
state.AskedAt = &now
|
||||
askModule(ctx, p, name, byName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A
|
||||
// variable so a test of what a plan does around an ask needs no build machine.
|
||||
var askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
|
||||
return buildOneAsked(ctx, source, path, ref, 0, false)
|
||||
}
|
||||
|
||||
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
|
||||
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
|
||||
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
|
||||
now := time.Now().UTC()
|
||||
state := p.Modules[name]
|
||||
if state == nil {
|
||||
state = &inventory.PlanModule{}
|
||||
p.Modules[name] = state
|
||||
}
|
||||
e, known := byName[name]
|
||||
if !known {
|
||||
state.State = "failed"
|
||||
state.Why = "no longer in the catalogue"
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = name + " is no longer in the catalogue"
|
||||
return
|
||||
}
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
fmt.Printf(" tier %d: ", p.Tier)
|
||||
// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
|
||||
id, err := askABuild(ctx, source, e.Source.Path, followedBranch(e.Source.Ref))
|
||||
if err != nil {
|
||||
state.State = "failed"
|
||||
state.Why = err.Error()
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
|
||||
return
|
||||
}
|
||||
state.State = "asked"
|
||||
state.AskedAt = &now
|
||||
state.Build = id
|
||||
state.Why, state.Commit, state.BuiltAt = "", "", nil
|
||||
}
|
||||
|
||||
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
|
||||
// advances what that completes. Called from the daemon's take-in of every outcome.
|
||||
func planBuilt(ctx context.Context, open *stores, module, commit, failed string) {
|
||||
//
|
||||
// **Only a build asked at or after the plan's ask is its outcome** (novox/hq 04-ISSUES/219). Two
|
||||
// plans a few minutes apart both ask for a module; the earlier plan's build, finishing late, is not
|
||||
// the later plan's answer — it stood on the bases from before the later plan's merge, and taking it
|
||||
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
|
||||
// build's request time is not known, and such an outcome is taken as before.
|
||||
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
|
||||
inv := open.inventory
|
||||
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
||||
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
||||
// which the holder settles the plan from (issue 214).
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("plans: cannot read them: %v\n", err)
|
||||
@@ -299,11 +414,28 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string)
|
||||
state = &inventory.PlanModule{}
|
||||
p.Modules[module] = state
|
||||
}
|
||||
// **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module
|
||||
// is, as before, when it was asked at or after the plan's ask (issue 219).
|
||||
// **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR
|
||||
// 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's
|
||||
// build, made from other source, and settling the plan with it would send that. A plan from
|
||||
// before ids were kept is matched as it was: by when the build was asked (issue 219).
|
||||
if state.Build != "" {
|
||||
if state.Build != id {
|
||||
continue
|
||||
}
|
||||
} else if askedBefore(asked, state.AskedAt) {
|
||||
continue
|
||||
}
|
||||
if failed != "" {
|
||||
state.State = "failed"
|
||||
state.Why = failed
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed to build in tier %d", module, p.Tier)
|
||||
sayUnsent(p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
} else {
|
||||
state.State = "built"
|
||||
state.BuiltAt = &now
|
||||
@@ -317,13 +449,30 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string)
|
||||
fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note)
|
||||
}
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
|
||||
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
|
||||
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
|
||||
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
|
||||
//
|
||||
// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two
|
||||
// controllers — the old and the new while a machine hands its controller over — would each ask a
|
||||
// tier the other had just asked. Taken without waiting: whoever holds the plans is moving them.
|
||||
func advancePlans(ctx context.Context, open *stores) {
|
||||
release, err := open.inventory.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
if !errors.Is(err, inventory.ErrPlansBusy) {
|
||||
fmt.Printf("plans: cannot hold them: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
advanceHeld(ctx, open)
|
||||
}
|
||||
|
||||
// advanceHeld is advancePlans for a caller already holding the plans.
|
||||
func advanceHeld(ctx context.Context, open *stores) {
|
||||
inv := open.inventory
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
@@ -348,8 +497,17 @@ func advancePlans(ctx context.Context, open *stores) {
|
||||
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
|
||||
if err != nil {
|
||||
fmt.Printf("%s: %v\n", p.ID, err)
|
||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||
// the state is left as it was and the step is tried again on the next tick.
|
||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
if p.State == inventory.PlanFailed {
|
||||
sayUnsent(p, rollsOut)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
break
|
||||
@@ -384,6 +542,35 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
// **Asked: settle from the build records first** (novox/hq 04-ISSUES/214). An outcome is taken
|
||||
// in by whichever controller hears it, and a merge to the controller's own repository replaces
|
||||
// the controller in its first tier: the build that produced the new one is recorded, and the
|
||||
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
||||
// outcome, whoever was listening.
|
||||
recorded := map[string][]inventory.Build{}
|
||||
byID := map[string]inventory.Build{}
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||
builds, err := inv.Builds(ctx, m, 5)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
recorded[m] = builds
|
||||
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
||||
if s.Build != "" {
|
||||
b, found, err := inv.BuildByID(ctx, s.Build)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if found {
|
||||
byID[s.Build] = b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if settleFromRecords(p, tier, recorded, byID) {
|
||||
return true, nil
|
||||
}
|
||||
// Asked: wait for every build.
|
||||
var latest time.Time
|
||||
for _, m := range tier {
|
||||
@@ -400,6 +587,15 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
// a module that packages another repository's source, keeps its commit; the catalogue announces
|
||||
// no move for it and its machines would keep the old image until somebody pushed (novox/hq
|
||||
// issue 189). A module whose policy records is built and left, as its policy says.
|
||||
//
|
||||
// **One machine first, unless the module's policy says together** (novox/hq issue 249, ADR
|
||||
// 0218). The plan sent every machine running the module at once, and the operator's policy —
|
||||
// one at a time, stopping at the first that fails, which an announced upgrade honours — was not
|
||||
// read here at all: a module whose new declarations broke it broke everywhere in the same
|
||||
// minute. Now the first machine is sent, the plan records it and waits for that machine's report
|
||||
// after the send to say it applied what it was sent; only then are the rest sent. A first machine
|
||||
// that fails or refuses stops the module's rollout and the plan with it, the rest untouched.
|
||||
var pending []string
|
||||
for _, m := range tier {
|
||||
state := p.Modules[m]
|
||||
if state == nil || state.SentAt != nil || !rollsOut(m) {
|
||||
@@ -409,17 +605,64 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
policy, err := inv.UpgradeOf(ctx, m)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var reports []inventory.Reported
|
||||
if !policy.Together {
|
||||
// Read for the choice of the first machine as well as for its report.
|
||||
if reports, err = inv.LastReports(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
state.SentAt = &now
|
||||
if len(running) == 0 {
|
||||
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
|
||||
switch {
|
||||
case step.failed != "":
|
||||
state.Why = step.failed
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s stopped at its first machine in tier %d: %s; %s left as it was",
|
||||
m, p.Tier, step.failed, orNone(strings.Join(step.rest, ", ")))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
case step.waiting != "":
|
||||
pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Local().Format("15:04")))
|
||||
continue
|
||||
case len(step.send) == 0:
|
||||
// No machine runs it: nothing to send, and nothing to wait for.
|
||||
state.SentAt = &now
|
||||
continue
|
||||
}
|
||||
if err := sendTo(ctx, open, running); err != nil {
|
||||
return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(running, ", "), p.Tier, err)
|
||||
// What sendToEach answers, not what was asked: the machine holding the bus is sent before
|
||||
// the first when its user list must change (issue 249), and the plan waits for it too.
|
||||
sent, err := sendToEach(ctx, open, step.send)
|
||||
if err != nil {
|
||||
// Not marked sent, so the next step tries again (issue 249): a grant that could not be
|
||||
// issued is a send that did not happen.
|
||||
return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(step.send, ", "), p.Tier, err)
|
||||
}
|
||||
fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(running, ", "))
|
||||
if step.first {
|
||||
state.First = sent
|
||||
state.FirstAt = &now
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first", p.Tier, m, strings.Join(sent, ", "))
|
||||
fmt.Printf("%s: tier %d built; sent %s to %s first, the rest once it reports it applied\n",
|
||||
p.ID, p.Tier, m, strings.Join(sent, ", "))
|
||||
return true, nil
|
||||
}
|
||||
state.SentAt = &now
|
||||
fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(sent, ", "))
|
||||
return true, nil
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(pending, "; ") +
|
||||
" to report it applied before the rest are sent"
|
||||
changed := p.State != inventory.PlanRolling || p.Note != note
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
}
|
||||
// And wait for what the next tier needs running.
|
||||
needed := gates(*p, edges, rollsOut)
|
||||
if len(needed) > 0 {
|
||||
@@ -446,10 +689,24 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if state.BuiltAt != nil {
|
||||
since = *state.BuiltAt
|
||||
}
|
||||
if state.SentAt != nil && state.SentAt.After(since) {
|
||||
since = *state.SentAt
|
||||
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
||||
// a module is sent twice — the first machine, then the rest — and SentAt is the second.
|
||||
// Asked of every machine, the first machine's report, made between the two sends, read as
|
||||
// older than the build, and the gate waited for a report it already had, for ever.
|
||||
sinceFor := func(node string) time.Time {
|
||||
at := since
|
||||
sent := state.SentAt
|
||||
for _, n := range state.First {
|
||||
if n == node {
|
||||
sent = state.FirstAt
|
||||
}
|
||||
}
|
||||
if sent != nil && sent.After(at) {
|
||||
at = *sent
|
||||
}
|
||||
return at
|
||||
}
|
||||
if ok, on := applied(m, since, running, reports); !ok {
|
||||
if ok, on := appliedEach(m, sinceFor, running, reports); !ok {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
||||
}
|
||||
}
|
||||
@@ -470,6 +727,110 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// rolloutStep is what a plan does next with one built module's machines (novox/hq issue 249).
|
||||
type rolloutStep struct {
|
||||
// send is the machines to send now; first, whether they are the first machine's send.
|
||||
send []string
|
||||
first bool
|
||||
// waiting names the first machines whose report the rest wait for.
|
||||
waiting string
|
||||
// failed says how a first machine did not take it; rest is what is then left alone.
|
||||
failed string
|
||||
rest []string
|
||||
}
|
||||
|
||||
// nextRollout is the next step of one module's rollout in a plan (novox/hq issue 249, ADR 0218).
|
||||
//
|
||||
// Together, every machine running it at once, as the policy says. Otherwise one machine first — the
|
||||
// first by name among those that have reported within the bound, so a laptop that is away is not
|
||||
// the one the rest wait on; the first by name when none has; the same choice on every controller and
|
||||
// every resume — and the rest once each machine the first send reached reports, about the
|
||||
// declaration it was last sent, that it applied it. Compared by the store's own record of what was
|
||||
// sent (Reported.Current), never by this controller's clock against the machine's.
|
||||
//
|
||||
// **A first machine that fails, refuses, or does not report within the bound stops the rollout
|
||||
// there** (ADR 0218 §2), naming the machine; the rest are not sent. A wait with no end is not a
|
||||
// rollout: it held the plan open for ever, read as work in progress (issue 254).
|
||||
func nextRollout(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported,
|
||||
now time.Time, bound time.Duration) rolloutStep {
|
||||
if len(running) == 0 {
|
||||
return rolloutStep{}
|
||||
}
|
||||
if together {
|
||||
return rolloutStep{send: running}
|
||||
}
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, r := range reports {
|
||||
byNode[r.Node] = r
|
||||
}
|
||||
if s.FirstAt == nil {
|
||||
sorted := append([]string{}, running...)
|
||||
sort.Strings(sorted)
|
||||
for _, n := range sorted {
|
||||
if r, said := byNode[n]; said && r.At != nil && now.Sub(*r.At) <= bound {
|
||||
return rolloutStep{send: []string{n}, first: true}
|
||||
}
|
||||
}
|
||||
return rolloutStep{send: sorted[:1], first: true}
|
||||
}
|
||||
sentFirst := map[string]bool{}
|
||||
for _, n := range s.First {
|
||||
sentFirst[n] = true
|
||||
}
|
||||
var rest []string
|
||||
for _, n := range running {
|
||||
if !sentFirst[n] {
|
||||
rest = append(rest, n)
|
||||
}
|
||||
}
|
||||
var waiting, failed []string
|
||||
for _, n := range s.First {
|
||||
r, said := byNode[n]
|
||||
// Only a report about what it was last sent says anything about this build.
|
||||
if !said || r.At == nil || !r.Current {
|
||||
waiting = append(waiting, n)
|
||||
continue
|
||||
}
|
||||
switch r.Outcome {
|
||||
case inventory.OutcomeApplied:
|
||||
case inventory.OutcomeFailed, inventory.OutcomeRefused:
|
||||
failed = append(failed, n+" "+r.Outcome+" what it was sent")
|
||||
default:
|
||||
waiting = append(waiting, n)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return rolloutStep{failed: strings.Join(failed, "; "), rest: rest}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
if now.Sub(*s.FirstAt) > bound {
|
||||
return rolloutStep{failed: fmt.Sprintf("%s did not report it applied within %s",
|
||||
strings.Join(waiting, ", "), bound), rest: rest}
|
||||
}
|
||||
return rolloutStep{waiting: strings.Join(waiting, ", ")}
|
||||
}
|
||||
return rolloutStep{send: rest}
|
||||
}
|
||||
|
||||
// sayUnsent adds to an ended plan's note the modules it built and never sent (novox/hq issue 249).
|
||||
// An announced move of a module a plan held was left to that plan; a plan that ends without
|
||||
// sending it — failed elsewhere, or closed by hand — would leave its machines behind with nothing
|
||||
// saying so. A module whose rollout stopped at its first machine is not among them: that stop was
|
||||
// the point. Said once.
|
||||
func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
|
||||
var unsent []string
|
||||
for name, s := range p.Modules {
|
||||
if s != nil && s.State == "built" && s.SentAt == nil && s.FirstAt == nil && rollsOut(name) {
|
||||
unsent = append(unsent, name)
|
||||
}
|
||||
}
|
||||
if len(unsent) == 0 || strings.Contains(p.Note, "built and never sent") {
|
||||
return
|
||||
}
|
||||
sort.Strings(unsent)
|
||||
p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
|
||||
}
|
||||
|
||||
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
||||
func planTicker(ctx context.Context, open *stores) {
|
||||
advancePlans(ctx, open)
|
||||
@@ -486,15 +847,24 @@ func planTicker(ctx context.Context, open *stores) {
|
||||
}
|
||||
|
||||
// planLine is one plan as `status` says it.
|
||||
func planLine(p inventory.Plan, now time.Time) string {
|
||||
func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) }
|
||||
|
||||
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
|
||||
// waiting on builds nobody will take until a person resumes the seat says so, and is not late.
|
||||
func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
|
||||
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
||||
switch p.State {
|
||||
case inventory.PlanDone:
|
||||
return fmt.Sprintf("%s %s done, %d tier(s)", p.Repository, short(p.Commit), len(p.Tiers))
|
||||
case inventory.PlanFailed:
|
||||
return fmt.Sprintf("%s %s FAILED at %s: %s", p.Repository, short(p.Commit), where, p.Note)
|
||||
case inventory.PlanSuperseded:
|
||||
return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
|
||||
}
|
||||
since := now.Sub(p.Updated).Round(time.Second)
|
||||
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||
return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
|
||||
}
|
||||
since := now.Sub(p.Updated).Round(time.Minute)
|
||||
late := ""
|
||||
if since > planWaitBound {
|
||||
late = " — LATE"
|
||||
@@ -508,19 +878,49 @@ func planLine(p inventory.Plan, now time.Time) string {
|
||||
|
||||
// planFailedBuild marks the module a failed build was for when the result names no module: by the
|
||||
// repository and path the plan's modules were asked at.
|
||||
//
|
||||
// **By the id first** (novox/hq ADR 0219): a plan keeps the id it asked each module under, so an
|
||||
// outcome that never learnt its module's name — cancelled, killed, failed at the clone — is matched
|
||||
// to the module it was asked for exactly. Repository and path remain for a plan from before ids
|
||||
// were kept.
|
||||
func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) {
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
if plans, err := open.inventory.OpenPlans(ctx); err == nil {
|
||||
if module := moduleAskedAs(plans, result.ID); module != "" {
|
||||
planBuilt(ctx, open, module, result.Commit, result.Failed, asked, result.ID)
|
||||
return
|
||||
}
|
||||
}
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path {
|
||||
planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed)
|
||||
planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked, result.ID)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// moduleAskedAs is the module an open plan's current tier asked for under this id, or nothing.
|
||||
func moduleAskedAs(plans []inventory.Plan, id string) string {
|
||||
if id == "" {
|
||||
return ""
|
||||
}
|
||||
for _, p := range plans {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.Build == id {
|
||||
return m
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func repositoryMatches(a, b string) bool {
|
||||
trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) }
|
||||
return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a))
|
||||
@@ -539,7 +939,7 @@ type planStatus struct {
|
||||
Late bool `json:"late"`
|
||||
}
|
||||
|
||||
func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
|
||||
func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus {
|
||||
out := make([]planStatus, 0, len(plans))
|
||||
for _, p := range plans {
|
||||
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
|
||||
@@ -550,6 +950,10 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
|
||||
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
|
||||
}
|
||||
ps.Late = now.Sub(p.Updated) > planWaitBound
|
||||
// Paused is a person's decision, not lateness (novox/hq ADR 0219).
|
||||
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||
ps.Waiting, ps.Late = waiting, false
|
||||
}
|
||||
}
|
||||
out = append(out, ps)
|
||||
}
|
||||
@@ -557,12 +961,15 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
|
||||
}
|
||||
|
||||
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
|
||||
func openPlans(plans []inventory.Plan) ([]inventory.Plan, int) {
|
||||
func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) {
|
||||
var open []inventory.Plan
|
||||
late := 0
|
||||
for _, p := range plans {
|
||||
if p.Open() {
|
||||
open = append(open, p)
|
||||
if _, paused := pausedWaiting(p, pause, time.Now()); paused {
|
||||
continue
|
||||
}
|
||||
if time.Since(p.Updated) > planWaitBound {
|
||||
late++
|
||||
}
|
||||
@@ -595,7 +1002,7 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s — %s\n", p.ID, planLine(p, now))
|
||||
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
|
||||
for i, tier := range p.Tiers {
|
||||
marker := " "
|
||||
if i == p.Tier && p.Open() {
|
||||
@@ -610,6 +1017,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if s.Commit != "" {
|
||||
state += " from " + short(s.Commit)
|
||||
}
|
||||
if s.Build != "" && s.State != "built" {
|
||||
state += " (" + s.Build + ")"
|
||||
}
|
||||
if s.Why != "" {
|
||||
state += ": " + s.Why
|
||||
}
|
||||
@@ -622,7 +1032,28 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if *whatIf != "" {
|
||||
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
|
||||
}
|
||||
if len(positionals) == 2 && positionals[0] == "stop" {
|
||||
// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
|
||||
if len(positionals) == 2 && positionals[0] == "retry" {
|
||||
said, err := retryPlan(ctx, open, positionals[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
// `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one
|
||||
// waiting on a report that cannot come — so it stops reading as work in progress. Marked failed
|
||||
// with who ended it; what it asked still builds and registers.
|
||||
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||
how := "stopped"
|
||||
if positionals[0] == "close" {
|
||||
how = "closed"
|
||||
}
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, positionals[1])
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -631,12 +1062,16 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = "stopped by hand at tier " + fmt.Sprint(p.Tier)
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s stopped at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
||||
p.ID, p.Tier, len(p.Tiers))
|
||||
fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
||||
p.ID, how, p.Tier, len(p.Tiers))
|
||||
return nil
|
||||
}
|
||||
plans, err := inv.RecentPlans(ctx, *limit)
|
||||
@@ -647,8 +1082,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
fmt.Println("no merge has produced a plan yet")
|
||||
return nil
|
||||
}
|
||||
pause := buildSeatPause(ctx, inv, plans)
|
||||
for _, p := range plans {
|
||||
fmt.Printf("%-28s %s\n", p.ID, planLine(p, now))
|
||||
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -711,7 +1147,13 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
|
||||
how := "built; its policy records, so nothing is sent"
|
||||
if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut {
|
||||
running, _ := inv.Running(ctx, name)
|
||||
reports, _ := inv.LastReports(ctx)
|
||||
how = "built, then sent to " + orNone(strings.Join(running, ", "))
|
||||
// One machine first unless the policy says together (novox/hq issue 249).
|
||||
if first := nextRollout(inventory.PlanModule{}, running, u.Together, reports, time.Now(), planWaitBound); first.first && len(running) > 1 {
|
||||
how = fmt.Sprintf("built, then sent to %s first and to the rest once it has applied it",
|
||||
first.send[0])
|
||||
}
|
||||
rolls[name] = how
|
||||
}
|
||||
fmt.Printf(" %-22s %s\n", name, how)
|
||||
@@ -740,3 +1182,64 @@ func splitList(s string) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
||||
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
||||
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
||||
//
|
||||
// The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR
|
||||
// 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is
|
||||
// found by nothing else.
|
||||
func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build,
|
||||
byID map[string]inventory.Build) bool {
|
||||
changed := false
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil || s.State != "asked" || s.AskedAt == nil {
|
||||
continue
|
||||
}
|
||||
var outcome *inventory.Build
|
||||
for i := range recorded[m] {
|
||||
// Asked under an id, only that id's record answers (ADR 0219), and it is looked up below.
|
||||
if s.Build != "" {
|
||||
break
|
||||
}
|
||||
b := recorded[m][i]
|
||||
if b.At.Before(*s.AskedAt) {
|
||||
break
|
||||
}
|
||||
// Recorded after the ask and asked before it: an earlier ask's late outcome, not this
|
||||
// one's (novox/hq 04-ISSUES/219).
|
||||
if askedBefore(b.Asked, s.AskedAt) {
|
||||
continue
|
||||
}
|
||||
outcome = &b
|
||||
}
|
||||
if own, found := byID[s.Build]; s.Build != "" && found {
|
||||
outcome = &own
|
||||
}
|
||||
if outcome == nil {
|
||||
continue
|
||||
}
|
||||
at := outcome.At
|
||||
if outcome.Worked() {
|
||||
s.State = "built"
|
||||
s.BuiltAt = &at
|
||||
s.Commit = outcome.Commit
|
||||
} else {
|
||||
s.State = "failed"
|
||||
s.Why = outcome.Failed
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed to build in tier %d", m, p.Tier)
|
||||
}
|
||||
fmt.Printf("%s: %s settled from the build records as %s (%s)\n", p.ID, m, s.State, outcome.ID)
|
||||
changed = true
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// askedBefore is whether a build asked at asked was asked before a plan asked for its module — and
|
||||
// so is not that plan's outcome (novox/hq 04-ISSUES/219). False when either time is not known.
|
||||
func askedBefore(asked time.Time, planAsked *time.Time) bool {
|
||||
return !asked.IsZero() && planAsked != nil && asked.Before(*planAsked)
|
||||
}
|
||||
|
||||
@@ -115,3 +115,100 @@ func TestACycleIsOneLastTierAndSaidSo(t *testing.T) {
|
||||
t.Fatalf("a cycle should be one tier of two, said: %v", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the
|
||||
// bundle a tier after the toolchain, not beside it.
|
||||
func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) {
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}}
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"},
|
||||
[]string{"mesh-tools", "node-tools"}, edges)
|
||||
if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" {
|
||||
t.Fatalf("the toolchain, then the bundle: %v", p.Tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/214: a plan whose build outcome was recorded while no controller followed it —
|
||||
// the controller rebuilding itself — settles from the build records instead of waiting for ever.
|
||||
func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 3, 19, 20, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Tiers: [][]string{{"mesh-controller", "builder"}, {"route-proxy"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"mesh-controller": {State: "asked", AskedAt: &asked},
|
||||
"builder": {State: "asked", AskedAt: &asked},
|
||||
}}
|
||||
records := map[string][]inventory.Build{
|
||||
// Newest first, as Builds answers: the build after the ask is the outcome.
|
||||
"mesh-controller": {
|
||||
{ID: "build-2", Commit: "2ebbb799", At: asked.Add(4 * time.Minute)},
|
||||
{ID: "build-1", Commit: "06ea2168", At: asked.Add(-10 * time.Minute)},
|
||||
},
|
||||
// Only a build from before the ask: not this ask's outcome.
|
||||
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
|
||||
}
|
||||
if !settleFromRecords(&p, p.Tiers[0], records, nil) {
|
||||
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
|
||||
}
|
||||
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
|
||||
t.Errorf("the controller's ask is %+v, want built from 2ebbb799", s)
|
||||
}
|
||||
if s := p.Modules["builder"]; s.State != "asked" {
|
||||
t.Errorf("an ask with no record after it was settled: %+v", s)
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: a build recorded after the ask but asked before it — an earlier
|
||||
// plan's late outcome — is not this ask's, built or failed.
|
||||
r := inventory.Plan{ID: "plan-3", Tiers: [][]string{{"postgres"}},
|
||||
Modules: map[string]*inventory.PlanModule{"postgres": {State: "asked", AskedAt: &asked}}}
|
||||
late := map[string][]inventory.Build{"postgres": {
|
||||
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
|
||||
}}
|
||||
if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
|
||||
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
|
||||
}
|
||||
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
|
||||
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
|
||||
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
|
||||
if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
|
||||
r.Modules["postgres"].Commit != "4bcd5f73" {
|
||||
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
|
||||
}
|
||||
|
||||
// A failure recorded after the ask fails the plan, as hearing it would have.
|
||||
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
|
||||
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
|
||||
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
|
||||
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
|
||||
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
|
||||
}
|
||||
}
|
||||
|
||||
// The first machine's report, made between the send to it and the send to the rest, opens the gate for it:
|
||||
// each machine is judged from its own send, not from the last one (novox/hq issue 256).
|
||||
func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
|
||||
built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC)
|
||||
firstSent := built.Add(31 * time.Second)
|
||||
firstReported := built.Add(43 * time.Second)
|
||||
restSent := built.Add(58 * time.Second)
|
||||
restReported := built.Add(74 * time.Second)
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", At: &firstReported},
|
||||
{Node: "g14", At: &restReported},
|
||||
}
|
||||
since := func(node string) time.Time {
|
||||
if node == "ace" {
|
||||
return firstSent
|
||||
}
|
||||
return restSent
|
||||
}
|
||||
if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok {
|
||||
t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting)
|
||||
}
|
||||
// The old reading, every machine from the last send, is what held the plan.
|
||||
if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok {
|
||||
t.Fatal("the single-moment reading should hold the first machine back")
|
||||
}
|
||||
early := built.Add(10 * time.Second)
|
||||
if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" {
|
||||
t.Fatal("a report from before the machine was sent opened the gate")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,7 +346,9 @@ func rolloutMint(ctx context.Context, again bool) error {
|
||||
}
|
||||
machines++
|
||||
|
||||
case broker.KindModule:
|
||||
case broker.KindModule, broker.KindNodeTools:
|
||||
// The runtime is minted and delivered exactly as a module is (novox/hq ADR 0175): it is
|
||||
// issued as the module it stands for, to that module's `broker` secret.
|
||||
if p.Module == "mesh-controller" {
|
||||
// The control plane is a module too, and its `broker` secret is the old bus's
|
||||
// credential it is still using while this runs. Writing the new bus's blob there
|
||||
@@ -365,7 +367,7 @@ func rolloutMint(ctx context.Context, again bool) error {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: busKindOf(p.Module), Node: p.Node, Module: p.Module})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only
|
||||
// once that machine has reported it applied; a module whose policy says together goes everywhere at
|
||||
// once, as before.
|
||||
func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) {
|
||||
running := []string{"novox", "ace", "g14"}
|
||||
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
now := sentAt.Add(5 * time.Minute)
|
||||
bound := 30 * time.Minute
|
||||
after := sentAt.Add(time.Minute)
|
||||
next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep {
|
||||
return nextRollout(s, running, together, reports, now, bound)
|
||||
}
|
||||
|
||||
// Together: every machine at once.
|
||||
if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first {
|
||||
t.Fatalf("a together policy did not send every machine at once: %+v", step)
|
||||
}
|
||||
|
||||
// Otherwise the first by name, alone, when none has reported lately.
|
||||
step := next(inventory.PlanModule{}, running, false, nil)
|
||||
if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) {
|
||||
t.Fatalf("the first send was %+v, wanted ace alone", step)
|
||||
}
|
||||
|
||||
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||
report := func(outcome string, current bool) []inventory.Reported {
|
||||
return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current},
|
||||
{Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}}
|
||||
}
|
||||
|
||||
// No report yet, or one about an older declaration than it was last sent: wait.
|
||||
for what, reports := range map[string][]inventory.Reported{
|
||||
"no report": nil,
|
||||
"a report about older": report(inventory.OutcomeApplied, false),
|
||||
} {
|
||||
step := next(state, running, false, reports)
|
||||
if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" {
|
||||
t.Errorf("%s: %+v, wanted to wait for ace", what, step)
|
||||
}
|
||||
}
|
||||
|
||||
// Applied what it was last sent: the rest, and only the rest.
|
||||
step = next(state, running, false, report(inventory.OutcomeApplied, true))
|
||||
if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) {
|
||||
t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step)
|
||||
}
|
||||
|
||||
// Failed or refused: stop, the rest untouched.
|
||||
for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} {
|
||||
step := next(state, running, false, report(outcome, true))
|
||||
if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) ||
|
||||
!reflect.DeepEqual(step.rest, []string{"novox", "g14"}) {
|
||||
t.Errorf("a first machine that %s it: %+v", outcome, step)
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus went with the first send: the rest wait for it too, and it is
|
||||
// not sent again.
|
||||
both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt}
|
||||
half := report(inventory.OutcomeApplied, true)
|
||||
if step := next(both, running, false, half); step.waiting != "novox" {
|
||||
t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step)
|
||||
}
|
||||
all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true})
|
||||
if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||
t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step)
|
||||
}
|
||||
|
||||
// One machine, or none: nothing is waited for that cannot come.
|
||||
if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first {
|
||||
t.Fatalf("a module nothing runs was sent: %+v", step)
|
||||
}
|
||||
if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" {
|
||||
t.Fatalf("a module on one machine waited for more: %+v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there,
|
||||
// naming the machine and the bound; the rest are left alone.
|
||||
func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) {
|
||||
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||
step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute)
|
||||
if !strings.Contains(step.failed, "ace did not report it applied within 30m") ||
|
||||
!reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 {
|
||||
t.Fatalf("a silent first machine: %+v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The first machine is the first by name among those heard from lately: a laptop that is away is
|
||||
// not the one the rest wait on. When none has been heard from, the first by name.
|
||||
func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) {
|
||||
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour)
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately},
|
||||
}
|
||||
step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute)
|
||||
if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||
t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step)
|
||||
}
|
||||
}
|
||||
|
||||
// An announced move of a module an open plan is still rolling out is left to the plan: sending it
|
||||
// here as well put the bundle on every machine at once (novox/hq issue 249).
|
||||
func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) {
|
||||
sent := time.Now()
|
||||
plans := []inventory.Plan{
|
||||
{ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}},
|
||||
{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}},
|
||||
}
|
||||
if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" {
|
||||
t.Fatalf("a module the plan is rolling out was not left to it: %q", got)
|
||||
}
|
||||
for _, m := range []string{"gitea", "keycloak"} {
|
||||
if got := rolledOutByAPlan(plans, m); got != "" {
|
||||
t.Errorf("%s, which no plan will send, was left to %s", m, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A plan that ends without sending what it built says so, with the remedy; a module whose rollout
|
||||
// stopped at its first machine is not among them.
|
||||
func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) {
|
||||
at := time.Now()
|
||||
p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1",
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built"},
|
||||
"stopped": {State: "built", First: []string{"ace"}, FirstAt: &at},
|
||||
"sent": {State: "built", SentAt: &at},
|
||||
"notes": {State: "built"},
|
||||
"later": {},
|
||||
}}
|
||||
rollsOut := func(m string) bool { return m != "notes" }
|
||||
sayUnsent(&p, rollsOut)
|
||||
sayUnsent(&p, rollsOut)
|
||||
if p.Note != "closed by hand at tier 1; built and never sent: agent — `push --behind` sends them" {
|
||||
t.Fatalf("the note reads %q", p.Note)
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A node's own set failing to compose, and the mesh being unable to answer at all, are different
|
||||
// things, and only the first may be passed over when something is gathered across every machine
|
||||
// (novox/hq 04-ISSUES/152). These pin that distinction where the three gatherers rely on it.
|
||||
// (novox/hq 04-ISSUES/152). These pin that distinction where the gatherers rely on it.
|
||||
|
||||
func TestASetThatDoesNotComposeIsMarkedAsTheNodesOwnProblem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
@@ -45,55 +44,3 @@ func TestAStoreThatCannotBeReadIsNotANodeThatDoesNotCompose(t *testing.T) {
|
||||
t.Fatalf("a question the mesh could not answer was read as a node that runs nothing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneIncoherentNodeDoesNotCostTheRestTheirNames(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{one.Module, two.Module} {
|
||||
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// laptop cannot compose. That is laptop's problem and nobody else's: the roster is still
|
||||
// answerable, and anchor keeps whatever it serves.
|
||||
if _, err := routeNamesInTheMesh(t.Context(), open); err != nil {
|
||||
t.Fatalf("one node's broken set cost the whole mesh its roster: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARosterIsNeverReturnedWithNamesItCouldNotRead(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
|
||||
stopped, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
names, err := routeNamesInTheMesh(stopped, open)
|
||||
if err == nil {
|
||||
t.Fatalf("a roster was composed from a store that could not be read: %v", names)
|
||||
}
|
||||
// The failure must be raised, not turned into an absence. A roster missing a machine's names
|
||||
// is indistinguishable, on every machine that receives it, from the operator withdrawing them —
|
||||
// and because the roster is part of every container's identity, it replaces all of them.
|
||||
if names != nil {
|
||||
t.Fatalf("a partial roster was returned beside the error: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// Kept so the reason survives the next person reading it: the message the gatherer raises must say
|
||||
// which machine could not be read, or the operator is left with a mesh-wide failure and no name.
|
||||
func TestTheRaisedFailureNamesTheMachineItCouldNotRead(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
stopped, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
_, err := routeNamesInTheMesh(stopped, open)
|
||||
if err == nil {
|
||||
t.Fatal("no failure was raised")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot be read") {
|
||||
t.Fatalf("the failure does not say the mesh could not be read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0207 at the controller's acts: `assign` refuses a module whose resources a
|
||||
// seat nothing on the node holds applies, `unassign` refuses taking the last holder from under its
|
||||
// dependents, and `status` reports what composition does not yet refuse.
|
||||
|
||||
func serviceManagerHolder() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "systemd", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
|
||||
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}},
|
||||
Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
|
||||
}
|
||||
|
||||
func packageManagerHolder() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "pacman", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.PackageManagerSeat, Scope: catalogue.ScopeNode}},
|
||||
Resources: []map[string]any{{"id": "refresh", "type": "service", "unit": "pacman-refresh.timer"}}}
|
||||
}
|
||||
|
||||
func aDaemon() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "sshd", Version: "1",
|
||||
Resources: []map[string]any{{"id": "sshd", "type": "service", "unit": "sshd.service"}}}
|
||||
}
|
||||
|
||||
func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, serviceManagerHolder())
|
||||
register(t, open, packageManagerHolder())
|
||||
register(t, open, aDaemon())
|
||||
|
||||
_, err := assign(ctx, open, "laptop", "sshd")
|
||||
if err == nil {
|
||||
t.Fatal("sshd went onto a machine nothing holds the service manager of")
|
||||
}
|
||||
for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q:\n%v", want, err)
|
||||
}
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if contains(assigned, "sshd") {
|
||||
t.Fatalf("a refused assignment was kept: %v", assigned)
|
||||
}
|
||||
|
||||
// The holders depend on each other, so neither goes on alone — and both go on in one act.
|
||||
if _, err := assign(ctx, open, "laptop", "systemd"); err == nil {
|
||||
t.Fatal("systemd went on alone though its package needs a package manager")
|
||||
}
|
||||
if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil {
|
||||
t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said)
|
||||
}
|
||||
if said, err := assign(ctx, open, "laptop", "sshd"); err != nil {
|
||||
t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) {
|
||||
argv, err := argvFor("assign", map[string]any{"node": "laptop", "module": "systemd, pacman"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(argv, " ") != "assign laptop systemd pacman" {
|
||||
t.Errorf("the seat's assign became %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, serviceManagerHolder())
|
||||
register(t, open, packageManagerHolder())
|
||||
register(t, open, aDaemon())
|
||||
if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err := unassign(ctx, open, "laptop", "systemd")
|
||||
if err == nil {
|
||||
t.Fatal("the service manager came off a machine still running services")
|
||||
}
|
||||
for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %q:\n%v", want, err)
|
||||
}
|
||||
}
|
||||
assigned, _ := open.inventory.Assigned(ctx, "laptop")
|
||||
if !contains(assigned, "systemd") {
|
||||
t.Fatalf("a refused unassignment took the module off anyway: %v", assigned)
|
||||
}
|
||||
if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil {
|
||||
t.Fatalf("a dependent could not come off: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) {
|
||||
// The mesh as it ran before the switch (novox/hq ADR 0207 §4).
|
||||
defer catalogue.EnforcingSeatDependencies(false)()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, serviceManagerHolder())
|
||||
register(t, open, aDaemon())
|
||||
// Assigned straight into the store: a machine whose modules predate the rule, which is every
|
||||
// machine on the day it ships.
|
||||
if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, refused := asked.refused["laptop"]; refused {
|
||||
t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"])
|
||||
}
|
||||
if asked.well() {
|
||||
t.Error("a mesh with an unheld dependency reads as all well")
|
||||
}
|
||||
got := printed(t, func() error { return printStatus(asked) })
|
||||
for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("status does not say %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Unheld []catalogue.Unheld `json:"unheld"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat {
|
||||
t.Errorf("the document's unheld is %+v", doc.Unheld)
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -48,17 +50,32 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
return nil
|
||||
}
|
||||
switch verb {
|
||||
case "command":
|
||||
// The generic verb: the command line as given, split as a shell would split it, with
|
||||
// nothing added — the named verbs add flags a caller cannot reach; this one is the whole
|
||||
// binary and says so in its description (novox/hq ADR 0154, 0175).
|
||||
if err := need("command"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv, err := splitCommandLine(str("command"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(argv) == 0 {
|
||||
return nil, errors.New("command names no command")
|
||||
}
|
||||
return argv, nil
|
||||
case "status":
|
||||
return []string{"status", "--json"}, nil
|
||||
case "nodes":
|
||||
return []string{"node", "list"}, nil
|
||||
return []string{"node", "list", "--json"}, nil
|
||||
case "node":
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"node", "show", str("node")}, nil
|
||||
case "modules":
|
||||
return []string{"module", "list"}, nil
|
||||
return []string{"module", "list", "--json"}, nil
|
||||
case "seats":
|
||||
return []string{"seats", "--json"}, nil
|
||||
case "builds":
|
||||
@@ -83,10 +100,51 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if id := str("stop"); id != "" {
|
||||
return []string{"plans", "stop", id}, nil
|
||||
}
|
||||
if id := str("close"); id != "" {
|
||||
return []string{"plans", "close", id}, nil
|
||||
}
|
||||
if id := str("retry"); id != "" {
|
||||
return []string{"plans", "retry", id}, nil
|
||||
}
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"plans", id}, nil
|
||||
}
|
||||
return []string{"plans"}, nil
|
||||
// The build queue (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return []string{"queue"}, nil
|
||||
case "cancel", "kill":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{verb, str("id")}, nil
|
||||
case "clear":
|
||||
if str("dead") == "true" {
|
||||
return []string{"clear", "--dead"}, nil
|
||||
}
|
||||
return []string{"clear"}, nil
|
||||
case "rebuild":
|
||||
if err := need("what"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"rebuild", str("what")}, nil
|
||||
case "replay":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"replay", str("id")}
|
||||
if str("register") == "true" {
|
||||
argv = append(argv, "--register")
|
||||
}
|
||||
if str("older") == "true" {
|
||||
argv = append(argv, "--older")
|
||||
}
|
||||
return argv, nil
|
||||
case "pause", "resume":
|
||||
if n := str("node"); n != "" {
|
||||
return []string{verb, n}, nil
|
||||
}
|
||||
return []string{verb}, nil
|
||||
case "plan":
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
@@ -96,7 +154,9 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if err := need("node", "module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{verb, str("node"), str("module")}, nil
|
||||
// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
|
||||
// the seats that apply resources depend on each other and go on together.
|
||||
return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
|
||||
case "pin":
|
||||
if err := need("node", "provision", "from", "module"); err != nil {
|
||||
return nil, err
|
||||
@@ -179,7 +239,7 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
}
|
||||
|
||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true}
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
||||
|
||||
// runVerb runs this binary with the given command line and gathers what it said.
|
||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
@@ -215,13 +275,15 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
}
|
||||
|
||||
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
|
||||
// store's row, so a verb the row does not carry is not served and a verb it carries that this binary
|
||||
// cannot run is said at start rather than at the first call.
|
||||
func seatToolHandlers() (map[string]link.ToolHandler, error) {
|
||||
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
|
||||
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
|
||||
// would take the whole control plane down for one word (novox/hq ADR 0185).
|
||||
func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
||||
if !known {
|
||||
return nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
|
||||
return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
|
||||
}
|
||||
var behind []string
|
||||
handlers := map[string]link.ToolHandler{}
|
||||
for _, v := range seat.Serves {
|
||||
verb := v.Name
|
||||
@@ -232,8 +294,27 @@ func seatToolHandlers() (map[string]link.ToolHandler, error) {
|
||||
continue
|
||||
}
|
||||
if _, err := argvFor(verb, sampleArguments(v)); err != nil {
|
||||
return nil, fmt.Errorf("the %s seat's row declares %q, which this control plane cannot run: %w",
|
||||
catalogue.ControllerSeatName, verb, err)
|
||||
// **A row ahead of this binary is not a reason to go silent.**
|
||||
//
|
||||
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
|
||||
// this build does not know means the row was widened by a newer one — the ordinary
|
||||
// state of a roll-out, and of a push that put an older control plane back. Refusing to
|
||||
// serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole
|
||||
// mesh off the bus for ten minutes, and the way back was a human running the binary by
|
||||
// hand, because the thing that would have repaired it is the thing that was down
|
||||
// (novox/hq 04-ISSUES/201, ADR 0185).
|
||||
//
|
||||
// So the verbs this binary knows are served, and this one answers the reason instead of
|
||||
// nothing: a caller gets a sentence naming the fault, and everything else keeps working
|
||||
// — including the push that replaces this binary with the one whose verb it is.
|
||||
behind = append(behind, verb)
|
||||
reason := err
|
||||
handlers[verb] = func(context.Context, json.RawMessage) (any, error) {
|
||||
return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+
|
||||
"here cannot run it: %w. It is a verb of a newer build; this one is behind",
|
||||
verb, catalogue.ControllerSeatName, reason)
|
||||
}
|
||||
continue
|
||||
}
|
||||
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
||||
args := map[string]any{}
|
||||
@@ -249,7 +330,7 @@ func seatToolHandlers() (map[string]link.ToolHandler, error) {
|
||||
return runVerb(ctx, argv)
|
||||
}
|
||||
}
|
||||
return handlers, nil
|
||||
return handlers, behind, nil
|
||||
}
|
||||
|
||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||
@@ -289,3 +370,100 @@ func sampleArguments(v catalogue.Verb) map[string]any {
|
||||
}
|
||||
return sample
|
||||
}
|
||||
|
||||
// splitCommandLine splits a command line into words the way a POSIX shell does for the simple
|
||||
// cases a controller command needs: spaces separate, single or double quotes group, a backslash
|
||||
// escapes the next character inside double quotes or outside any. No expansion of anything.
|
||||
func splitCommandLine(line string) ([]string, error) {
|
||||
var words []string
|
||||
var cur strings.Builder
|
||||
inWord := false
|
||||
quote := rune(0)
|
||||
runes := []rune(line)
|
||||
for i := 0; i < len(runes); i++ {
|
||||
r := runes[i]
|
||||
switch {
|
||||
case quote == '\'':
|
||||
if r == '\'' {
|
||||
quote = 0
|
||||
} else {
|
||||
cur.WriteRune(r)
|
||||
}
|
||||
case quote == '"':
|
||||
if r == '"' {
|
||||
quote = 0
|
||||
} else if r == '\\' && i+1 < len(runes) {
|
||||
i++
|
||||
cur.WriteRune(runes[i])
|
||||
} else {
|
||||
cur.WriteRune(r)
|
||||
}
|
||||
case r == '\'' || r == '"':
|
||||
quote = r
|
||||
inWord = true
|
||||
case r == '\\' && i+1 < len(runes):
|
||||
i++
|
||||
cur.WriteRune(runes[i])
|
||||
inWord = true
|
||||
case r == ' ' || r == '\t' || r == '\n':
|
||||
if inWord {
|
||||
words = append(words, cur.String())
|
||||
cur.Reset()
|
||||
inWord = false
|
||||
}
|
||||
default:
|
||||
cur.WriteRune(r)
|
||||
inWord = true
|
||||
}
|
||||
}
|
||||
if quote != 0 {
|
||||
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
|
||||
}
|
||||
if inWord {
|
||||
words = append(words, cur.String())
|
||||
}
|
||||
return words, nil
|
||||
}
|
||||
|
||||
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
||||
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
||||
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
||||
func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
||||
about := map[string]catalogue.Verb{}
|
||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||
if s.Name == catalogue.ControllerSeatName {
|
||||
for _, v := range s.Serves {
|
||||
about[v.Name] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
verbs := make([]string, 0, len(handlers))
|
||||
for verb := range handlers {
|
||||
verbs = append(verbs, verb)
|
||||
}
|
||||
sort.Strings(verbs)
|
||||
var endpoints []micro.EndpointInfo
|
||||
for _, verb := range verbs {
|
||||
schema, _ := json.Marshal(about[verb].Input)
|
||||
// The same shape every tool runtime announces in (node-tools' announce package): the name is
|
||||
// `<seat>__<verb>`, as the protocol's characters allow; the metadata is what identifies it.
|
||||
endpoints = append(endpoints, micro.EndpointInfo{
|
||||
Name: catalogue.ControllerSeatName + "__" + verb,
|
||||
Subject: link.SeatToolSubject(catalogue.ControllerSeatName, verb),
|
||||
QueueGroup: "seat." + catalogue.ControllerSeatName,
|
||||
Metadata: map[string]string{
|
||||
"kind": "seat", "module": catalogue.ControllerSeatName, "tool": verb,
|
||||
"seat": catalogue.ControllerSeatName, "scope": "mesh", "interchangeable": "false",
|
||||
"description": about[verb].Description, "schema": string(schema),
|
||||
},
|
||||
})
|
||||
}
|
||||
return micro.Info{
|
||||
ServiceIdentity: micro.ServiceIdentity{
|
||||
Name: catalogue.ControllerSeatName, ID: "controller", Version: "0.1.0",
|
||||
Metadata: map[string]string{"seat": catalogue.ControllerSeatName, "scope": "mesh"},
|
||||
},
|
||||
Description: "the mesh's own verbs, answered by the holder of the mesh-controller seat",
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -130,10 +133,13 @@ func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||
|
||||
// What `tools` answers is the seats' records, with each verb's schema.
|
||||
func TestToolsAnswersTheSeatsRecords(t *testing.T) {
|
||||
handlers, err := seatToolHandlers()
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(behind) != 0 {
|
||||
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
|
||||
}
|
||||
if len(handlers) != len(catalogue.ControllerVerbs) {
|
||||
t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs))
|
||||
}
|
||||
@@ -171,3 +177,122 @@ func TestAJSONVerbsAnswerIsItsStandardOutput(t *testing.T) {
|
||||
t.Fatalf("stderr and stdout are both what the command said: %s", answer.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// `command` is the generic verb: the command line as given, split as a shell would, nothing added —
|
||||
// so an operator's `node account g14 jochen` is one call through the console rather than a shell on
|
||||
// the control node (novox/hq ADR 0154, ADR 0175).
|
||||
func TestCommandRunsTheLineAsGiven(t *testing.T) {
|
||||
argv, err := argvFor("command", map[string]any{"command": "node account g14 jochen"})
|
||||
if err != nil || strings.Join(argv, " ") != "node account g14 jochen" {
|
||||
t.Fatalf("a plain line: %v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("command", map[string]any{"command": `settings set dnsmasq '{"a": "b c"}' --node ace`})
|
||||
if err != nil || len(argv) != 6 || argv[3] != `{"a": "b c"}` {
|
||||
t.Fatalf("a quoted word stays one word: %q %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("command", map[string]any{"command": `node add "the box" --adopted`})
|
||||
if err != nil || len(argv) != 4 || argv[2] != "the box" {
|
||||
t.Fatalf("double quotes group: %q %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("command", map[string]any{"command": " "}); err == nil {
|
||||
t.Fatal("an empty line was accepted")
|
||||
}
|
||||
if _, err := argvFor("command", map[string]any{"command": `node "unclosed`}); err == nil {
|
||||
t.Fatal("an unclosed quote was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A verb in the row that this binary cannot run does not take the control plane off the bus: the
|
||||
// rest are served, the unknown one answers the reason, and the start-up names it (novox/hq ADR
|
||||
// 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by
|
||||
// a person running the binary by hand — the push that would have repaired it needs the control
|
||||
// plane that was down.
|
||||
func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) {
|
||||
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
||||
if !known {
|
||||
t.Fatal("no controller seat")
|
||||
}
|
||||
// The row as a newer control plane would have written it: every verb this build knows, and one
|
||||
// it does not.
|
||||
widened := seat
|
||||
widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...),
|
||||
catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"})
|
||||
rows := catalogue.DefaultSeats()
|
||||
for i := range rows {
|
||||
if rows[i].Name == catalogue.ControllerSeatName {
|
||||
rows[i] = widened
|
||||
}
|
||||
}
|
||||
catalogue.UseSeats(rows)
|
||||
t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
|
||||
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatalf("a row with one unknown verb refused to serve at all: %v", err)
|
||||
}
|
||||
if len(behind) != 1 || behind[0] != "teleport" {
|
||||
t.Fatalf("the verbs this build cannot run were reported as %v", behind)
|
||||
}
|
||||
if len(handlers) != len(widened.Serves) {
|
||||
t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves))
|
||||
}
|
||||
for _, known := range []string{"status", "nodes", "push"} {
|
||||
if handlers[known] == nil {
|
||||
t.Errorf("%s is not served although this build knows it", known)
|
||||
}
|
||||
}
|
||||
_, err = handlers["teleport"](context.Background(), nil)
|
||||
if err == nil {
|
||||
t.Fatal("the unknown verb answered as though it had run")
|
||||
}
|
||||
for _, want := range []string{"teleport", "cannot run it", "behind"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the answer does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0195: the console's discovery reads the machines and the modules; they answer as JSON,
|
||||
// as status and seats do, so nothing parses a printed column.
|
||||
func TestTheNodesAndModulesVerbsAnswerAsJSON(t *testing.T) {
|
||||
for verb, want := range map[string]string{"nodes": "[node list --json]", "modules": "[module list --json]"} {
|
||||
argv, err := argvFor(verb, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(argv) != want {
|
||||
t.Errorf("%s runs %v, want %s", verb, argv, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0197: the controller announces exactly the verbs it serves, each on the subject and
|
||||
// queue it serves it on, with the seat's own description and schema, in NATS's services format.
|
||||
func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) {
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info := seatAnnouncement(handlers)
|
||||
if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" {
|
||||
t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity)
|
||||
}
|
||||
if len(info.Endpoints) != len(handlers) {
|
||||
t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers))
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
verb := e.Metadata["tool"]
|
||||
if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb {
|
||||
t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb)
|
||||
}
|
||||
if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName {
|
||||
t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata)
|
||||
}
|
||||
if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName {
|
||||
t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup)
|
||||
}
|
||||
if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" {
|
||||
t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,10 @@ type sendable struct {
|
||||
// the same composition as its received files, and every machine's private-network address.
|
||||
Received map[string]map[string][]catalogue.Contribution
|
||||
Mesh []string
|
||||
// BusUsers is the bus's user list this declaration carries, empty for every machine but the one
|
||||
// holding the bus; not sent apart from the file it is in. Its digest is recorded once sent, so
|
||||
// whether that machine must go first is read from the list alone (novox/hq issue 249).
|
||||
BusUsers string
|
||||
// LeftOut is every module of the machine's set left out of this declaration because a stored
|
||||
// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
|
||||
// keeps that module's held things and touches none of its containers; a machine is told
|
||||
|
||||
@@ -138,14 +138,14 @@ func printStatus(asked answers) error {
|
||||
len(quiet), strings.Join(said, "\n "))
|
||||
}
|
||||
|
||||
if open, late := openPlans(asked.plans); len(open) > 0 {
|
||||
if open, late := openPlans(asked.plans, asked.paused); len(open) > 0 {
|
||||
fmt.Printf("%d plan(s) open", len(open))
|
||||
if late > 0 {
|
||||
fmt.Printf(", %d waiting past %s", late, planWaitBound)
|
||||
}
|
||||
fmt.Println(":")
|
||||
for _, p := range open {
|
||||
fmt.Printf(" %s\n", planLine(p, time.Now()))
|
||||
fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.paused))
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
@@ -282,6 +282,22 @@ func printStatus(asked answers) error {
|
||||
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
||||
}
|
||||
|
||||
if len(asked.unheld) > 0 {
|
||||
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
||||
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
||||
// holds, until every machine has its holders and the switch makes it a refusal.
|
||||
fmt.Printf("%d module dependenc(ies) on a seat nothing on the machine holds (unheld, ADR 0207):\n",
|
||||
len(asked.unheld))
|
||||
for _, u := range asked.unheld {
|
||||
holders := "no module in the catalogue claims it yet"
|
||||
if len(u.Holders) > 0 {
|
||||
holders = "could be held by " + strings.Join(u.Holders, ", ")
|
||||
}
|
||||
fmt.Printf(" %-12s %-24s %-24s %s\n", u.Node, u.Module, u.Seat, holders)
|
||||
}
|
||||
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||
}
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||
@@ -386,10 +402,26 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// And which machines run a module whose resources a seat nothing there holds applies (novox/hq
|
||||
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||
// the private network is built from and should say nothing else; a machine that does not
|
||||
// resolve is already in refused, and is passed over here.
|
||||
for _, n := range out.nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
continue
|
||||
}
|
||||
return answers{}, err
|
||||
}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
}
|
||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||
|
||||
// And which machines are not running what the mesh would send them. The same question as a
|
||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||
@@ -496,7 +528,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.filtered) == 0
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0
|
||||
}
|
||||
|
||||
// hostSplit is which machines report which host version, for every version more than one machine
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 254, ADR 0218: a newer plan takes over what the older open plans of its repository
|
||||
// and branch had not built, and closes them as superseded; another repository's plan, another
|
||||
// branch's, and a plan made after it are left alone.
|
||||
func TestANewerPlanSupersedesTheOlderOpenPlansOfItsRepository(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
sent := at.Add(time.Minute)
|
||||
plan := func(id, repository, branch string, created time.Time, modules map[string]*inventory.PlanModule) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: repository, Branch: branch, Commit: id + "-commit",
|
||||
Created: created, State: inventory.PlanRolling, Modules: modules}
|
||||
}
|
||||
older := plan("plan-1", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{
|
||||
"gitea": {State: "built", SentAt: &sent}, // done with: stays done
|
||||
"keycloak": {State: "asked"}, // asked, not answered: folded
|
||||
"plex": {}, // not yet asked: folded
|
||||
"agent": {State: "built"}, // built, rolls out, not sent: folded
|
||||
"notes": {State: "built"}, // built, records: nothing to send
|
||||
})
|
||||
stuck := plan("plan-0", "Novox/Mesh-Catalog", "", at.Add(-time.Hour), map[string]*inventory.PlanModule{
|
||||
"runtime": {State: "asked"},
|
||||
})
|
||||
other := plan("plan-2", "novox/mesh-controller", "main", at, map[string]*inventory.PlanModule{"mesh-controller": {}})
|
||||
release := plan("plan-3", "novox/mesh-catalog", "release", at, map[string]*inventory.PlanModule{"lemurs": {}})
|
||||
later := plan("plan-5", "novox/mesh-catalog", "main", at.Add(2*time.Hour), map[string]*inventory.PlanModule{"later": {}})
|
||||
done := plan("plan-6", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{"finished": {}})
|
||||
done.State = inventory.PlanDone
|
||||
|
||||
newer := plan("plan-4", "novox/mesh-catalog", "main", at.Add(time.Hour), nil)
|
||||
newer.Commit = "97b1b2b0c0ffee"
|
||||
rollsOut := func(m string) bool { return m != "notes" }
|
||||
folded, closed := supersededBy(newer, []inventory.Plan{stuck, older, other, release, later, done, newer}, rollsOut)
|
||||
|
||||
if want := []string{"agent", "keycloak", "plex", "runtime"}; !reflect.DeepEqual(folded, want) {
|
||||
t.Fatalf("folded %v, wanted %v", folded, want)
|
||||
}
|
||||
var ids []string
|
||||
for _, p := range closed {
|
||||
ids = append(ids, p.ID)
|
||||
if p.State != inventory.PlanSuperseded || p.Open() {
|
||||
t.Errorf("%s was left %s", p.ID, p.State)
|
||||
}
|
||||
if !strings.Contains(p.Note, "plan-4") || !strings.Contains(p.Note, "97b1b2b0") {
|
||||
t.Errorf("%s does not name the plan that superseded it: %q", p.ID, p.Note)
|
||||
}
|
||||
}
|
||||
if want := []string{"plan-0", "plan-1"}; !reflect.DeepEqual(ids, want) {
|
||||
t.Fatalf("superseded %v, wanted %v — another repository, another branch, a later plan and a "+
|
||||
"finished one are left alone", ids, want)
|
||||
}
|
||||
if other.State != inventory.PlanRolling {
|
||||
t.Fatal("the plan handed in was changed in place")
|
||||
}
|
||||
if line := planLine(closed[1], time.Now()); !strings.Contains(line, "superseded") {
|
||||
t.Fatalf("a superseded plan reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 254: a person closes a plan that will not move again, by its id.
|
||||
func TestAPersonClosesAStuckPlan(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
|
||||
Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
|
||||
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
||||
t.Fatal("a plan already closed was closed again")
|
||||
}
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// An act says what it changed about the node it acted on, and nothing about the rest of the mesh
|
||||
// (novox/hq ADR 0207): after the seat dependencies shipped, every `push <node>` and `assign` printed
|
||||
// every node's unmet dependencies, a hundred lines around the one about the module just assigned.
|
||||
|
||||
func aContainer(name string) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: name, Version: "1",
|
||||
Resources: []map[string]any{{"id": name, "type": "container", "image": name}}}
|
||||
}
|
||||
|
||||
func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aContainer("web"))
|
||||
register(t, open, aContainer("db"))
|
||||
// anchor already lacks a runtime for db: true, and not this act's to say.
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := assign(ctx, open, "laptop", "web")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, said)
|
||||
}
|
||||
if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) {
|
||||
t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said)
|
||||
}
|
||||
for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} {
|
||||
if strings.Contains(said, not) {
|
||||
t.Errorf("the assignment says %q, which it did not change:\n%s", not, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) {
|
||||
shelf := map[string]catalogue.Manifest{
|
||||
"web": aContainer("web"),
|
||||
"docker": {Module: "docker", Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat}},
|
||||
Resources: []map[string]any{{"id": "d", "type": "container", "image": "dind"}}},
|
||||
}
|
||||
lines := unheldChange(shelf, "laptop", []string{"web"}, []string{"web", "docker"})
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") {
|
||||
t.Errorf("meeting a dependency said %v", lines)
|
||||
}
|
||||
// Taking the dependent off says nothing: the dependency went with its module.
|
||||
if lines := unheldChange(shelf, "laptop", []string{"web"}, nil); len(lines) != 0 {
|
||||
t.Errorf("unassigning the dependent said %v", lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPushSaysANamedNodesDependenciesAndCountsTheRest(t *testing.T) {
|
||||
unheld := map[string][]catalogue.Unheld{
|
||||
"anchor": {{Node: "anchor", Module: "db", Seat: catalogue.ContainerRuntimeSeat}},
|
||||
"laptop": {{Node: "laptop", Module: "web", Seat: catalogue.ContainerRuntimeSeat},
|
||||
{Node: "laptop", Module: "sshd", Seat: catalogue.ServiceManagerSeat}},
|
||||
}
|
||||
var named bytes.Buffer
|
||||
reportUnheldPushed(&named, true, []string{"laptop"}, unheld)
|
||||
got := named.String()
|
||||
if !strings.Contains(got, "laptop has 2 unmet") || !strings.Contains(got, "web on laptop") ||
|
||||
!strings.Contains(got, "sshd on laptop") || strings.Contains(got, "anchor") {
|
||||
t.Errorf("a named push said:\n%s", got)
|
||||
}
|
||||
var all bytes.Buffer
|
||||
reportUnheldPushed(&all, false, []string{"anchor", "laptop", "quiet"}, unheld)
|
||||
want := "anchor: 1 unmet seat dependenc(ies) — see `status`\nlaptop: 2 unmet seat dependenc(ies) — see `status`\n"
|
||||
if all.String() != want {
|
||||
t.Errorf("a push to every node said:\n%s\nwant\n%s", all.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheServingControllerLogsEachChange(t *testing.T) {
|
||||
read := func(f func()) string {
|
||||
old := os.Stderr
|
||||
r, w, _ := os.Pipe()
|
||||
os.Stderr = w
|
||||
f()
|
||||
_ = w.Close()
|
||||
os.Stderr = old
|
||||
var b bytes.Buffer
|
||||
_, _ = b.ReadFrom(r)
|
||||
return b.String()
|
||||
}
|
||||
u := []catalogue.Unheld{{Node: "n1", Module: "web", Seat: catalogue.ContainerRuntimeSeat}}
|
||||
if got := read(func() { logUnheld("n1", u) }); got != "" {
|
||||
t.Errorf("a command logged:\n%s", got)
|
||||
}
|
||||
logUnheldChanges = true
|
||||
defer func() { logUnheldChanges = false }()
|
||||
if got := read(func() { logUnheld("n1", u) }); !strings.Contains(got, "web on n1") {
|
||||
t.Errorf("the serving controller did not log a change:\n%s", got)
|
||||
}
|
||||
if got := read(func() { logUnheld("n1", u) }); got != "" {
|
||||
t.Errorf("an unchanged report was logged again:\n%s", got)
|
||||
}
|
||||
}
|
||||
+171
-11
@@ -5,6 +5,8 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -54,10 +56,26 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// **A plan that holds the module rolls it out, and this does not** (novox/hq issue 249, ADR
|
||||
// 0218). A merge's plan builds the module and sends it one machine first, the rest once that one
|
||||
// has applied it; this announcement arrives as the build registers, and sending here too — one
|
||||
// machine after another without waiting for any to apply — put the new bundle on every machine in
|
||||
// the same minute, whatever the plan was waiting for. A move no plan answers (a build asked by
|
||||
// hand) is still this handler's.
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil {
|
||||
return notNow(err)
|
||||
} else if id := rolledOutByAPlan(plans, u.Module); id != "" {
|
||||
// Said with its remedy: a plan that ends without sending it — failed, or closed by hand — leaves
|
||||
// these machines behind, which `status` lists and `push --behind` sends (novox/hq issue 249).
|
||||
fmt.Printf("%s moved to %s; %s rolls it out to %s — if that plan ends without sending it, "+
|
||||
"`status` lists them as behind and `push --behind` sends it\n",
|
||||
u.Module, shortCommit(u.Commit), id, readableList(on))
|
||||
return nil
|
||||
}
|
||||
if decision.Together {
|
||||
fmt.Printf("%s moved to %s; sending %s together\n",
|
||||
u.Module, shortCommit(u.Commit), readableList(on))
|
||||
return sendTo(ctx, f.open, on)
|
||||
return askAgainOnGrants(sendTo(ctx, f.open, on))
|
||||
}
|
||||
// One at a time, and stopping at the first that fails.
|
||||
//
|
||||
@@ -68,13 +86,34 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
||||
u.Module, shortCommit(u.Commit), readableList(on))
|
||||
for _, node := range on {
|
||||
if err := sendTo(ctx, f.open, []string{node}); err != nil {
|
||||
return fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
|
||||
node, u.Module, err)
|
||||
return askAgainOnGrants(fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
|
||||
node, u.Module, err))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249):
|
||||
// an announcement handled by a send whose memberships could not be issued is held and redelivered,
|
||||
// rather than taken as handled with the machines left on the old version.
|
||||
func askAgainOnGrants(err error) error {
|
||||
if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// rolledOutByAPlan is the open plan that will send a module's machines its new build — one holding
|
||||
// the module that has not finished sending it — or empty when none will (novox/hq issue 249).
|
||||
func rolledOutByAPlan(plans []inventory.Plan, module string) string {
|
||||
for _, p := range plans {
|
||||
if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || (s.SentAt == nil && s.State != "failed")) {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// readableList names machines the way a sentence does, because this is read by a person deciding
|
||||
// whether an upgrade went where they expected.
|
||||
func readableList(names []string) string {
|
||||
@@ -283,6 +322,14 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
if isHistory(m.MergedAt, lastLookAt(entries, m)) {
|
||||
packaging = nil
|
||||
}
|
||||
// Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows
|
||||
// another branch is not part of this merge, and whoever is waiting for its change should read why.
|
||||
for _, e := range entries {
|
||||
if sameRepository(e.Source.Repository, m) && !sourceIs(e.Source, m) {
|
||||
fmt.Printf(" %s is built from %s/%s and follows %s, not %s; this merge leaves it out\n",
|
||||
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
|
||||
}
|
||||
}
|
||||
touched := whatTheMergeTouched(from, entries, m)
|
||||
for _, e := range touched {
|
||||
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
|
||||
@@ -306,7 +353,53 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
for _, e := range moved {
|
||||
movedNames = append(movedNames, e.Manifest.Module)
|
||||
}
|
||||
// Written and its first tier asked as one act on the plans (novox/hq issue 213): a timer on
|
||||
// another controller reading it between the two would ask the tier again.
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
defer release()
|
||||
plan := planOfMerge(m, movedNames, edges)
|
||||
// **A newer plan supersedes the older open plans of this repository and branch** (novox/hq issue
|
||||
// 254, ADR 0218): what they had not built is planned here again, and they are closed, so one plan
|
||||
// works a repository's modules at a time and a stuck one ends at the next merge.
|
||||
working, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
rollsOut := func(module string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return err == nil && u.RollOut
|
||||
}
|
||||
folded, superseded := supersededBy(plan, working, rollsOut)
|
||||
if len(folded) > 0 {
|
||||
held := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
held[e.Manifest.Module] = true
|
||||
}
|
||||
names := map[string]bool{}
|
||||
for _, name := range movedNames {
|
||||
names[name] = true
|
||||
}
|
||||
var also []string
|
||||
for _, name := range folded {
|
||||
// One the catalogue no longer holds would fail the newer plan's ask; it is not this
|
||||
// merge's to build.
|
||||
if held[name] && !names[name] {
|
||||
names[name] = true
|
||||
movedNames = append(movedNames, name)
|
||||
also = append(also, name)
|
||||
}
|
||||
}
|
||||
if len(also) > 0 {
|
||||
again := planOfMerge(m, movedNames, edges)
|
||||
again.ID, again.Created = plan.ID, plan.Created
|
||||
plan = again
|
||||
fmt.Printf(" %s, left unbuilt by an older plan of %s, are planned here again\n",
|
||||
strings.Join(also, ", "), plan.Repository)
|
||||
}
|
||||
}
|
||||
if hasCycle(plan.Tiers, edges) {
|
||||
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
||||
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
||||
@@ -314,6 +407,14 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
|
||||
// both open, which the next merge settles, rather than neither.
|
||||
for _, old := range superseded {
|
||||
if err := inv.SavePlan(ctx, old); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
fmt.Printf(" %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
|
||||
}
|
||||
var tiers []string
|
||||
for i, t := range plan.Tiers {
|
||||
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
||||
@@ -345,7 +446,24 @@ func sourceIs(s inventory.Source, m link.SourceMoved) bool {
|
||||
if !sameRepository(s.Repository, m) {
|
||||
return false
|
||||
}
|
||||
return s.Ref == "" || s.Ref == m.Base
|
||||
ref := followedBranch(s.Ref)
|
||||
return ref == "" || ref == m.Base
|
||||
}
|
||||
|
||||
// commitRef is a ref that names a commit rather than a branch: what `build --ref <commit>` asks for.
|
||||
var commitRef = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
|
||||
|
||||
// followedBranch is the branch a recorded ref means a module follows (novox/hq 04-ISSUES/215). **A
|
||||
// commit is never a branch to follow.** A build asked at a commit — to try one, or to pin it during a
|
||||
// fix — recorded that commit as the module's ref; every merge after it then failed to match the
|
||||
// module, its plan left it out without saying so, and every plan that rebuilt it asked for that same
|
||||
// old commit again. A commit recorded so is read as the repository's default branch, which is what
|
||||
// the module followed before it; a branch is followed as named.
|
||||
func followedBranch(ref string) string {
|
||||
if commitRef.MatchString(strings.TrimSpace(ref)) {
|
||||
return ""
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
// sameRepository is whether a recorded repository is the one a merge names, in either spelling it
|
||||
@@ -390,25 +508,45 @@ func lastLookAt(entries []inventory.Entry, m link.SourceMoved) time.Time {
|
||||
//
|
||||
// A change inside *another* module's directory is that module's business and not this one's, even
|
||||
// when the mesh does not hold that module: `known` is every module this repository is known to hold,
|
||||
// whatever branch it was registered from. That is also the limit of this — a repository whose shared
|
||||
// code sits inside a directory the mesh has never seen a module in reads as shared, and everything
|
||||
// is rebuilt. Rebuilding too much is the safe direction: the fault this whole path exists for is a
|
||||
// mesh that believes it is current and is not (novox/hq 04-ISSUES/131).
|
||||
// whatever branch it was registered from.
|
||||
//
|
||||
// **And a module the mesh has never seen is still a module** (novox/hq issue 252). A merge adding a
|
||||
// new module to the catalogue repository — `modules/newmod/module.json` and its files — read as a
|
||||
// change to shared code, because `modules/newmod` was nobody's known directory, and every module
|
||||
// built from the repository was rebuilt and rolled out for a module none of them is. So the
|
||||
// directories that hold modules are known too: the parents of the known modules' directories
|
||||
// (`modules`, never the root). A changed path `<parent>/<name>/…` belongs to the module at
|
||||
// `<parent>/<name>` — held or not — and rebuilds nothing else, **provided it is shown to be a
|
||||
// module**: its `module.json` is among the changed files (added, changed, or removed with it). A
|
||||
// directory under the same parent whose manifest the merge did not touch may as well be a shared
|
||||
// library (`modules/lib`), and that is still read as shared. Rebuilding too much remains the safe
|
||||
// direction: the fault this whole path exists for is a mesh that believes it is current and is not
|
||||
// (novox/hq 04-ISSUES/131). A file at the root, or directly in a parent, is shared as it always was.
|
||||
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry {
|
||||
// Nothing said about the files, or not all of them said: everything built from it is affected.
|
||||
if len(m.Paths) == 0 || m.PathsTruncated {
|
||||
return candidates
|
||||
}
|
||||
var dirs []string
|
||||
parents := map[string]bool{}
|
||||
for _, e := range known {
|
||||
if e.Source.Path != "" && sameRepository(e.Source.Repository, m) {
|
||||
dirs = append(dirs, e.Source.Path)
|
||||
dir := strings.Trim(e.Source.Path, "/")
|
||||
dirs = append(dirs, dir)
|
||||
if parent := path.Dir(dir); parent != "." && parent != "/" {
|
||||
parents[parent] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
changed := map[string]bool{}
|
||||
for _, p := range m.Paths {
|
||||
if !insideAny(p, dirs) {
|
||||
return candidates
|
||||
changed[strings.TrimPrefix(p, "/")] = true
|
||||
}
|
||||
for _, p := range m.Paths {
|
||||
if insideAny(p, dirs) || inAModuleOfItsOwn(p, parents, changed) {
|
||||
continue
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
var out []inventory.Entry
|
||||
for _, e := range candidates {
|
||||
@@ -419,6 +557,28 @@ func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved
|
||||
return out
|
||||
}
|
||||
|
||||
// inAModuleOfItsOwn is whether a changed file is inside a module directory the mesh does not know —
|
||||
// `<parent>/<name>/…` under a directory known to hold modules, whose `module.json` the same merge
|
||||
// changed (novox/hq issue 252). Such a file is that module's business and nobody else's.
|
||||
func inAModuleOfItsOwn(p string, parents, changed map[string]bool) bool {
|
||||
p = strings.TrimPrefix(p, "/")
|
||||
for parent := range parents {
|
||||
rest, under := strings.CutPrefix(p, parent+"/")
|
||||
if !under {
|
||||
continue
|
||||
}
|
||||
name, _, inADirectory := strings.Cut(rest, "/")
|
||||
if !inADirectory || name == "" {
|
||||
// A file directly in the parent — `modules/README.md` — is about all of them.
|
||||
continue
|
||||
}
|
||||
if changed[parent+"/"+name+"/module.json"] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// inside is whether a changed file is in a directory: that directory itself, or under it.
|
||||
func inside(path, dir string) bool {
|
||||
dir = strings.Trim(dir, "/")
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The holder of the build seat follows the controller that defines its worker (novox/hq issue 206).
|
||||
// On 2026-10-03 a plan put the build machine in tier 0 and the controller in tier 1; the new build
|
||||
// machine could not bind the worker the old controller had defined, and nothing could build the
|
||||
// controller that would have redefined it. The built-by edge from the controller to its build
|
||||
// machine yields to that order: the controller is built by whichever build machine is running.
|
||||
func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.T) {
|
||||
edges := []inventory.Edge{
|
||||
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
||||
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||
{From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
}
|
||||
set := reachableFrom([]string{"mesh-controller"}, edges)
|
||||
if len(set) != 3 {
|
||||
t.Fatalf("the controller, what packages it, and nothing more: %v", set)
|
||||
}
|
||||
tiers := tiersOf(set, edges)
|
||||
pos := map[string]int{}
|
||||
for i, tier := range tiers {
|
||||
for _, m := range tier {
|
||||
pos[m] = i
|
||||
}
|
||||
}
|
||||
if pos["mesh-controller"] != 0 {
|
||||
t.Fatalf("the controller first, built by the build machine that is running: %v", tiers)
|
||||
}
|
||||
if pos["build-agent"] <= pos["mesh-controller"] {
|
||||
t.Fatalf("the build machine after the controller that defines its worker: %v", tiers)
|
||||
}
|
||||
if pos["route-proxy"] <= pos["build-agent"] {
|
||||
t.Fatalf("what the build machine builds comes after it: %v", tiers)
|
||||
}
|
||||
if hasCycle(tiers, edges) {
|
||||
t.Fatalf("no cycle here: %v", tiers)
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,10 @@
|
||||
# The client is copied from the vendor's own image rather than installed from a distribution:
|
||||
# `apk add mc` on Alpine installs Midnight Commander, which is a different program with the same
|
||||
# name, and the failure would be a provisioner that starts cleanly and cannot do anything.
|
||||
FROM golang:1.25-alpine AS build
|
||||
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
|
||||
# build machine alike; the default only serves a hand build, and matches go.mod.
|
||||
ARG GO_BASE=golang:1.26-alpine
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
# Built here so a machine can be given it by the mesh rather than by somebody putting a binary on
|
||||
# it. Static and FROM scratch for the same reason the control plane's image is: it is fetched by
|
||||
# digest and run on a machine, and everything in it is something a person would have to audit.
|
||||
FROM golang:1.25-alpine AS build
|
||||
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
|
||||
# build machine alike; the default only serves a hand build, and matches go.mod.
|
||||
ARG GO_BASE=golang:1.26-alpine
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
#
|
||||
# FROM scratch, like the postgres one and unlike the bucket one: it speaks the store's own wire
|
||||
# protocol directly and needs no client in the image.
|
||||
FROM golang:1.25-alpine AS build
|
||||
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
|
||||
# build machine alike; the default only serves a hand build, and matches go.mod.
|
||||
ARG GO_BASE=golang:1.26-alpine
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
#
|
||||
# Static and FROM scratch like the control plane's image, and for the same reason: it is fetched
|
||||
# by digest and run on a machine, so everything in it is something a person would have to audit.
|
||||
FROM golang:1.25-alpine AS build
|
||||
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
|
||||
# build machine alike; the default only serves a hand build, and matches go.mod.
|
||||
ARG GO_BASE=golang:1.26-alpine
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A backend behind the proxy learns the client used TLS and which name it asked for, so the addresses
|
||||
// it writes into its own pages are the ones a client can use (2026-10-03: a forge's Go import tag
|
||||
// named an http clone URL, and Go refused the module path).
|
||||
func TestABackendIsToldTheRequestWasHTTPSAndForWhichName(t *testing.T) {
|
||||
var proto, host, fwdHost, fwdFor string
|
||||
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
proto, host, fwdHost, fwdFor = r.Header.Get("X-Forwarded-Proto"), r.Host, r.Header.Get("X-Forwarded-Host"), r.Header.Get("X-Forwarded-For")
|
||||
}))
|
||||
defer backend.Close()
|
||||
where, _ := url.Parse(backend.URL)
|
||||
req := httptest.NewRequest(http.MethodGet, "https://git.example.org/novox/mesh-sdk/go?go-get=1", nil)
|
||||
req.TLS = &tls.ConnectionState{}
|
||||
req.Host = "git.example.org"
|
||||
req.RemoteAddr = "192.0.2.7:51000"
|
||||
towards(where).ServeHTTP(httptest.NewRecorder(), req)
|
||||
if proto != "https" || fwdHost != "git.example.org" || host != "git.example.org" || fwdFor != "192.0.2.7" {
|
||||
t.Errorf("the backend was told proto=%q host=%q forwarded-host=%q for=%q", proto, host, fwdHost, fwdFor)
|
||||
}
|
||||
}
|
||||
@@ -273,7 +273,7 @@ func (t *table) set(routes map[string][]rule, public map[string]bool) {
|
||||
log.Printf("route %s points at %q, which is not a URL: %v", host, r.target, err)
|
||||
continue
|
||||
}
|
||||
r.to = httputil.NewSingleHostReverseProxy(where)
|
||||
r.to = towards(where)
|
||||
if r.insecure {
|
||||
r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
}
|
||||
@@ -732,6 +732,11 @@ func handler(held *table) http.Handler {
|
||||
// And since a host may now be routed only on some paths, those are a third thing:
|
||||
// saying "no route for this name" while listing that very name as served is a
|
||||
// contradiction an operator would have to disbelieve the proxy to get past.
|
||||
// **Said in the log as well as to the client.** A name this mesh does not serve, asked
|
||||
// for from outside, is what a scanner does, and the machine's intrusion prevention reads
|
||||
// this proxy's log for exactly that line (novox/hq ADR 0179): the address last, as the
|
||||
// jail's filter expects it.
|
||||
log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr)
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
if !hidden && held.routed(r.Host) {
|
||||
@@ -1055,3 +1060,17 @@ func asPort(v any) (int, bool) {
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// towards proxies to one backend and tells it what the client asked: **X-Forwarded-Proto, -Host and
|
||||
// -For**, set from the request this proxy received. A backend that builds its own addresses — a forge
|
||||
// writing its clone URL into a page, a login redirect — otherwise sees the plain HTTP hop from this
|
||||
// proxy and writes `http://`, though every client reached it over TLS: Go refused the forge's module
|
||||
// path for exactly that on 2026-10-03, its import tag naming an http clone URL.
|
||||
// The standard library's NewSingleHostReverseProxy sets only X-Forwarded-For.
|
||||
func towards(where *url.URL) *httputil.ReverseProxy {
|
||||
return &httputil.ReverseProxy{Rewrite: func(pr *httputil.ProxyRequest) {
|
||||
pr.SetURL(where)
|
||||
pr.Out.Host = pr.In.Host
|
||||
pr.SetXForwarded()
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
|
||||
// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
|
||||
// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
|
||||
// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
|
||||
// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
|
||||
// bare blob is unreferenced, so the first real collection would have deleted every archive the
|
||||
// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
|
||||
// collector runs `--dry-run` until this is true.
|
||||
//
|
||||
// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
|
||||
// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
|
||||
// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
|
||||
// holder first, which is what lets the bytes go at the next collection.
|
||||
//
|
||||
// **Nothing a machine reads changes.** The recorded reference stays
|
||||
// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
|
||||
// before. The holder is the store's bookkeeping, not a second way to reach anything.
|
||||
//
|
||||
// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
|
||||
// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
|
||||
// compute which manifest holds any archive from the reference it already has plus one HEAD for the
|
||||
// size. No schema change, no second record that could disagree with the store.
|
||||
|
||||
const (
|
||||
// mediaManifest is the type a holder is put and asked for as.
|
||||
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
|
||||
// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
|
||||
// whose only purpose is to name its layer.
|
||||
mediaEmpty = "application/vnd.oci.empty.v1+json"
|
||||
// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
|
||||
emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
|
||||
// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
|
||||
// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
|
||||
// not read from anywhere, because the holder must be recomputable from the reference alone.
|
||||
mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
|
||||
)
|
||||
|
||||
// emptyConfig is the content emptyDigest names.
|
||||
var emptyConfig = []byte("{}")
|
||||
|
||||
// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
|
||||
// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
|
||||
// (measured 2026-09-28; internal/builder/registry.go says how that was found).
|
||||
var manifestAccept = []string{
|
||||
mediaManifest,
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
|
||||
type descriptor struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type holderManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Config descriptor `json:"config"`
|
||||
Layers []descriptor `json:"layers"`
|
||||
}
|
||||
|
||||
// Holder is the manifest that holds an archive in the store, and its digest.
|
||||
//
|
||||
// A pure function of the archive's digest and size: the same two in give the same bytes out,
|
||||
// always, because `encoding/json` writes a struct's fields in their declared order and there is
|
||||
// nothing here that varies by when or where it was composed.
|
||||
func Holder(digest string, size int64) (body []byte, holder string) {
|
||||
body, err := json.Marshal(holderManifest{
|
||||
SchemaVersion: 2,
|
||||
MediaType: mediaManifest,
|
||||
Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
|
||||
Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
|
||||
})
|
||||
if err != nil {
|
||||
// Marshalling a struct of strings and integers cannot fail.
|
||||
panic(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return body, "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
|
||||
//
|
||||
// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
|
||||
// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
|
||||
// is what lets the sweep run it over every kept archive on every build and so backfill the bare
|
||||
// blobs published before holders existed (novox/hq issue 253).
|
||||
//
|
||||
// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
|
||||
// fact the caller reports rather than one this invents a remedy for.
|
||||
func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil || !archive {
|
||||
return false, err
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return s.HoldBlob(ctx, repository, digest, size)
|
||||
}
|
||||
|
||||
// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
|
||||
// question an operator needs answered with "none unheld" before the collector is let loose.
|
||||
//
|
||||
// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
|
||||
func (s Store) Held(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !archive {
|
||||
return true, nil
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
}
|
||||
|
||||
// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
|
||||
// there already. Answers whether it wrote one.
|
||||
//
|
||||
// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
|
||||
// the store reports. Both arrive at the same holder, which is the point of composing it.
|
||||
func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
|
||||
if s.Address == "" {
|
||||
return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
|
||||
}
|
||||
body, holder := Holder(digest, size)
|
||||
there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if there {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// The config must be in the repository before a manifest naming it is accepted: a registry
|
||||
// refuses a manifest whose blobs it cannot find there, which is the property that makes a
|
||||
// holder mean something.
|
||||
if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// **By digest, never by tag.** A tag would be one more name to move and one more thing the
|
||||
// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
|
||||
// it pins by digest, and an untagged manifest is kept by plain collection.
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
|
||||
s.url(repository, "manifests", holder), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
request.Header.Set("Content-Type", mediaManifest)
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
|
||||
repository, digest, response.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
|
||||
//
|
||||
// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
|
||||
// collector would keep its bytes for ever on the strength of it — the sweep would record the
|
||||
// archive collected while the disk said otherwise. Deleting the holder first and failing before
|
||||
// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
|
||||
//
|
||||
// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
|
||||
// and without the blob there is nothing left for a holder to keep. A store that never had a holder
|
||||
// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
|
||||
// that is the outcome wanted.
|
||||
func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
|
||||
if err == Gone {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// archive reads a recorded reference into its repository and digest, and whether it is an archive
|
||||
// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
|
||||
func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
return repository, digest, kind == "blobs", nil
|
||||
}
|
||||
|
||||
// blobSize is how large the store says a blob is; Gone when it does not have it.
|
||||
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return 0, Gone
|
||||
default:
|
||||
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
|
||||
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
|
||||
// it, and the header is what the registry itself wrote.
|
||||
if length := response.Header.Get("Content-Length"); length != "" {
|
||||
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
if response.ContentLength >= 0 {
|
||||
return response.ContentLength, nil
|
||||
}
|
||||
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
|
||||
}
|
||||
|
||||
// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
|
||||
// naming the digest — the registry's own two steps, the same the builder takes for an archive.
|
||||
func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
|
||||
if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
|
||||
return err
|
||||
} else if there {
|
||||
return nil
|
||||
}
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
begun, err := s.client().Do(start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
|
||||
}
|
||||
begun.Body.Close()
|
||||
if begun.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
|
||||
}
|
||||
where := begun.Header.Get("Location")
|
||||
if where == "" {
|
||||
return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
|
||||
}
|
||||
if strings.HasPrefix(where, "/") {
|
||||
where = "http://" + s.Address + where
|
||||
}
|
||||
separator := "?"
|
||||
if strings.Contains(where, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put.Header.Set("Content-Type", "application/octet-stream")
|
||||
done, err := s.client().Do(put)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done.Body.Close()
|
||||
if done.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||
return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// has is whether the store answers 200 for a HEAD at that URL.
|
||||
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, media := range accept {
|
||||
request.Header.Add("Accept", media)
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) url(repository, kind, digest string) string {
|
||||
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
|
||||
}
|
||||
|
||||
func (s Store) client() *http.Client {
|
||||
if s.HTTP != nil {
|
||||
return s.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
|
||||
// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
|
||||
// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
|
||||
// type. What is asserted is this side's decisions; the live test below asserts the registry's.
|
||||
|
||||
type memRegistry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // repository + "@" + digest
|
||||
manifests map[string][]byte // repository + "@" + digest
|
||||
writes []string // every PUT and DELETE, as "METHOD path"
|
||||
}
|
||||
|
||||
func digestOf(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (m *memRegistry) serve(t *testing.T) Store {
|
||||
t.Helper()
|
||||
m.blobs = map[string][]byte{}
|
||||
m.manifests = map[string][]byte{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
path := strings.TrimPrefix(r.URL.Path, "/v2/")
|
||||
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
|
||||
m.writes = append(m.writes, r.Method+" "+r.URL.Path)
|
||||
}
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
|
||||
repository := strings.TrimSuffix(path, "/blobs/uploads/")
|
||||
w.Header().Set("Location", "/upload/"+repository+"?state=x")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
|
||||
repository := strings.TrimPrefix(r.URL.Path, "/upload/")
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
digest := r.URL.Query().Get("digest")
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case strings.Contains(path, "/blobs/"):
|
||||
repository, digest, _ := strings.Cut(path, "/blobs/")
|
||||
key := repository + "@" + digest
|
||||
body, ok := m.blobs[key]
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodDelete:
|
||||
delete(m.blobs, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
case strings.Contains(path, "/manifests/"):
|
||||
repository, digest, _ := strings.Cut(path, "/manifests/")
|
||||
key := repository + "@" + digest
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodPut:
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var named holderManifest
|
||||
if err := json.Unmarshal(body, &named); err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, d := range append([]descriptor{named.Config}, named.Layers...) {
|
||||
if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
|
||||
return
|
||||
}
|
||||
}
|
||||
m.manifests[key] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case http.MethodDelete:
|
||||
if _, ok := m.manifests[key]; !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(m.manifests, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
|
||||
func (m *memRegistry) bare(repository string, body []byte) string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
digest := digestOf(body)
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||
}
|
||||
|
||||
func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
|
||||
// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
|
||||
// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
|
||||
// never be mistaken for one of a different blob.
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
one, first := Holder(digest, 42)
|
||||
two, second := Holder(digest, 42)
|
||||
if !bytes.Equal(one, two) || first != second {
|
||||
t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
|
||||
}
|
||||
if _, other := Holder(digest, 43); other == first {
|
||||
t.Fatal("a different size composed the same holder")
|
||||
}
|
||||
want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
|
||||
`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
|
||||
`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
|
||||
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
|
||||
if string(one) != want {
|
||||
t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
|
||||
}
|
||||
if digestOf(emptyConfig) != emptyDigest {
|
||||
t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
|
||||
// The archives published before this have no holder. Hold, run over every kept archive on
|
||||
// every sweep, writes one the first time and nothing after.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("a theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
|
||||
if held, err := store.Held(ctx, reference); err != nil || held {
|
||||
t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
|
||||
}
|
||||
wrote, err := store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !wrote {
|
||||
t.Fatal("holding a bare archive wrote nothing")
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
if _, ok := m.manifests["shell/config@"+holder]; !ok {
|
||||
t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
|
||||
}
|
||||
if held, err := store.Held(ctx, reference); err != nil || !held {
|
||||
t.Fatalf("after holding, held=%v (%v)", held, err)
|
||||
}
|
||||
|
||||
writes := len(m.writes)
|
||||
wrote, err = store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrote || len(m.writes) != writes {
|
||||
t.Fatalf("holding again wrote %v", m.writes[writes:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// An image is its own manifest: nothing is asked.
|
||||
wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil || wrote || len(m.writes) != 0 {
|
||||
t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
|
||||
}
|
||||
// An archive the store does not have is a fact to report, not something to invent a holder for.
|
||||
_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("holding a missing archive answered %v, want Gone", err)
|
||||
}
|
||||
// And a reference that is not the mesh's is refused as such.
|
||||
if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
|
||||
// A holder left behind would keep the bytes through every collection while the record said
|
||||
// collected; the link deleted first and the holder failing after would be that exactly.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("an old theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
if _, err := store.Hold(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.writes = nil
|
||||
|
||||
if err := store.LetGo(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
want := []string{
|
||||
"DELETE /v2/shell/config/manifests/" + holder,
|
||||
"DELETE /v2/shell/config/blobs/" + digestOf(body),
|
||||
}
|
||||
if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if len(m.manifests) != 0 {
|
||||
t.Fatalf("a holder survived: %v", m.manifests)
|
||||
}
|
||||
// Asked again, the archive is already gone, which is the outcome wanted.
|
||||
if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
|
||||
t.Fatalf("letting go twice answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
|
||||
// An archive published before holders and let go of before any sweep held it: the holder's
|
||||
// delete answers 404, which is the outcome wanted, and the blob still goes.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
reference := m.bare("shell/config", []byte("never held"))
|
||||
if err := store.LetGo(context.Background(), reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.blobs) != 0 {
|
||||
t.Fatalf("the blob survived: %v", m.blobs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The registry's own collector keeps a held archive and takes a bare one (novox/hq issue 253,
|
||||
// ADR 0189).
|
||||
//
|
||||
// Everything else here is asserted against a fake, which can only say what this side asks. This is
|
||||
// the one question a fake cannot answer — what `registry garbage-collect` actually does with what
|
||||
// this side wrote — and it is the whole of whether the store's nightly step may stop being a dry
|
||||
// run. Against the very image the mesh's store runs:
|
||||
//
|
||||
// docker run -d --rm --name mesh-controller-registry -p 15000:5000 \
|
||||
// -e REGISTRY_STORAGE_DELETE_ENABLED=true registry:2.8.3
|
||||
// MESH_TEST_REGISTRY=127.0.0.1:15000 MESH_TEST_REGISTRY_CONTAINER=mesh-controller-registry \
|
||||
// go test -run Live ./internal/artifacts/
|
||||
// docker stop mesh-controller-registry
|
||||
//
|
||||
// Skipped without both variables: it needs a registry it may write to and collect, and a container
|
||||
// to run the collector in.
|
||||
func TestLiveTheRegistrysCollectorKeepsWhatIsHeldAndTakesWhatIsNot(t *testing.T) {
|
||||
address := os.Getenv("MESH_TEST_REGISTRY")
|
||||
container := os.Getenv("MESH_TEST_REGISTRY_CONTAINER")
|
||||
if address == "" || container == "" {
|
||||
t.Skip("no MESH_TEST_REGISTRY / MESH_TEST_REGISTRY_CONTAINER; see this test's comment for the registry to raise")
|
||||
}
|
||||
ctx := context.Background()
|
||||
store := Store{Address: address}
|
||||
run := time.Now().UnixNano()
|
||||
|
||||
// Four archives in four repositories, each a different story. Distinct bytes per run, so a
|
||||
// registry reused across runs cannot answer for an earlier one.
|
||||
put := func(name string) (repository, digest string, body []byte) {
|
||||
repository = fmt.Sprintf("live-%d/%s", run, name)
|
||||
body = []byte(fmt.Sprintf("%s archive of run %d", name, run))
|
||||
digest = digestOf(body)
|
||||
if err := store.putBlob(ctx, repository, digest, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return repository, digest, body
|
||||
}
|
||||
reference := func(repository, digest string) string {
|
||||
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||
}
|
||||
|
||||
// Published held — what PublishArchive now does.
|
||||
heldRepo, heldDigest, heldBody := put("held")
|
||||
if _, err := store.HoldBlob(ctx, heldRepo, heldDigest, int64(len(heldBody))); err != nil {
|
||||
t.Fatalf("the registry refused a holder: %v", err)
|
||||
}
|
||||
// Published bare, as before, and never held: what the collector must take.
|
||||
_, bareDigest, _ := put("bare")
|
||||
// Published bare and then held by the sweep: the backfill.
|
||||
backRepo, backDigest, backBody := put("backfilled")
|
||||
if wrote, err := store.Hold(ctx, reference(backRepo, backDigest)); err != nil || !wrote {
|
||||
t.Fatalf("backfilling wrote=%v: %v", wrote, err)
|
||||
}
|
||||
if held, err := store.Held(ctx, reference(backRepo, backDigest)); err != nil || !held {
|
||||
t.Fatalf("after backfilling, held=%v: %v", held, err)
|
||||
}
|
||||
// Held, and then let go of by the sweep: holder first, then the link.
|
||||
goneRepo, goneDigest, _ := put("let-go")
|
||||
if _, err := store.Hold(ctx, reference(goneRepo, goneDigest)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.LetGo(ctx, reference(goneRepo, goneDigest)); err != nil {
|
||||
t.Fatalf("letting go of a held archive: %v", err)
|
||||
}
|
||||
|
||||
collected, err := exec.CommandContext(ctx, "docker", "exec", container,
|
||||
"registry", "garbage-collect", "/etc/docker/registry/config.yml").CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("the collector failed: %v\n%s", err, collected)
|
||||
}
|
||||
t.Logf("the collector said:\n%s", lastLines(string(collected), 12))
|
||||
|
||||
// What is asserted is the bytes on the store's disk, not what the running server answers: the
|
||||
// server caches blob descriptors in memory and can answer for a blob the collector removed.
|
||||
onDisk := func(digest string) bool {
|
||||
hex := strings.TrimPrefix(digest, "sha256:")
|
||||
path := "/var/lib/registry/docker/registry/v2/blobs/sha256/" + hex[:2] + "/" + hex + "/data"
|
||||
return exec.CommandContext(ctx, "docker", "exec", container, "test", "-f", path).Run() == nil
|
||||
}
|
||||
if !onDisk(heldDigest) {
|
||||
t.Error("the collector took an archive published held")
|
||||
}
|
||||
if !onDisk(backDigest) {
|
||||
t.Error("the collector took an archive the sweep backfilled a holder for")
|
||||
}
|
||||
if onDisk(bareDigest) {
|
||||
t.Error("the collector kept a bare archive — then the holders prove nothing, and this test is wrong")
|
||||
}
|
||||
if onDisk(goneDigest) {
|
||||
t.Error("the collector kept an archive the sweep let go of: its holder outlived its link")
|
||||
}
|
||||
// And what survived is still fetched exactly as machines fetch it: the blob, by digest.
|
||||
for repository, want := range map[string][]byte{heldRepo: heldBody, backRepo: backBody} {
|
||||
digest := digestOf(want)
|
||||
response, err := http.Get(catalogue.Routed(reference(repository, digest), address))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK || !bytes.Equal(got, want) {
|
||||
t.Errorf("%s answered %s with %q after collection", repository, response.Status, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func lastLines(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Package artifacts speaks to the mesh's artifact store over its own door.
|
||||
//
|
||||
// Only what the mesh needs that nothing else does: letting go of something it put there
|
||||
// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
|
||||
// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
|
||||
// is every machine's, through its runtime. This is the one operation that belongs to the thing
|
||||
// holding the records, because it is the only one that is a decision rather than a transfer.
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Store is the artifact store at an address, as this machine reaches it.
|
||||
type Store struct {
|
||||
// Address is `host:port` — the store as the caller reaches it now, composed and never
|
||||
// recorded (novox/hq 04-ISSUES/102).
|
||||
Address string
|
||||
// HTTP is the client used; nil is a client with a modest timeout.
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
|
||||
var Gone = errors.New("the store does not hold it")
|
||||
|
||||
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing
|
||||
// the store holds by digest.
|
||||
//
|
||||
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite
|
||||
// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an
|
||||
// error" is how a cautious loop became one that did nothing while reporting the right number.
|
||||
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||
|
||||
// LetGo asks the store to drop one artifact the mesh recorded making.
|
||||
//
|
||||
// Takes a reference as the mesh records it — `artifact-store://<module>/<artifact>@sha256:…` for
|
||||
// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
|
||||
// and composes the address here at the moment of use.
|
||||
//
|
||||
// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
|
||||
// issue 253); an image in one.
|
||||
//
|
||||
// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
|
||||
// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
|
||||
// which of the two happened.
|
||||
func (s Store) LetGo(ctx context.Context, reference string) error {
|
||||
// **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its
|
||||
// own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…`
|
||||
// as the mesh's too — it cannot tell one registry host from another — so normalising belongs
|
||||
// where the provenance is known, which is the sweep reading its own build records, not here
|
||||
// where the only job is to refuse anything that is not plainly ours.
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
// Nothing the mesh put in its own store. Refused rather than attempted: composing a
|
||||
// delete for a reference of unknown shape is how a sweep reaches something that is not
|
||||
// the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on.
|
||||
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if kind == "blobs" {
|
||||
// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
|
||||
// naming the blob would keep its bytes through every collection while the record said
|
||||
// collected. Gone here means the store has no such blob, so there is nothing to let go.
|
||||
if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.remove(ctx, s.url(repository, kind, digest), reference)
|
||||
}
|
||||
|
||||
// remove asks the store to delete what is at url. Gone when it has no such thing.
|
||||
func (s Store) remove(ctx context.Context, url, what string) error {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusAccepted, http.StatusOK, http.StatusNoContent:
|
||||
return nil
|
||||
case http.StatusNotFound:
|
||||
return Gone
|
||||
case http.StatusMethodNotAllowed:
|
||||
// The registry was started without deletion enabled. Said plainly, because the remedy is
|
||||
// a setting on the store's module and not anything about this artifact.
|
||||
return fmt.Errorf(
|
||||
"the artifact store refuses deletion: its server was started without it enabled "+
|
||||
"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
|
||||
"module is applied again (novox/hq ADR 0189). Asking about %s", what)
|
||||
default:
|
||||
return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
|
||||
}
|
||||
}
|
||||
|
||||
// split reads a recorded path into the repository, which endpoint names the thing, and the digest.
|
||||
//
|
||||
// Two shapes, which are the two the mesh records: `<repository>@sha256:<hex>` is a manifest, and
|
||||
// `<repository>/blobs/sha256:<hex>` is a blob.
|
||||
func split(path string) (repository, kind, digest string, err error) {
|
||||
if before, after, ok := strings.Cut(path, "@sha256:"); ok {
|
||||
return before, "manifests", "sha256:" + after, nil
|
||||
}
|
||||
if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok {
|
||||
return before, "blobs", "sha256:" + after, nil
|
||||
}
|
||||
return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path)
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||
//
|
||||
// A fake store records what it was asked to delete, so what is asserted is the mesh's decision
|
||||
// and the shape of the request — not the registry's behaviour, which is the registry's to test.
|
||||
|
||||
func fakeStore(t *testing.T, answer int) (Store, *[]string) {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
|
||||
// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
|
||||
// that does not have the thing does not have its blob either.
|
||||
if answer == http.StatusNotFound {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", "7")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodDelete {
|
||||
t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
|
||||
}
|
||||
asked = append(asked, r.URL.Path)
|
||||
w.WriteHeader(answer)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
|
||||
}
|
||||
|
||||
func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
|
||||
// The two shapes the mesh records: a manifest by digest, and a blob by digest. They are
|
||||
// different endpoints, and asking at the wrong one answers 404 — which this would then
|
||||
// record as collected, leaving the bytes on disk for ever while the record says otherwise.
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
ctx := context.Background()
|
||||
|
||||
image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123"
|
||||
archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456"
|
||||
if err := store.LetGo(ctx, image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.LetGo(ctx, archive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The archive's holder goes first, then the archive (novox/hq issue 253).
|
||||
_, holder := Holder("sha256:def456", 7)
|
||||
want := []string{
|
||||
"/v2/web/app/manifests/sha256:abc123",
|
||||
"/v2/web/config/manifests/" + holder,
|
||||
"/v2/web/config/blobs/sha256:def456",
|
||||
}
|
||||
if strings.Join(*asked, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("the store was asked %v; want %v", *asked, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) {
|
||||
// The outcome wanted, already true. Told apart from success only so the sweep can say which
|
||||
// happened; both are recorded, because retrying for ever is the thing to avoid.
|
||||
store, _ := fakeStore(t, http.StatusNotFound)
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("a store that does not hold it answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) {
|
||||
// The registry answers 405 when it was started without deletion enabled. The remedy is a
|
||||
// setting on the store's module, and saying "405" would send somebody to the wrong place.
|
||||
store, _ := fakeStore(t, http.StatusMethodNotAllowed)
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||
if err == nil {
|
||||
t.Fatal("a store that refuses deletion was read as success")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") {
|
||||
t.Fatalf("the refusal does not name the remedy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) {
|
||||
// The whole safety of the sweep is that it names only what the mesh recorded putting there.
|
||||
// A reference of another shape — a vendor's image, a package version — is refused rather
|
||||
// than composed into a delete somewhere that is not the mesh's store.
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
for _, reference := range []string{
|
||||
"docker.io/library/registry@sha256:abc123",
|
||||
"registry@sha256:abc123",
|
||||
"1.4.2",
|
||||
} {
|
||||
if err := store.LetGo(context.Background(), reference); err == nil {
|
||||
t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("the store was asked about %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the
|
||||
// record and not about the store (novox/hq issue 226).
|
||||
//
|
||||
// The sweep skips one and abandons itself for the other, so they cannot be the same error. The
|
||||
// first live run met a reference recorded with the store's old address, read the refusal as "the
|
||||
// store refuses everything", and collected none of the 1681 it had found.
|
||||
func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) {
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
for _, reference := range []string{
|
||||
"docker.io/library/registry@sha256:abc123",
|
||||
"127.0.0.1:5100/mesh-tools/build@sha256:abc123",
|
||||
"1.4.2",
|
||||
} {
|
||||
err := store.LetGo(context.Background(), reference)
|
||||
if !errors.Is(err, ErrNotOurs) {
|
||||
t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("the store was asked about %v", *asked)
|
||||
}
|
||||
}
|
||||
@@ -234,3 +234,13 @@ func admitsSubject(pattern, subject []string) bool {
|
||||
}
|
||||
return len(pattern) == len(subject)
|
||||
}
|
||||
|
||||
// The two packages name the runtime module separately — the broker's types stay free of the
|
||||
// catalogue's on purpose — so this is what holds them to one string. A rename that reached only one
|
||||
// side would compose a runtime principal for a module nobody assigns, silently, and leave the one
|
||||
// that is assigned with a module's own grants.
|
||||
func TestTheBrokerAndTheCatalogueAgreeOnTheRuntimeModule(t *testing.T) {
|
||||
if RuntimeModule != catalogue.RuntimeModule {
|
||||
t.Fatalf("the broker calls the runtime %q and the catalogue %q", RuntimeModule, catalogue.RuntimeModule)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A seat's cancelled set (novox/hq ADR 0219).
|
||||
//
|
||||
// **Cancelling an ask is deleting its message from the seat's work queue** — and that alone has a
|
||||
// race no ordering on one side closes: a holder may fetch the ask in the moment between the
|
||||
// controller reading the queue and deleting the message, and build what a person cancelled. So the
|
||||
// controller first writes the ask's id into the seat's cancelled set, and a holder looks its ask up
|
||||
// there after taking it and before building: listed, it terminates the ask and announces it failed
|
||||
// rather than starting it.
|
||||
//
|
||||
// **A key-value bucket, because that is the shape the bus already has for "a small set the
|
||||
// controller writes and many read cheaply"** (ADR 0201's state buckets): one direct read by key per
|
||||
// ask taken — no consumer, no subscription a holder must keep, nothing that grows a holder's grants
|
||||
// beyond one read subject. Kept beside the seat's own stream and worker and named like them, so the
|
||||
// three objects of one work queue read as one family; asserted by the controller with the queue,
|
||||
// and aged out with it — an id cancelled a week ago names an ask the queue no longer holds.
|
||||
|
||||
// CancelledSetName is the bucket holding a seat's cancelled asks.
|
||||
func CancelledSetName(seat string) string { return "SEAT_" + upperSnake(seat) + "_cancelled" }
|
||||
|
||||
// hasCancelledSet is whether a seat's queue can be cancelled from: the work queues the controller
|
||||
// asks, whose asks it alone shows and changes. A module's own seat's queue is that module's affair.
|
||||
func hasCancelledSet(seat string) bool {
|
||||
for _, s := range seatsTheControllerAsks {
|
||||
if s == seat {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsCancelledSet says a bucket is a seat's cancelled set rather than a module's state — the mesh's
|
||||
// own, and never one to report as state nothing declares.
|
||||
func IsCancelledSet(bucket string) bool {
|
||||
return strings.HasPrefix(bucket, "SEAT_") && strings.HasSuffix(bucket, "_cancelled")
|
||||
}
|
||||
|
||||
// cancelledSetAge is how long a cancelled id is kept: as long as the seat's queue keeps an ask.
|
||||
const cancelledSetAge = 7 * 24 * time.Hour
|
||||
|
||||
// A CancelledSetAsserter is what raising the cancelled sets needs of a connection.
|
||||
type CancelledSetAsserter interface {
|
||||
EnsureCancelledSet(seat string) error
|
||||
}
|
||||
|
||||
// RaiseCancelledSets asserts the cancelled set of every work queue the controller asks.
|
||||
func RaiseCancelledSets(a CancelledSetAsserter, seats []DeclaredSeat) error {
|
||||
for _, s := range seats {
|
||||
if len(s.Accepts) == 0 || !hasCancelledSet(s.Name) {
|
||||
continue
|
||||
}
|
||||
if err := a.EnsureCancelledSet(s.Name); err != nil {
|
||||
return fmt.Errorf("asserting the cancelled set of %s: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureCancelledSet creates a seat's cancelled set if absent and brings its options to match.
|
||||
// Direct reads on, which is how a holder looks an id up with one request.
|
||||
func (j *JetStream) EnsureCancelledSet(seat string) error {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: CancelledSetName(seat),
|
||||
Description: fmt.Sprintf("the asks of the %s seat cancelled by hand (novox/hq ADR 0219): written "+
|
||||
"by the controller before it deletes an ask from the queue, read by a holder on taking one, "+
|
||||
"so an ask fetched in that moment is ended rather than built", seat),
|
||||
History: 1,
|
||||
TTL: cancelledSetAge,
|
||||
MaxValueSize: 4 * 1024,
|
||||
MaxBytes: 4 * 1024 * 1024,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CancelledSetName(seat), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package broker
|
||||
|
||||
import "testing"
|
||||
|
||||
// A build agent reads its seat's cancelled set by key and nothing more, answers its verbs on its own
|
||||
// machine's subjects, and says whether it is paused under its own machine's name; the controller may
|
||||
// ask any machine's holder its verbs (novox/hq ADR 0219).
|
||||
func TestTheBuildQueueIsControlledWithTheGrantsItNeedsAndNoMore(t *testing.T) {
|
||||
seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"},
|
||||
Emits: []string{"started", "built", "log.*", "paused.*"},
|
||||
Serves: []string{"current", "kill", "pause", "resume"}}
|
||||
holder, err := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "build-agent",
|
||||
Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, holder.Publish, "$JS.API.DIRECT.GET.KV_SEAT_NODE_BUILD_AGENT_cancelled.$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
|
||||
hasNot(t, holder.Publish, "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
|
||||
has(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.ace")
|
||||
hasNot(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.*")
|
||||
has(t, holder.Publish, "mesh.seat.node-build-agent.event.log.*")
|
||||
has(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.ace")
|
||||
hasNot(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.g14")
|
||||
|
||||
// A module's own seat's queue is its own affair: no cancelled set, no grant for one.
|
||||
other, _ := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "telegram",
|
||||
Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, PasswordHash: "x"})
|
||||
hasNot(t, other.Publish, "$JS.API.DIRECT.GET.KV_SEAT_TELEGRAM_SENDER_cancelled.$KV.SEAT_TELEGRAM_SENDER_cancelled.>")
|
||||
|
||||
controller, _ := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
has(t, controller.Publish, "mesh.seat.node-build-agent.tool.>")
|
||||
|
||||
if CancelledSetName("node-build-agent") != "SEAT_NODE_BUILD_AGENT_cancelled" ||
|
||||
!IsCancelledSet("SEAT_NODE_BUILD_AGENT_cancelled") || IsCancelledSet("build-agent_cancelled") {
|
||||
t.Error("the cancelled set is not named as its seat's family")
|
||||
}
|
||||
}
|
||||
|
||||
// Only the work queues the controller asks get a cancelled set.
|
||||
func TestOnlyTheControllersQueuesHaveACancelledSet(t *testing.T) {
|
||||
var asserted []string
|
||||
a := asserterFunc(func(seat string) error { asserted = append(asserted, seat); return nil })
|
||||
if err := RaiseCancelledSets(a, []DeclaredSeat{
|
||||
{Name: "node-build-agent", Accepts: []string{"build"}},
|
||||
{Name: "telegram-sender", Accepts: []string{"send"}},
|
||||
{Name: "mesh-controller"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asserted) != 1 || asserted[0] != "node-build-agent" {
|
||||
t.Fatalf("asserted %v", asserted)
|
||||
}
|
||||
}
|
||||
|
||||
type asserterFunc func(string) error
|
||||
|
||||
func (f asserterFunc) EnsureCancelledSet(seat string) error { return f(seat) }
|
||||
|
||||
// A seat's cancelled set is never reported as state nothing declares.
|
||||
func TestACancelledSetIsNotUndeclaredState(t *testing.T) {
|
||||
undeclared, err := RaiseBuckets(fakeBuckets{names: []string{"SEAT_NODE_BUILD_AGENT_cancelled", "gone_state"}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(undeclared) != 1 || undeclared[0] != "gone_state" {
|
||||
t.Fatalf("undeclared %v", undeclared)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeBuckets struct{ names []string }
|
||||
|
||||
func (f fakeBuckets) EnsureBucket(Bucket) error { return nil }
|
||||
func (f fakeBuckets) BucketNames() ([]string, error) { return f.names, nil }
|
||||
@@ -0,0 +1,91 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A consumer that reacts to announcements, made on a stream that keeps a week of them, starts from now:
|
||||
// made from the start it replays every merge and build of that week (novox/hq issue 248). And a reset
|
||||
// re-makes a stuck one from now, its configuration otherwise kept, and refuses a work queue.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAConsumerMadeFromNow
|
||||
func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
const stream = "HISTORY_TEST"
|
||||
_ = js.js.DeleteStream(stream)
|
||||
if _, err := js.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{"history.>"}, Storage: nats.MemoryStorage}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.js.DeleteStream(stream) }()
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := js.js.Publish("history.merged", []byte(fmt.Sprint(i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
fresh := Consumer{Name: "fresh", Stream: stream, Filters: []string{"history.merged"}, Push: true,
|
||||
AckWaitSeconds: 30, MaxDeliver: 5, MaxAckPending: 1, FromNow: true}
|
||||
if err := js.EnsureConsumer(fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, _ := js.js.ConsumerInfo(stream, "fresh")
|
||||
if info.NumPending != 0 || info.Config.DeliverPolicy != nats.DeliverNewPolicy {
|
||||
t.Fatalf("a consumer made from now holds %d of the stream's past (policy %v)", info.NumPending, info.Config.DeliverPolicy)
|
||||
}
|
||||
_, _ = js.js.Publish("history.merged", []byte("new"))
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if info, _ = js.js.ConsumerInfo(stream, "fresh"); info.NumPending != 1 {
|
||||
t.Fatalf("a new announcement is not pending: %d", info.NumPending)
|
||||
}
|
||||
// Asserted again, it keeps where it is.
|
||||
if err := js.EnsureConsumer(fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One made from the start, as the server's default makes it, and stuck behind its history.
|
||||
old := fresh
|
||||
old.Name, old.FromNow = "old", false
|
||||
if err := js.EnsureConsumer(old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info, _ = js.js.ConsumerInfo(stream, "old"); info.NumPending != 6 {
|
||||
t.Fatalf("the default should replay all six: %d", info.NumPending)
|
||||
}
|
||||
before, after, err := js.ResetConsumer(stream, "old")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, _ = js.js.ConsumerInfo(stream, "old")
|
||||
if before.Pending != 6 || after.Pending != 0 || info.Config.MaxAckPending != 1 || info.Config.MaxDeliver != 5 ||
|
||||
info.Config.AckWait != 30*time.Second || info.Config.DeliverSubject == "" {
|
||||
t.Fatalf("before %+v after %+v config %+v", before, after, info.Config)
|
||||
}
|
||||
|
||||
// Never a work queue: what is pending there is work.
|
||||
const queue = "QUEUE_TEST"
|
||||
_ = js.js.DeleteStream(queue)
|
||||
if _, err := js.js.AddStream(&nats.StreamConfig{Name: queue, Subjects: []string{"queue.>"}, Retention: nats.WorkQueuePolicy, Storage: nats.MemoryStorage}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.js.DeleteStream(queue) }()
|
||||
if _, err := js.js.AddConsumer(queue, &nats.ConsumerConfig{Durable: "w", AckPolicy: nats.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := js.ResetConsumer(queue, "w"); err == nil {
|
||||
t.Fatal("a work queue's consumer was reset")
|
||||
}
|
||||
}
|
||||
+61
-16
@@ -47,7 +47,13 @@ type Consumer struct {
|
||||
// and a merge that came back rebuilt what it had just built, five times over on 2026-09-30.
|
||||
// With one outstanding, the server holds the rest, and the heartbeat is keeping the message.
|
||||
MaxAckPending int
|
||||
Why string
|
||||
// FromNow makes a consumer that does not exist yet start at the stream's end rather than its
|
||||
// beginning (novox/hq issue 248). For a consumer that reacts to announcements — a merge, a build's
|
||||
// outcome — on a stream that keeps a week of them: the server's default, everything the stream
|
||||
// holds, replays every merge and every build of that week as if it had just happened. A consumer
|
||||
// that exists keeps where it is, whatever this says; only its making is decided here.
|
||||
FromNow bool
|
||||
Why string
|
||||
}
|
||||
|
||||
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
|
||||
@@ -144,12 +150,56 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
}, true
|
||||
}
|
||||
|
||||
// HolderConsumerFor is the worker a seat's holder gets on that seat's work queue.
|
||||
// ModuleConsumer is one module's durable consumer, with the module and node it is for.
|
||||
type ModuleConsumer struct {
|
||||
Node, Module string
|
||||
Consumer Consumer
|
||||
}
|
||||
|
||||
// ConsumersOf is every module's durable consumer the composed users imply: each module user's, and
|
||||
// each module a runtime carries — a carried module with no account of its own is no user (novox/hq
|
||||
// issue 195), and its consumer is still the controller's to make, since the runtime reads it on the
|
||||
// module's behalf (ADR 0198). One per module and node, whichever of the two named it first.
|
||||
func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
var out []ModuleConsumer
|
||||
seen := map[string]bool{}
|
||||
add := func(p Principal) {
|
||||
c, needed := ConsumerFor(p)
|
||||
if !needed || seen[p.Node+"/"+p.Module] {
|
||||
return
|
||||
}
|
||||
seen[p.Node+"/"+p.Module] = true
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: c})
|
||||
}
|
||||
for _, p := range users {
|
||||
if p.Kind == KindModule {
|
||||
add(p)
|
||||
}
|
||||
}
|
||||
for _, p := range users {
|
||||
if p.Kind != KindNodeTools {
|
||||
continue
|
||||
}
|
||||
for _, d := range p.Carries {
|
||||
add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HolderConsumerFor is the worker a seat's holders share on that seat's work queue.
|
||||
//
|
||||
// **A queue group even though the seat guarantees one holder.** The seat is *authority* — who may
|
||||
// be the telegram sender — and the queue group is *delivery*. Tie delivery to the seat and the
|
||||
// day somebody allows two holders for throughput, every message is processed twice with nothing
|
||||
// reporting it. Kept separate, relaxing one changes nothing about the other.
|
||||
// **One worker for every holder, and each holder pulls one ask when it is idle** (novox/hq ADR
|
||||
// 0190). The seat is *authority* — who may be the telegram sender — and the worker is *delivery*,
|
||||
// kept separate so that relaxing one changes nothing about the other: a node-scoped seat has a
|
||||
// holder per machine, and all of them take from this one consumer, so the work is shared without
|
||||
// any holder knowing about the others. Pulled rather than pushed because a push consumer hands the
|
||||
// next ask to whichever subscriber the server picks, busy or not, and a pulled one is asked for by
|
||||
// a holder that has just become free. Which is also what ends the race issue 186 describes — asks
|
||||
// delivered behind the one being worked, expiring unacknowledged and dropped after the fifth
|
||||
// redelivery: nothing is delivered that nobody asked for. A long build keeps its own ask alive
|
||||
// (stillWorking); the ack wait is for a holder that died.
|
||||
func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) {
|
||||
if len(seat.Accepts) == 0 {
|
||||
return Consumer{}, false
|
||||
@@ -158,18 +208,13 @@ func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool)
|
||||
Name: "SEAT_" + upperSnake(seat.Name) + "_worker",
|
||||
Stream: seatStreamName(seat.Name),
|
||||
Filters: []string{"mesh.seat." + seat.Name + ".accept.>"},
|
||||
Queue: "holders",
|
||||
AckWaitSeconds: 60,
|
||||
MaxDeliver: 5,
|
||||
// **One in flight.** A holder works one ask at a time, so the server hands it one at a
|
||||
// time: with the default of many, every ask behind the one being worked was delivered,
|
||||
// left unacknowledged for the length of the work, redelivered after the ack wait, and
|
||||
// after the fifth time dropped — on 2026-10-01 twenty-six of forty-three builds asked in
|
||||
// two minutes were never built, and the queue read as empty (novox/hq issue 186).
|
||||
MaxAckPending: 1,
|
||||
Why: fmt.Sprintf("%s on %s holds %s; it acknowledges after the work is done, so a "+
|
||||
"crash mid-work redelivers rather than loses; one in flight, so a queue of asks is a "+
|
||||
"queue and not a race against the ack wait", module, node, seat.Name),
|
||||
// As many in flight as there are holders working, which pulling bounds by itself: a holder
|
||||
// fetches one and fetches again only after it acknowledged. The server's default stands.
|
||||
Why: fmt.Sprintf("%s on %s holds %s; every holder pulls one ask at a time from this worker "+
|
||||
"and acknowledges after the work is done, so a crash mid-work redelivers rather than "+
|
||||
"loses and an idle holder is the one that takes the next ask", module, node, seat.Name),
|
||||
}, true
|
||||
}
|
||||
|
||||
|
||||
@@ -88,15 +88,20 @@ func TestAModuleThatConsumesNothingGetsNoConsumer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The seat is authority and the queue group is delivery. Tie them together and the day somebody
|
||||
// allows two holders, every message is processed twice with nothing reporting it.
|
||||
func TestAHoldersWorkerUsesAQueueGroupAnyway(t *testing.T) {
|
||||
// The seat is authority and the worker is delivery (novox/hq ADR 0190): one worker per seat, shared
|
||||
// by every holder and pulled from, so a second holder takes the next ask rather than a copy of the
|
||||
// same one — which is what a queue group used to guard, and what pulling one durable gives outright.
|
||||
func TestAHoldersWorkerIsOneSharedByItsHolders(t *testing.T) {
|
||||
c, ok := HolderConsumerFor("one", "telegram", telegramSeat())
|
||||
if !ok {
|
||||
t.Fatal("the holder of a seat with inbound work got no worker")
|
||||
}
|
||||
if c.Queue == "" {
|
||||
t.Fatal("the worker is not in a queue group, so a second holder would double-process")
|
||||
two, _ := HolderConsumerFor("two", "telegram", telegramSeat())
|
||||
if c.Name != two.Name || c.Stream != two.Stream {
|
||||
t.Fatal("two holders got two workers, so each would process every ask")
|
||||
}
|
||||
if c.Push || c.Queue != "" {
|
||||
t.Fatal("the worker is pushed, so the server would hand an ask to a busy holder")
|
||||
}
|
||||
if c.Stream != "SEAT_TELEGRAM_SENDER" {
|
||||
t.Fatalf("the worker reads %q, not the seat's own stream", c.Stream)
|
||||
@@ -154,15 +159,23 @@ func TestANodesDeclarationConsumerIsWhatItsOwnGrantAllows(t *testing.T) {
|
||||
has(t, perms.Subscribe, c.Filters[0])
|
||||
}
|
||||
|
||||
// A holder works one ask at a time, so the server hands it one at a time (novox/hq issue 186):
|
||||
// asks queued behind the one being worked wait in the stream rather than being delivered,
|
||||
// left to expire and dropped after the fifth redelivery.
|
||||
func TestAHoldersWorkerTakesOneAskAtATime(t *testing.T) {
|
||||
c, found := HolderConsumerFor("anchor", "builder", DeclaredSeat{Name: "mesh-build-machine", Accepts: []string{"build"}})
|
||||
// Every holder of a seat shares one worker and pulls from it (novox/hq ADR 0190): no queue group
|
||||
// and no delivery subject, because a push consumer hands the next ask to whichever subscriber the
|
||||
// server picks, busy or not; and no cap of one in flight, because pulling bounds the asks in flight
|
||||
// by the holders that are free — which is what ended the race of issue 186, where asks delivered
|
||||
// behind the one being worked expired and were dropped.
|
||||
func TestAHoldersWorkerIsPulledByEveryHolder(t *testing.T) {
|
||||
c, found := HolderConsumerFor("anchor", "build-agent", DeclaredSeat{Name: "node-build-agent", Accepts: []string{"build"}})
|
||||
if !found {
|
||||
t.Fatal("a seat that accepts work has no worker")
|
||||
}
|
||||
if c.MaxAckPending != 1 {
|
||||
t.Fatalf("the worker may have %d asks in flight; one, so a queue is a queue", c.MaxAckPending)
|
||||
if c.Queue != "" || c.Push {
|
||||
t.Fatalf("the worker is pushed (queue %q, push %v); a holder pulls when it is free", c.Queue, c.Push)
|
||||
}
|
||||
if c.MaxAckPending != 0 {
|
||||
t.Fatalf("the worker caps asks in flight at %d; pulling bounds them by the holders working", c.MaxAckPending)
|
||||
}
|
||||
if c.Name != "SEAT_NODE_BUILD_AGENT_worker" || c.Stream != "SEAT_NODE_BUILD_AGENT" {
|
||||
t.Fatalf("the worker is %s on %s; one per seat, shared by its holders", c.Name, c.Stream)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The JetStream side of the controller: the one place the mesh's streams and consumers are
|
||||
@@ -84,6 +86,13 @@ func pinnedTo(path string) (*tls.Config, error) {
|
||||
return PinnedToFingerprint(want), nil
|
||||
}
|
||||
|
||||
// OnConn is the JetStream handle over a connection the caller already holds — the control plane's
|
||||
// link — for asserting what the bus holds without dialling a second time.
|
||||
func OnConn(conn *nats.Conn) *JetStream {
|
||||
js, _ := conn.JetStream()
|
||||
return &JetStream{conn: conn, js: js}
|
||||
}
|
||||
|
||||
// DialPinned is Dial with the server's certificate pinned by a fingerprint the caller already holds
|
||||
// — a module or a build machine that was handed one beside its credential, and has no file.
|
||||
func DialPinned(url, fingerprint string, opts ...nats.Option) (*JetStream, error) {
|
||||
@@ -214,6 +223,51 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
|
||||
switch have, err := j.js.ConsumerInfo(c.Stream, c.Name); {
|
||||
case err == nil:
|
||||
// **The controller owns the worker's shape, type included** (novox/hq issue 206). A holder
|
||||
// built for a pull worker cannot bind a push one — `cannot pull subscribe to push based
|
||||
// consumer` — and on 2026-10-03 the build machine rolled before the controller that would
|
||||
// have redefined its worker, restarted on that for an hour, and nothing could build the
|
||||
// controller that would have ended it. The server cannot change a consumer's type in place,
|
||||
// so one of the wrong type is re-made: on a work queue nothing is lost, because what was
|
||||
// acknowledged is gone from the stream and what was not is delivered again from the start.
|
||||
// On any other stream a re-made consumer would replay what this one acknowledged (issue
|
||||
// 156), so there it is said and left, and the person re-makes it knowing the cost.
|
||||
if havePush, wantPush := have.Config.DeliverSubject != "", want.DeliverSubject != ""; havePush != wantPush {
|
||||
shape := func(push bool) string {
|
||||
if push {
|
||||
return "push"
|
||||
}
|
||||
return "pull"
|
||||
}
|
||||
info, err := j.js.StreamInfo(c.Stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("asking about stream %s to re-make consumer %s: %w", c.Stream, c.Name, err)
|
||||
}
|
||||
if info.Config.Retention != nats.WorkQueuePolicy {
|
||||
// **A stream that keeps its history is re-made from now on, never from the start.**
|
||||
// Left for a hand, the hand re-makes it with the server's default — everything the
|
||||
// stream holds — which on 2026-10-03 replayed every build ask since 1 October and
|
||||
// re-registered nine modules from the past (novox/hq issue 207). What this consumer
|
||||
// had not yet acknowledged is lost with it, and said: on a history stream that is
|
||||
// the smaller cost, and the asks in flight are visible to whoever asked.
|
||||
j.note("consumer %s on %s changes from %s to %s delivery on a stream that keeps its history: "+
|
||||
"re-made to deliver from now on, so nothing this one acknowledged comes back (novox/hq issue "+
|
||||
"207); %d ask(s) it had not acknowledged are not carried over and must be asked again",
|
||||
c.Name, c.Stream, shape(havePush), shape(wantPush), have.NumPending+uint64(have.NumAckPending))
|
||||
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||
} else {
|
||||
j.note("consumer %s on %s changes from %s to %s delivery: re-made where it left off, nothing "+
|
||||
"acknowledged comes back and nothing pending is lost (novox/hq issue 206); a holder bound to "+
|
||||
"the old shape binds again", c.Name, c.Stream, shape(havePush), shape(wantPush))
|
||||
}
|
||||
if err := j.js.DeleteConsumer(c.Stream, c.Name); err != nil {
|
||||
return fmt.Errorf("re-making consumer %s on %s as %s: %w", c.Name, c.Stream, shape(wantPush), err)
|
||||
}
|
||||
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||
return fmt.Errorf("re-making consumer %s on %s as %s: %w", c.Name, c.Stream, shape(wantPush), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Where an existing consumer starts is its history, not something an assertion may move:
|
||||
// the server refuses a changed deliver policy outright. Carried across, so asserting twice
|
||||
// is the no-op a restart depends on.
|
||||
@@ -254,6 +308,9 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
}
|
||||
return nil
|
||||
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||
if c.FromNow {
|
||||
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||
}
|
||||
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
@@ -263,6 +320,51 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
}
|
||||
}
|
||||
|
||||
// ConsumerState is what a reset says about a consumer, before and after.
|
||||
type ConsumerState struct {
|
||||
DeliverPolicy string
|
||||
Delivered uint64
|
||||
AckFloor uint64
|
||||
Pending uint64
|
||||
AckPending int
|
||||
}
|
||||
|
||||
func stateOf(info *nats.ConsumerInfo) ConsumerState {
|
||||
policy, _ := info.Config.DeliverPolicy.MarshalJSON()
|
||||
return ConsumerState{DeliverPolicy: strings.Trim(string(policy), `"`), Delivered: info.Delivered.Stream,
|
||||
AckFloor: info.AckFloor.Stream, Pending: info.NumPending, AckPending: info.NumAckPending}
|
||||
}
|
||||
|
||||
// ResetConsumer re-makes a consumer to start from now, its configuration otherwise unchanged (novox/hq
|
||||
// issue 248): what it had not yet delivered or acknowledged is dropped, which is the point — on a stream
|
||||
// that keeps history, a consumer replaying a week of announcements does nothing anyone wants. Refused on
|
||||
// a work queue, where what is pending is work nobody else will do.
|
||||
func (j *JetStream) ResetConsumer(stream, name string) (before, after ConsumerState, err error) {
|
||||
info, err := j.js.StreamInfo(stream)
|
||||
if err != nil {
|
||||
return before, after, fmt.Errorf("asking about stream %s: %w", stream, err)
|
||||
}
|
||||
if info.Config.Retention == nats.WorkQueuePolicy {
|
||||
return before, after, fmt.Errorf("%s is a work queue: what its consumer has pending is work, and a reset would drop it", stream)
|
||||
}
|
||||
have, err := j.js.ConsumerInfo(stream, name)
|
||||
if err != nil {
|
||||
return before, after, fmt.Errorf("asking about consumer %s on %s: %w", name, stream, err)
|
||||
}
|
||||
before = stateOf(have)
|
||||
want := have.Config
|
||||
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||
want.OptStartSeq, want.OptStartTime = 0, nil
|
||||
if err := j.js.DeleteConsumer(stream, name); err != nil {
|
||||
return before, after, fmt.Errorf("removing consumer %s on %s: %w", name, stream, err)
|
||||
}
|
||||
made, err := j.js.AddConsumer(stream, &want)
|
||||
if err != nil {
|
||||
return before, after, fmt.Errorf("re-making consumer %s on %s — it is gone until the controller asserts it at its next start: %w", name, stream, err)
|
||||
}
|
||||
return before, stateOf(made), nil
|
||||
}
|
||||
|
||||
func retentionOf(r Retention) nats.RetentionPolicy {
|
||||
switch r {
|
||||
case RetentionWorkQueue:
|
||||
@@ -271,3 +373,50 @@ func retentionOf(r Retention) nats.RetentionPolicy {
|
||||
return nats.LimitsPolicy
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureBucket creates a module's bucket if it is absent and brings its options to match if it is
|
||||
// present (novox/hq ADR 0201).
|
||||
//
|
||||
// **An update, never a delete and recreate**, for the reason a stream is updated: recreating
|
||||
// discards what the bucket holds, and what a module's state holds is data. The mesh's caps are
|
||||
// asserted with the owner's options, so a bucket made by hand converges to them.
|
||||
func (j *JetStream) EnsureBucket(b Bucket) error {
|
||||
history := b.History
|
||||
if history == 0 {
|
||||
history = 1
|
||||
}
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: b.Bucket(),
|
||||
Description: b.Why(),
|
||||
History: uint8(history),
|
||||
TTL: time.Duration(b.TTLSeconds) * time.Second,
|
||||
MaxValueSize: StateMaxValueBytes,
|
||||
MaxBytes: StateMaxBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", b.Bucket(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BucketNames is every key-value bucket on the server, the mesh's and anybody else's.
|
||||
func (j *JetStream) BucketNames() ([]string, error) {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
lister := js.KeyValueStoreNames(ctx)
|
||||
var out []string
|
||||
for name := range lister.Name() {
|
||||
out = append(out, name)
|
||||
}
|
||||
return out, lister.Error()
|
||||
}
|
||||
|
||||
@@ -45,6 +45,11 @@ type Membership struct {
|
||||
// module that must tell the mesh from the world, the route proxy serving an internal name, reads
|
||||
// it here rather than keeping a definition of its own.
|
||||
Mesh []string `json:"mesh,omitempty"`
|
||||
// State is every bucket this module's code may reach, by the name it uses for each, and whether
|
||||
// it may write it (novox/hq ADR 0201): the runtime answers a bundle's state verbs from this list
|
||||
// and refuses, with the reason, what is not on it — the bus enforces only the union over every
|
||||
// module on the machine.
|
||||
State []StateIssued `json:"state,omitempty"`
|
||||
}
|
||||
|
||||
// Served is one address a tool is answered on.
|
||||
@@ -103,6 +108,7 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
|
||||
m.Seats = append(m.Seats, SeatServed{Seat: s.Name, Verb: verb, Subject: seatToolSubject(s, verb, node)})
|
||||
}
|
||||
}
|
||||
m.State = stateIssuedFor(d)
|
||||
if len(d.Invokes) > 0 {
|
||||
m.Reaches = map[string][]string{}
|
||||
for _, t := range d.Invokes {
|
||||
|
||||
@@ -73,3 +73,37 @@ func TestAnAccountMayReadItsOwnMembershipAndNoOthers(t *testing.T) {
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.anchor.postgres")
|
||||
hasNot(t, perms.Subscribe, "mesh.assignment.>")
|
||||
}
|
||||
|
||||
// The runtime arriving on a machine changes nothing about what each module is issued (to-be 38 WP2):
|
||||
// the memberships are composed as before and the runtime reads several of them. What the machine's
|
||||
// user list gains is one runtime principal, and loses nothing but the runtime module's own.
|
||||
func TestTheRuntimeArrivingLeavesEveryMembershipAsItWas(t *testing.T) {
|
||||
filter := Seat{Name: "node-packet-filter", Scope: "node", Serves: []string{"rules", "reload"}}
|
||||
three := []Declared{
|
||||
{Module: "nftables", Holds: []Seat{filter}, Serves: []string{"firewall_rules"}},
|
||||
{Module: "zsh", Serves: []string{"execute"}},
|
||||
{Module: "systemd", Serves: []string{"units"}},
|
||||
}
|
||||
before := Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": three}}
|
||||
after := Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{
|
||||
"anchor": append(append([]Declared{}, three...), Declared{Module: RuntimeModule}),
|
||||
}}
|
||||
for _, d := range three {
|
||||
was := MembershipFor("anchor", d, PlacementsOf(before, nil))
|
||||
is := MembershipFor("anchor", d, PlacementsOf(after, nil))
|
||||
if !reflect.DeepEqual(was, is) {
|
||||
t.Errorf("%s's membership changed when the runtime arrived:\n%+v\n%+v", d.Module, was, is)
|
||||
}
|
||||
}
|
||||
users, err := Users(after)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kinds := map[Kind]int{}
|
||||
for _, p := range users {
|
||||
kinds[p.Kind]++
|
||||
}
|
||||
if kinds[KindNodeTools] != 1 || kinds[KindModule] != 3 || kinds[KindNode] != 1 || kinds[KindController] != 1 {
|
||||
t.Errorf("the machine's users are %v; one runtime, the three modules, the host and the controller", kinds)
|
||||
}
|
||||
}
|
||||
|
||||
+206
-10
@@ -34,8 +34,20 @@ const (
|
||||
// authority is a list of tools and nothing else — not control, not declarations, not builds,
|
||||
// and no ability to answer anything, because a person asks.
|
||||
KindPerson Kind = "person"
|
||||
// KindNodeTools is a machine's tool runtime (novox/hq ADR 0175, to-be 38): one process per
|
||||
// node, on the host side, serving every assigned module's tools and every held seat's verbs.
|
||||
// Its authority is the union of what the modules it carries would each have had for their
|
||||
// tools — and nothing of what they consume, because tools are what it runs, not reactions.
|
||||
KindNodeTools Kind = "node-tools"
|
||||
)
|
||||
|
||||
// RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is
|
||||
// assigned, the mesh composes one runtime principal for the machine in place of that module's own,
|
||||
// and the per-module containers that served tools until then stop being the way tools reach a node.
|
||||
// Mirrored in the catalogue package, which the agreement test holds to the same string; one
|
||||
// constant, so a rename is one edit and the two packages cannot drift.
|
||||
const RuntimeModule = "node-tools"
|
||||
|
||||
// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it
|
||||
// emits (novox/hq ADR 0118, design 29 §5).
|
||||
type Seat struct {
|
||||
@@ -74,6 +86,13 @@ type Principal struct {
|
||||
// a namespace no such module owns. Every service started and the graph stayed empty.
|
||||
Watches []Seat
|
||||
|
||||
// Carries are the modules whose tools this principal serves, for a KindNodeTools principal
|
||||
// (novox/hq ADR 0175): every module assigned to its node, as each declares itself. Its
|
||||
// serving authority is the union of theirs — each module's own tool namespace and each held
|
||||
// seat's verbs on this node — derived from the same declarations the modules' own principals
|
||||
// are, so the runtime can serve nothing a module could not have served for itself.
|
||||
Carries []Declared
|
||||
|
||||
// Invokes are the tools this principal may call, as `<module>.<tool>`; a single `*` is every
|
||||
// tool. A person's whole authority (design 25 §7), and a module's only if its manifest says so
|
||||
// (novox/hq ADR 0152) — the console's does, and nothing else's.
|
||||
@@ -84,6 +103,12 @@ type Principal struct {
|
||||
// permission and nothing beside it.
|
||||
Invokes []string
|
||||
|
||||
// State is the local names of the state this principal's module keeps, and Reads the state of
|
||||
// others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's
|
||||
// instances and read by whoever declares it.
|
||||
State []string
|
||||
Reads []string
|
||||
|
||||
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
||||
// and never appears here: this file is written to a node's disk and read by a server, and a
|
||||
// secret that can be read from a configuration file is a secret with a wider blast radius
|
||||
@@ -91,10 +116,17 @@ type Principal struct {
|
||||
PasswordHash string
|
||||
}
|
||||
|
||||
// meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than
|
||||
// seatsTheControllerAsks are the roles the mesh's own flows submit work to. Named rather than
|
||||
// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a
|
||||
// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable.
|
||||
var meshSeatsTheControllerUses = []string{"mesh-build-machine"}
|
||||
// Both build roles while the handover runs (novox/hq ADR 0190): the controller asks whichever has a
|
||||
// holder, and the retired one has one until build-agent replaces the builder. The second entry
|
||||
// goes with the retired seat row.
|
||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||
|
||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
||||
var perMachineEvents = map[string]bool{"paused.*": true}
|
||||
|
||||
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
|
||||
// controller may answer an enrolment is derived from the same constant the user is named from.
|
||||
@@ -112,7 +144,10 @@ func (p Principal) Username() string {
|
||||
switch p.Kind {
|
||||
case KindPerson:
|
||||
return "person." + p.Module
|
||||
case KindModule:
|
||||
case KindModule, KindNodeTools:
|
||||
// The runtime is named exactly as the module it stands for would have been: the mesh
|
||||
// issues its credential through the same path a module's takes (`module issue`), and
|
||||
// that path knows the node and the module, not the kind.
|
||||
return p.Node + "." + p.Module
|
||||
case KindNode:
|
||||
return "node." + p.Node
|
||||
@@ -186,8 +221,14 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A
|
||||
// build is the one today: the controller asks, and reads the answer from the seat's event
|
||||
// like the catalogue does — which is why no holder needs to publish into anybody's inbox.
|
||||
for _, seat := range meshSeatsTheControllerUses {
|
||||
// A node-scoped seat's work subject carries no node (novox/hq ADR 0190): the ask goes to
|
||||
// the role, and whichever machine holding it is idle takes it.
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
||||
// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
|
||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
||||
}
|
||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||
// facts under the seat it holds, because a role's events belong to the role and keep their
|
||||
@@ -209,6 +250,8 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// which this package mirrors rather than reads, and a verb the seat does not declare is a
|
||||
// subject nothing publishes.
|
||||
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||
sub = append(sub, announcing(ControllerSeat)...)
|
||||
|
||||
// The two events it reacts to, and its ack subject on the stream they arrive from
|
||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||
@@ -244,6 +287,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
return Permissions{}, err
|
||||
}
|
||||
pub = append(pub, invoked...)
|
||||
// And may ask what answers (novox/hq ADR 0197): a question every service answers about
|
||||
// itself, its replies to the asker's own inbox.
|
||||
pub = append(pub, discovering()...)
|
||||
|
||||
case KindEnrolment:
|
||||
// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
|
||||
@@ -300,6 +346,15 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// away — no other principal may subscribe this namespace, and a caller's authority is
|
||||
// still granted per tool, by name, on the publish side.
|
||||
sub = append(sub, own+".tool.>")
|
||||
// It says what it serves (novox/hq ADR 0197): discovery for its own name and every seat it
|
||||
// holds a verb of, answered by the runtime that serves them.
|
||||
announced := []string{p.Module}
|
||||
for _, s := range p.Holds {
|
||||
if len(s.Serves) > 0 {
|
||||
announced = append(announced, s.Name)
|
||||
}
|
||||
}
|
||||
sub = append(sub, announcing(announced...)...)
|
||||
// Its own membership (ADR 0160): the one subject a runtime derives for itself, read
|
||||
// directly from the stream and followed live. Nothing else's.
|
||||
sub = append(sub, MembershipSubject(p.Node, p.Module))
|
||||
@@ -346,17 +401,37 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
|
||||
// 3. Seats it holds: full participation.
|
||||
for _, s := range p.Holds {
|
||||
// Taking work from the role's queue: the worker consumer it binds (asked about,
|
||||
// delivered on, acknowledged), each on the seat's own stream. The first machine to
|
||||
// take work over the new bus was refused the asking (2026-09-28).
|
||||
// Taking work from the role's queue: the worker consumer every holder shares (asked
|
||||
// about, pulled from, acknowledged), on the seat's own stream (novox/hq ADR 0190). A
|
||||
// holder pulls — asks the consumer for its next message, answered on its own inbox —
|
||||
// so what it needs is MSG.NEXT on that worker and nothing delivered to it. The first
|
||||
// machine to take work over the new bus was refused the asking (2026-09-28).
|
||||
worker := "SEAT_" + upperSnake(s.Name) + "_worker"
|
||||
stream := seatStreamName(s.Name)
|
||||
sub = append(sub, "_DELIVER."+worker, "_DELIVER."+worker+".>")
|
||||
pub = append(pub, "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>")
|
||||
pub = append(pub,
|
||||
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
|
||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
|
||||
"$JS.ACK."+stream+"."+worker+".>")
|
||||
// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
|
||||
// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
|
||||
// moment the controller cancelled it ends it instead of building it. Read, never written:
|
||||
// the set is the controller's, and only the work queues the controller asks — and so may
|
||||
// cancel from — have one.
|
||||
if hasCancelledSet(s.Name) {
|
||||
set := CancelledSetName(s.Name)
|
||||
pub = append(pub, "$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+".>")
|
||||
}
|
||||
for _, a := range s.Accepts {
|
||||
sub = append(sub, seatSubject(s, "accept", a))
|
||||
}
|
||||
for _, e := range s.Emits {
|
||||
// **A machine says its own state and no other's** (novox/hq ADR 0219): on a node-scoped
|
||||
// seat, an event about the holder itself carries the machine as its last token, and
|
||||
// each holder is granted its own machine's alone.
|
||||
if s.Scope == "node" && p.Node != "" && perMachineEvents[e] {
|
||||
pub = append(pub, seatSubject(s, "event", strings.TrimSuffix(e, "*")+p.Node))
|
||||
continue
|
||||
}
|
||||
pub = append(pub, seatSubject(s, "event", e))
|
||||
}
|
||||
for _, t := range s.Serves {
|
||||
@@ -375,6 +450,86 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
pub = append(pub, seatToolSubject(s, t, "*"))
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Its state, and the state of others it reads (novox/hq ADR 0201): every one read and
|
||||
// watched, its own written too.
|
||||
pub = append(pub, stateGrants(p.Module, p.State, p.Reads)...)
|
||||
|
||||
case KindNodeTools:
|
||||
// **One process serves what every module on the machine would have served for itself**
|
||||
// (novox/hq ADR 0175). Each carried module's whole tool namespace — the same grant that
|
||||
// module's own principal has, for the same reason: the tools a module serves are what its
|
||||
// code answers, and a list here would be a second copy of it. Each held seat's verbs on
|
||||
// this node, as the holder's own principal would be granted them.
|
||||
var serves []string
|
||||
for _, d := range p.Carries {
|
||||
if !safeSubject.MatchString(d.Module) {
|
||||
return Permissions{}, fmt.Errorf(
|
||||
"%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", d.Module)
|
||||
}
|
||||
serves = append(serves, d.Module)
|
||||
for _, s := range d.Holds {
|
||||
serves = append(serves, s.Name)
|
||||
}
|
||||
own := "mesh.mod." + d.Module
|
||||
sub = append(sub, own+".tool.>")
|
||||
// A tool that emits an event is the module's code and emits under the module's name
|
||||
// (ADR 0042); the runtime carrying that code may publish what the module declared it
|
||||
// emits, and nothing it did not.
|
||||
for _, e := range d.Emits {
|
||||
pub = append(pub, own+".event."+e)
|
||||
}
|
||||
for _, s := range d.Holds {
|
||||
for _, t := range s.Serves {
|
||||
sub = append(sub, seatToolSubject(s, t, p.Node))
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every assigned module's membership on this node (ADR 0160): one per module, read
|
||||
// directly from the stream and followed live. This node's and no other's — the one token
|
||||
// that varies is the module, so the pattern is the machine's own assignments.
|
||||
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
||||
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
||||
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
||||
// node, as the console already could — the runtime is the console's serving mode.
|
||||
invoked, err := invokedSubjects([]string{"*"})
|
||||
if err != nil {
|
||||
return Permissions{}, err
|
||||
}
|
||||
pub = append(pub, invoked...)
|
||||
// It says what it serves and may ask what answers (novox/hq ADR 0197): the runtime answers
|
||||
// discovery for each module and seat it carries, and the console it is asks the bus.
|
||||
// One service per runtime process, named for the runtime: the bus lets a principal answer each
|
||||
// request once, so the runtime announces everything it carries under its own name.
|
||||
sub = append(sub, announcing(append([]string{RuntimeModule}, serves...)...)...)
|
||||
pub = append(pub, discovering()...)
|
||||
// **And it consumes for the modules it carries** (novox/hq ADR 0198, which changes ADR 0175's
|
||||
// "it consumes nothing"): a module's long-running code is a bundle this runtime launches, and
|
||||
// the runtime is its bus — it reads the module's own durable consumer and acknowledges what
|
||||
// the module's code took. Exactly the grants the module's own principal has for that consumer,
|
||||
// on its name and no other's: asking about it, pulling from it, acknowledging it. The
|
||||
// consumer is still the controller's to make, from the module's own principal.
|
||||
for _, d := range p.Carries {
|
||||
own := Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches}
|
||||
if _, consumes := ConsumerFor(own); !consumes {
|
||||
continue
|
||||
}
|
||||
stream, durable := consumerStream(own), consumerDurable(own)
|
||||
pub = append(pub,
|
||||
"$JS.API.CONSUMER.INFO."+stream+"."+durable,
|
||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+durable,
|
||||
"$JS.ACK."+stream+"."+durable+".>")
|
||||
}
|
||||
// **And it keeps and reads state for the modules it carries** (novox/hq ADR 0201): the union
|
||||
// of what each may do with a bucket — an owner's write, a reader's read. That one module's code
|
||||
// does not write another's bucket through it is the runtime's to keep, from the membership
|
||||
// each assignment is issued, as it keeps each module's events under that module's own name.
|
||||
for _, d := range p.Carries {
|
||||
pub = append(pub, stateGrants(d.Module, stateNames(d.State), d.Reads)...)
|
||||
}
|
||||
sub = unique(sub)
|
||||
pub = unique(pub)
|
||||
}
|
||||
|
||||
if p.Kind == KindPerson {
|
||||
@@ -382,6 +537,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// consumer, because nothing is delivered to a person — they ask and are answered.
|
||||
sub = append(sub, p.inbox())
|
||||
}
|
||||
if p.Kind == KindNodeTools {
|
||||
// Its reply space, so the answers to what its tools call come back to it. No ack subject
|
||||
// for the same reason a person has none: nothing is delivered to it.
|
||||
sub = append(sub, p.inbox())
|
||||
}
|
||||
|
||||
if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController {
|
||||
// Its own reply space, and nothing wider.
|
||||
@@ -403,7 +563,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// A module answers what it was asked — a tool call reaches it on its own namespace, so the
|
||||
// authority is bounded by having been asked — and so does the controller. A node and a
|
||||
// person are never asked anything, and are granted nothing here.
|
||||
AllowResponses: p.Kind == KindModule || p.Kind == KindController,
|
||||
AllowResponses: p.Kind == KindModule || p.Kind == KindController || p.Kind == KindNodeTools,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -625,6 +785,20 @@ func ComposeAccounts(principals []Principal) (string, error) {
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// unique is a sorted list with each subject once. Two carried modules holding seats with the same
|
||||
// verb, or the runtime module itself carried beside the others, would otherwise write a grant twice
|
||||
// — harmless to the server, and noise in a file that is read as the mesh's authority model.
|
||||
func unique(values []string) []string {
|
||||
sort.Strings(values)
|
||||
out := values[:0]
|
||||
for i, v := range values {
|
||||
if i == 0 || v != values[i-1] {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func quoted(values []string) string {
|
||||
if len(values) == 0 {
|
||||
return ""
|
||||
@@ -673,3 +847,25 @@ func invokedSubjects(invokes []string) ([]string, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// announcing is what a principal that serves tools subscribes to answer the NATS services
|
||||
// protocol's discovery (novox/hq ADR 0197): the questions asked of every service, and those asked of
|
||||
// each name it serves — its own and no other's, so it cannot answer for a service it is not.
|
||||
func announcing(names ...string) []string {
|
||||
out := []string{"$SRV.PING", "$SRV.INFO", "$SRV.STATS"}
|
||||
for _, n := range names {
|
||||
if !safeSubject.MatchString(n) {
|
||||
continue
|
||||
}
|
||||
for _, verb := range []string{"PING", "INFO", "STATS"} {
|
||||
out = append(out, "$SRV."+verb+"."+n, "$SRV."+verb+"."+n+".>")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// discovering is what a principal publishes to ask what answers (novox/hq ADR 0197): the services
|
||||
// protocol's discovery requests, whose replies come to its own inbox.
|
||||
func discovering() []string {
|
||||
return []string{"$SRV.PING", "$SRV.PING.>", "$SRV.INFO", "$SRV.INFO.>"}
|
||||
}
|
||||
|
||||
@@ -233,7 +233,9 @@ func TestAPersonReachesNothingButTools(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
for _, p := range perms.Publish {
|
||||
if !strings.Contains(p, ".tool.") {
|
||||
// A tool call, or asking what answers (novox/hq ADR 0197) — a question every service
|
||||
// answers about itself, which claims nothing and controls nothing.
|
||||
if !strings.Contains(p, ".tool.") && !strings.HasPrefix(p, "$SRV.") {
|
||||
t.Errorf("a person may publish %q, which is not a tool call", p)
|
||||
}
|
||||
}
|
||||
@@ -371,3 +373,109 @@ func TestAModulePullsItsOwnConsumerAndNoOthers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The runtime's authority is the union of what the modules it carries would have been granted for
|
||||
// their tools (novox/hq ADR 0175): every carried module's tool namespace, every held seat's verbs
|
||||
// on this node, every module's membership on this node, and a call to anything. Nothing it
|
||||
// consumes, because it reacts to nothing.
|
||||
func TestTheRuntimeServesTheUnionAndConsumesForItsModules(t *testing.T) {
|
||||
filter := Seat{Name: "node-packet-filter", Scope: "node", Serves: []string{"rules", "reload"}}
|
||||
p := Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, Carries: []Declared{
|
||||
{Module: "nftables", Holds: []Seat{filter}, Serves: []string{"firewall_rules"}},
|
||||
{Module: "zsh", Emits: []string{"shell.opened"}, Consumes: []string{"shop.order.placed"}},
|
||||
{Module: RuntimeModule},
|
||||
}}
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"mesh.mod.nftables.tool.>", "mesh.mod.zsh.tool.>", "mesh.mod." + RuntimeModule + ".tool.>",
|
||||
"mesh.seat.node-packet-filter.tool.rules.anchor", "mesh.seat.node-packet-filter.tool.reload.anchor",
|
||||
"mesh.assignment.anchor.*",
|
||||
"_INBOX.anchor." + RuntimeModule + ".>",
|
||||
} {
|
||||
if !contains(perms.Subscribe, want) {
|
||||
t.Errorf("the runtime may not subscribe %s: %v", want, perms.Subscribe)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"mesh.mod.*.tool.>", "mesh.seat.*.tool.>",
|
||||
"$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.anchor.*",
|
||||
"mesh.mod.zsh.event.shell.opened",
|
||||
} {
|
||||
if !contains(perms.Publish, want) {
|
||||
t.Errorf("the runtime may not publish %s: %v", want, perms.Publish)
|
||||
}
|
||||
}
|
||||
// It reads the consumer of every carried module that consumes — that module's, by its name, as
|
||||
// the module's own principal could (novox/hq ADR 0198) — and of no module that consumes nothing.
|
||||
for _, want := range []string{
|
||||
"$JS.API.CONSUMER.INFO.EVENTS.anchor_zsh",
|
||||
"$JS.API.CONSUMER.MSG.NEXT.EVENTS.anchor_zsh",
|
||||
"$JS.ACK.EVENTS.anchor_zsh.>",
|
||||
} {
|
||||
if !contains(perms.Publish, want) {
|
||||
t.Errorf("the runtime may not read zsh's consumer: %s missing from %v", want, perms.Publish)
|
||||
}
|
||||
}
|
||||
for _, s := range perms.Publish {
|
||||
if (strings.HasPrefix(s, "$JS.ACK.") || strings.Contains(s, "CONSUMER")) && !strings.Contains(s, "anchor_zsh") {
|
||||
t.Errorf("the runtime was granted a consumer no carried module of it consumes on: %s", s)
|
||||
}
|
||||
}
|
||||
// It pulls; nothing is pushed to it, and it subscribes no event subject directly.
|
||||
for _, s := range perms.Subscribe {
|
||||
if strings.Contains(s, ".event.") || strings.HasPrefix(s, "_DELIVER.") {
|
||||
t.Errorf("the runtime was granted a delivery: %s", s)
|
||||
}
|
||||
}
|
||||
if !perms.AllowResponses {
|
||||
t.Error("the runtime answers what it is asked, and may not reply")
|
||||
}
|
||||
if _, needed := ConsumerFor(p); needed {
|
||||
t.Error("a consumer would be made for the runtime itself; it reads its modules' consumers, never one of its own")
|
||||
}
|
||||
// Each subject once in each list: the file is read as the mesh's authority model. One subject may
|
||||
// stand in both — the runtime answers discovery on `$SRV.INFO` and, as the console, asks it
|
||||
// (novox/hq ADR 0197) — because subscribing and publishing are two different grants.
|
||||
for _, list := range [][]string{perms.Subscribe, perms.Publish} {
|
||||
seen := map[string]bool{}
|
||||
for _, s := range list {
|
||||
if seen[s] {
|
||||
t.Errorf("%s is granted twice", s)
|
||||
}
|
||||
seen[s] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A node-scoped seat's work is shared (novox/hq ADR 0190): its holder on any machine subscribes the
|
||||
// seat's one work subject, with no node in it, so holders on several machines read one queue. The
|
||||
// node token belongs to a seat's tools, which are asked of one machine (design 33 §4), not to its work.
|
||||
func TestANodeSeatsWorkSubjectCarriesNoNode(t *testing.T) {
|
||||
seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Serves: []string{"status"}}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "anchor", Module: "build-agent", Holds: []Seat{seat}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, perms.Subscribe, "mesh.seat.node-build-agent.accept.build")
|
||||
hasNot(t, perms.Subscribe, "mesh.seat.node-build-agent.accept.build.anchor")
|
||||
// And its tools still carry the machine.
|
||||
has(t, perms.Subscribe, "mesh.seat.node-build-agent.tool.status.anchor")
|
||||
// The controller asks the role, not a machine.
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, controller.Publish, "mesh.seat.node-build-agent.accept.>")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A module's state on the bus (novox/hq ADR 0201, design 32 §4, design 25 §3).
|
||||
//
|
||||
// A module names the state it keeps (`state`) and the state of others it reads (`reads`), and each
|
||||
// is a key-value bucket: the server's own last-per-subject stream with direct reads, delete markers
|
||||
// and watches, which is the state relationship the mesh already uses for declarations, opened to
|
||||
// modules. The controller creates every bucket from the catalogue — from registration, like a
|
||||
// seat's stream, so a reader may watch before the owner runs anywhere — and no module can.
|
||||
//
|
||||
// Pure, like everything else in this package that decides what the bus holds; jetstream.go is the
|
||||
// part that asks a server.
|
||||
|
||||
// The mesh's caps on a bucket, the same for every module: a value is a piece of state, not a file,
|
||||
// and a bucket that grew without bound would be one module filling the bus's disk for everyone.
|
||||
const (
|
||||
StateMaxValueBytes = 256 * 1024
|
||||
StateMaxBytes = 64 * 1024 * 1024
|
||||
)
|
||||
|
||||
// A Bucket is one module's declared state as the bus holds it.
|
||||
type Bucket struct {
|
||||
Module string
|
||||
Name string
|
||||
// History is how many values a key keeps; zero is one.
|
||||
History int
|
||||
// TTLSeconds is how long a value lives; zero is until replaced or deleted.
|
||||
TTLSeconds int
|
||||
}
|
||||
|
||||
// BucketName is the bucket a module's state lives in: the module and the local name joined by an
|
||||
// underscore, which neither may contain, so two modules can never derive one bucket.
|
||||
func BucketName(module, name string) string { return module + "_" + name }
|
||||
|
||||
// Bucket is this bucket's name on the bus.
|
||||
func (b Bucket) Bucket() string { return BucketName(b.Module, b.Name) }
|
||||
|
||||
// Why is carried into the server's description of the bucket, so somebody reading the server's
|
||||
// own state finds whose it is and why it is kept.
|
||||
func (b Bucket) Why() string {
|
||||
return fmt.Sprintf("%s's state %q (novox/hq ADR 0201): its current value per key, written by %s, "+
|
||||
"read by whatever declares it reads it; kept when %s is unassigned, because it is data",
|
||||
b.Module, b.Name, b.Module, b.Module)
|
||||
}
|
||||
|
||||
// bucketOfRead is the bucket a read names, `<module>.<name>`, or false when it names none.
|
||||
func bucketOfRead(read string) (string, bool) {
|
||||
at := strings.LastIndex(read, ".")
|
||||
if at <= 0 || at == len(read)-1 {
|
||||
return "", false
|
||||
}
|
||||
module, name := read[:at], read[at+1:]
|
||||
if !safeSubject.MatchString(module) || !safeSubject.MatchString(name) {
|
||||
return "", false
|
||||
}
|
||||
return BucketName(module, name), true
|
||||
}
|
||||
|
||||
// stateGrants is what a principal publishes to reach the state its modules keep and read: for every
|
||||
// bucket, binding to it, reading a key directly, and an ordered consumer for listing and watching,
|
||||
// created and deleted on the bucket's own stream, with its flow control answered; for a bucket an
|
||||
// owner keeps, writing under the bucket's own subjects too.
|
||||
//
|
||||
// **Measured against a running server, 2026-10-04** (novox/hq research 024), and each one is there
|
||||
// because leaving it out failed: without STREAM.INFO nothing binds; without DIRECT.GET nothing is
|
||||
// read; without CONSUMER.CREATE no key is listed and nothing is watched; without CONSUMER.DELETE a
|
||||
// watch cannot be stopped and lingers on the server. A write outside these is refused by the server
|
||||
// — and reaches the writer as a timeout, not a refusal, which is why the runtime refuses first.
|
||||
func stateGrants(module string, keeps []string, reads []string) []string {
|
||||
var out []string
|
||||
read := func(bucket string) {
|
||||
stream := "KV_" + bucket
|
||||
out = append(out,
|
||||
"$JS.API.STREAM.INFO."+stream,
|
||||
"$JS.API.DIRECT.GET."+stream+".>",
|
||||
"$JS.API.CONSUMER.CREATE."+stream+".>",
|
||||
"$JS.API.CONSUMER.DELETE."+stream+".>",
|
||||
"$JS.FC."+stream+".>")
|
||||
}
|
||||
for _, name := range keeps {
|
||||
if !safeSubject.MatchString(name) {
|
||||
continue
|
||||
}
|
||||
bucket := BucketName(module, name)
|
||||
read(bucket)
|
||||
out = append(out, "$KV."+bucket+".>")
|
||||
}
|
||||
for _, r := range reads {
|
||||
if bucket, ok := bucketOfRead(r); ok {
|
||||
read(bucket)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// StateIssued is one bucket an assignment may reach, by the name its module uses for it: its own
|
||||
// state by the local name, another's as `<module>.<name>` (novox/hq ADR 0201).
|
||||
type StateIssued struct {
|
||||
Name string `json:"name"`
|
||||
Bucket string `json:"bucket"`
|
||||
Writes bool `json:"writes,omitempty"`
|
||||
}
|
||||
|
||||
// stateIssuedFor is every bucket a module's code may reach, as its membership lists them.
|
||||
func stateIssuedFor(d Declared) []StateIssued {
|
||||
var out []StateIssued
|
||||
for _, b := range d.State {
|
||||
out = append(out, StateIssued{Name: b.Name, Bucket: BucketName(d.Module, b.Name), Writes: true})
|
||||
}
|
||||
for _, r := range d.Reads {
|
||||
if bucket, ok := bucketOfRead(r); ok {
|
||||
out = append(out, StateIssued{Name: r, Bucket: bucket})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stateNames is the local names of a module's own buckets.
|
||||
func stateNames(buckets []Bucket) []string {
|
||||
out := make([]string, 0, len(buckets))
|
||||
for _, b := range buckets {
|
||||
out = append(out, b.Name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A BucketAsserter is the part of a JetStream connection bucket assertion needs.
|
||||
type BucketAsserter interface {
|
||||
// EnsureBucket creates the bucket if absent and brings its options to match if present, never
|
||||
// discarding what it holds.
|
||||
EnsureBucket(b Bucket) error
|
||||
// BucketNames is every key-value bucket on the server.
|
||||
BucketNames() ([]string, error)
|
||||
}
|
||||
|
||||
// RaiseBuckets asserts every declared bucket and answers the buckets on the server that nothing
|
||||
// declares any more.
|
||||
//
|
||||
// **Those are reported, never removed** (novox/hq ADR 0201, ADR 0030): what a module stored is
|
||||
// data, and a manifest edited, a module renamed or a catalogue entry dropped is an ordinary day's
|
||||
// work that must not take data with it. Removing one is a person's act.
|
||||
func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err error) {
|
||||
sorted := append([]Bucket(nil), buckets...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Bucket() < sorted[j].Bucket() })
|
||||
declared := map[string]bool{}
|
||||
for _, b := range sorted {
|
||||
if err := a.EnsureBucket(b); err != nil {
|
||||
return nil, fmt.Errorf("asserting %s's state %q: %w", b.Module, b.Name, err)
|
||||
}
|
||||
declared[b.Bucket()] = true
|
||||
}
|
||||
names, err := a.BucketNames()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
||||
}
|
||||
for _, n := range names {
|
||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
|
||||
if !declared[n] && !IsCancelledSet(n) {
|
||||
undeclared = append(undeclared, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(undeclared)
|
||||
return undeclared, nil
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// The grants measured against a running server (novox/hq research 024): an owner reads and writes
|
||||
// its bucket, a reader only reads, and neither reaches any other bucket.
|
||||
func TestAnOwnerWritesItsStateAndAReaderOnlyReads(t *testing.T) {
|
||||
owner, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "claude-code",
|
||||
State: []string{"servers"}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{
|
||||
"$KV.claude-code_servers.>",
|
||||
"$JS.API.STREAM.INFO.KV_claude-code_servers",
|
||||
"$JS.API.DIRECT.GET.KV_claude-code_servers.>",
|
||||
"$JS.API.CONSUMER.CREATE.KV_claude-code_servers.>",
|
||||
"$JS.API.CONSUMER.DELETE.KV_claude-code_servers.>",
|
||||
"$JS.FC.KV_claude-code_servers.>",
|
||||
} {
|
||||
has(t, owner.Publish, s)
|
||||
}
|
||||
hasNot(t, owner.Publish, "$KV.>")
|
||||
hasNot(t, owner.Publish, "$JS.API.>")
|
||||
|
||||
reader, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "console",
|
||||
Reads: []string{"claude-code.servers"}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, reader.Publish, "$JS.API.DIRECT.GET.KV_claude-code_servers.>")
|
||||
has(t, reader.Publish, "$JS.API.CONSUMER.CREATE.KV_claude-code_servers.>")
|
||||
hasNot(t, reader.Publish, "$KV.claude-code_servers.>")
|
||||
for _, s := range reader.Subscribe {
|
||||
if s == "$KV.claude-code_servers.>" {
|
||||
t.Fatalf("a reader subscribes the bucket's subjects directly: %v", reader.Subscribe)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One runtime carries every module on its machine, so its grant is the union: the owner's write
|
||||
// where an owner is carried, a read where only a reader is.
|
||||
func TestTheRuntimeKeepsAndReadsStateForItsModules(t *testing.T) {
|
||||
perms, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "one", Module: RuntimeModule,
|
||||
Carries: []Declared{
|
||||
{Module: "claude-code", State: []Bucket{{Module: "claude-code", Name: "servers"}},
|
||||
Reads: []string{"licence-manager.bindings"}},
|
||||
{Module: "audit"},
|
||||
}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, perms.Publish, "$KV.claude-code_servers.>")
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET.KV_licence-manager_bindings.>")
|
||||
hasNot(t, perms.Publish, "$KV.licence-manager_bindings.>")
|
||||
}
|
||||
|
||||
// A module with no state is granted nothing of any bucket — the composition of every module that
|
||||
// existed before this is unchanged.
|
||||
func TestAModuleWithNoStateReachesNoBucket(t *testing.T) {
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||
Emits: []string{"order.placed"}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range perms.Publish {
|
||||
if strings.HasPrefix(s, "$KV.") || strings.HasPrefix(s, "$JS.FC.") || strings.Contains(s, ".KV_") {
|
||||
t.Fatalf("granted %q without declaring state", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A read that names no bucket grants nothing rather than something that happens to parse.
|
||||
func TestAReadThatNamesNoBucketGrantsNothing(t *testing.T) {
|
||||
if got := stateGrants("a", nil, []string{"nodot", "x.", ".y", "a.b>"}); len(got) != 0 {
|
||||
t.Fatalf("granted %v for reads that name no bucket", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The membership lists every bucket the module's code may reach, by the name the module uses for
|
||||
// it, and whether it may write it — the list the runtime refuses from.
|
||||
func TestAMembershipListsTheStateItsModuleMayReach(t *testing.T) {
|
||||
m := MembershipFor("one", Declared{Module: "claude-code",
|
||||
State: []Bucket{{Module: "claude-code", Name: "servers"}},
|
||||
Reads: []string{"licence-manager.bindings"}}, Placements{})
|
||||
want := []StateIssued{
|
||||
{Name: "servers", Bucket: "claude-code_servers", Writes: true},
|
||||
{Name: "licence-manager.bindings", Bucket: "licence-manager_bindings"},
|
||||
}
|
||||
if !slices.Equal(m.State, want) {
|
||||
t.Fatalf("issued %+v, want %+v", m.State, want)
|
||||
}
|
||||
if none := MembershipFor("one", Declared{Module: "audit"}, Placements{}); none.State != nil {
|
||||
t.Fatalf("a module with no state was issued %+v", none.State)
|
||||
}
|
||||
}
|
||||
|
||||
type buckets struct {
|
||||
ensured []string
|
||||
on []string
|
||||
}
|
||||
|
||||
func (b *buckets) EnsureBucket(x Bucket) error {
|
||||
b.ensured = append(b.ensured, x.Bucket())
|
||||
return nil
|
||||
}
|
||||
func (b *buckets) BucketNames() ([]string, error) { return b.on, nil }
|
||||
|
||||
// Every declared bucket is asserted; one on the server that nothing declares is said, not removed.
|
||||
func TestRaisingStateReportsWhatNothingDeclares(t *testing.T) {
|
||||
b := &buckets{on: []string{"claude-code_servers", "gone_old", "ours_by_hand"}}
|
||||
undeclared, err := RaiseBuckets(b, []Bucket{{Module: "claude-code", Name: "servers"}, {Module: "a", Name: "b"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(b.ensured, []string{"a_b", "claude-code_servers"}) {
|
||||
t.Fatalf("asserted %v", b.ensured)
|
||||
}
|
||||
if !slices.Equal(undeclared, []string{"gone_old", "ours_by_hand"}) {
|
||||
t.Fatalf("reported %v", undeclared)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real server: a bucket is created with the owner's options and the mesh's caps,
|
||||
// asserting it again changes nothing and keeps what it holds, and a changed option is brought to
|
||||
// match in place.
|
||||
func TestABucketIsAssertedInPlace(t *testing.T) {
|
||||
js := aLiveBus(t)
|
||||
b := Bucket{Module: "statetest", Name: "servers"}
|
||||
if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
|
||||
t.Fatalf("a real server refused a module's bucket: %v", err)
|
||||
}
|
||||
kv, err := js.Context().KeyValue(b.Bucket())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put("all.one", []byte(`{"kept":true}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.History = 3
|
||||
if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
|
||||
t.Fatalf("asserting the bucket again failed, so a restart would: %v", err)
|
||||
}
|
||||
got, err := kv.Get("all.one")
|
||||
if err != nil || string(got.Value()) != `{"kept":true}` {
|
||||
t.Fatalf("asserting again lost what the bucket held: %v %v", got, err)
|
||||
}
|
||||
status, err := kv.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status.History() != 3 {
|
||||
t.Fatalf("history is %d, the owner declared 3", status.History())
|
||||
}
|
||||
if s, ok := status.(*nats.KeyValueBucketStatus); ok {
|
||||
if c := s.StreamInfo().Config; c.MaxMsgSize != StateMaxValueBytes || c.MaxBytes != StateMaxBytes {
|
||||
t.Fatalf("the mesh's caps are not on the bucket: value %d, bucket %d", c.MaxMsgSize, c.MaxBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real server: the handle over a connection the control plane already holds asserts a
|
||||
// bucket as Dial's does — what a push uses, so a module registered since the last start has its
|
||||
// bucket before its membership names it.
|
||||
func TestABucketIsAssertedOverAHeldConnection(t *testing.T) {
|
||||
js := aLiveBus(t)
|
||||
held := OnConn(js.Conn())
|
||||
if _, err := RaiseBuckets(held, []Bucket{{Module: "statetest", Name: "held"}}); err != nil {
|
||||
t.Fatalf("asserting over a held connection failed: %v", err)
|
||||
}
|
||||
if _, err := js.Context().KeyValue("statetest_held"); err != nil {
|
||||
t.Fatalf("the bucket is not there: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -217,10 +217,15 @@ var ControllerFollows = []string{
|
||||
// A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a
|
||||
// message on the control branch. Same three audiences, one publish: whoever asked, this, and the
|
||||
// catalogue.
|
||||
seatEventSubject("mesh-build-machine", "built"),
|
||||
seatEventSubject("node-build-agent", "built"),
|
||||
// The forge's merges: what moved a source, so the mesh builds what that source produces
|
||||
// without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name.
|
||||
moduleEventSubject("gitea", "pull.merged"),
|
||||
// The retired build role's outcome too, while the handover runs (novox/hq ADR 0190): the one
|
||||
// build machine keeps answering on its seat until build-agent replaces it, and the outcome that
|
||||
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
|
||||
// with the retired seat row.
|
||||
seatEventSubject("mesh-build-machine", "built"),
|
||||
}
|
||||
|
||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||
@@ -265,6 +270,7 @@ func MeshConsumers() []Consumer {
|
||||
// announcement handed over behind it must wait on the server, not time out on the
|
||||
// client and come back to be acted on again.
|
||||
MaxAckPending: 1,
|
||||
FromNow: true,
|
||||
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
|
||||
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
||||
"become actionable",
|
||||
|
||||
+6
-6
@@ -24,8 +24,8 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused"] }
|
||||
subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||
@@ -37,18 +37,18 @@ accounts {
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.declare"] }
|
||||
} }
|
||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||
subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_DELIVER.SEAT_TELEGRAM_SENDER_worker.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.audit"] }
|
||||
subscribe: { allow: ["_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
|
||||
subscribe: { allow: ["_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
]
|
||||
|
||||
@@ -33,6 +33,14 @@ type Declared struct {
|
||||
Watches []Seat
|
||||
// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
|
||||
Invokes []string
|
||||
// State is the state it keeps, each a bucket its instances write (novox/hq ADR 0201).
|
||||
State []Bucket
|
||||
// Reads are other modules' state it reads, each `<module>.<name>` (novox/hq ADR 0201).
|
||||
Reads []string
|
||||
// NoAccount says the module declares no own secret named broker, so no account could ever be
|
||||
// delivered to it and nothing can connect as it (novox/hq issue 195). Said in the negative so a
|
||||
// record that does not say is composed as it always was.
|
||||
NoAccount bool
|
||||
}
|
||||
|
||||
// Records is what composing a user list needs to know about the mesh, and nothing more.
|
||||
@@ -62,11 +70,41 @@ func Users(r Records) ([]Principal, error) {
|
||||
|
||||
for _, node := range sortedCopy(r.Nodes) {
|
||||
out = append(out, Principal{Kind: KindNode, Node: node})
|
||||
// **Where the runtime is assigned, the machine gets one runtime principal in place of the
|
||||
// runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the
|
||||
// node: its serving grants are the union of theirs. Every other module keeps its own
|
||||
// principal — a module still serving tools from its own container holds its own
|
||||
// credential until it moves, and the two serve side by side in the meantime.
|
||||
runtimeHere := false
|
||||
for _, d := range r.Assigned[node] {
|
||||
if d.Module == RuntimeModule {
|
||||
runtimeHere = true
|
||||
}
|
||||
}
|
||||
for _, d := range r.Assigned[node] {
|
||||
if runtimeHere && d.Module == RuntimeModule {
|
||||
continue
|
||||
}
|
||||
// **A module with nowhere to read an account is no user** (novox/hq issue 195). `module
|
||||
// issue` refuses it one (issue 078: an account nothing reads is an orphan), so its user
|
||||
// could only ever be left out of the file for want of a password — and every such module
|
||||
// was named, on every status and plan, as a credential the mesh had not minted. Where the
|
||||
// runtime is, it speaks for the module; where it is not, the module cannot speak at all,
|
||||
// and a user would not change that.
|
||||
if d.NoAccount {
|
||||
continue
|
||||
}
|
||||
out = append(out, Principal{
|
||||
Kind: KindModule, Node: node, Module: d.Module,
|
||||
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
|
||||
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
||||
State: stateNames(d.State), Reads: d.Reads,
|
||||
})
|
||||
}
|
||||
if runtimeHere {
|
||||
out = append(out, Principal{
|
||||
Kind: KindNodeTools, Node: node, Module: RuntimeModule,
|
||||
Carries: append([]Declared(nil), r.Assigned[node]...),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,3 +245,108 @@ func TestAUserListIsComposedBeforeAnythingMovesOntoTheBus(t *testing.T) {
|
||||
t.Errorf("the composed list does not contain the machine running the bus")
|
||||
}
|
||||
}
|
||||
|
||||
// Where the runtime module is assigned, the machine gets one runtime principal in place of the
|
||||
// runtime module's own (novox/hq ADR 0175, to-be 38). Every other module keeps its own: a module
|
||||
// still serving tools from its own container holds its own credential until it moves.
|
||||
func TestTheRuntimeModuleBecomesTheMachinesRuntimePrincipal(t *testing.T) {
|
||||
r := someRecords()
|
||||
r.Assigned["one"] = append(r.Assigned["one"], Declared{Module: RuntimeModule})
|
||||
users, err := Users(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var runtime *Principal
|
||||
for i := range users {
|
||||
p := &users[i]
|
||||
if p.Node == "one" && p.Module == RuntimeModule {
|
||||
if p.Kind == KindModule {
|
||||
t.Fatalf("%s on one was composed as an ordinary module beside the runtime", RuntimeModule)
|
||||
}
|
||||
runtime = p
|
||||
}
|
||||
}
|
||||
if runtime == nil || runtime.Kind != KindNodeTools {
|
||||
t.Fatalf("one runs %s and got no runtime principal: %v", RuntimeModule, namesOf(t, r))
|
||||
}
|
||||
if runtime.Username() != "one."+RuntimeModule {
|
||||
t.Errorf("the runtime is named %q; `module issue` names it as the module it stands for", runtime.Username())
|
||||
}
|
||||
carried := map[string]bool{}
|
||||
for _, d := range runtime.Carries {
|
||||
carried[d.Module] = true
|
||||
}
|
||||
if !carried["telegram"] || !carried[RuntimeModule] {
|
||||
t.Errorf("the runtime carries %v; it carries every module on its node", carried)
|
||||
}
|
||||
// And the other node, where the runtime is not assigned, is exactly as before.
|
||||
for _, p := range users {
|
||||
if p.Node == "two" && p.Kind == KindNodeTools {
|
||||
t.Fatal("two runs no runtime and was given a runtime principal")
|
||||
}
|
||||
}
|
||||
// A module serving its own tools beside the runtime keeps its own principal.
|
||||
found := false
|
||||
for _, p := range users {
|
||||
if p.Kind == KindModule && p.Node == "one" && p.Module == "telegram" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("telegram lost its own principal when the runtime arrived on its node")
|
||||
}
|
||||
}
|
||||
|
||||
// A module that declares nowhere to read an account is no user: it could never be issued one, so it
|
||||
// was only ever named as a credential the mesh had not minted (novox/hq issue 195). Where the runtime
|
||||
// is, the runtime still carries it — its grants are the runtime's.
|
||||
func TestAModuleThatCannotReadAnAccountIsNoUser(t *testing.T) {
|
||||
r := someRecords()
|
||||
r.Assigned["one"] = append(r.Assigned["one"],
|
||||
Declared{Module: "packet-filter", NoAccount: true, Emits: []string{"rule.changed"}},
|
||||
Declared{Module: RuntimeModule})
|
||||
users, err := Users(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Username() == "one.packet-filter" {
|
||||
t.Fatalf("packet-filter declares no broker secret and was composed as a user: %v", namesOf(t, r))
|
||||
}
|
||||
if u.Kind == KindNodeTools {
|
||||
carried := false
|
||||
for _, d := range u.Carries {
|
||||
carried = carried || d.Module == "packet-filter"
|
||||
}
|
||||
if !carried {
|
||||
t.Error("the runtime stopped carrying a module that has no account of its own")
|
||||
}
|
||||
}
|
||||
}
|
||||
if names := strings.Join(namesOf(t, r), ","); !strings.Contains(names, "one.telegram") {
|
||||
t.Errorf("a module that does read an account lost its user: %s", names)
|
||||
}
|
||||
}
|
||||
|
||||
// A carried module with no account still has its consumer made: the runtime reads it on the module's
|
||||
// behalf (ADR 0198), and the consumer was derived from the module's own user until issue 195 took
|
||||
// that user away.
|
||||
func TestACarriedModuleWithNoAccountStillHasItsConsumer(t *testing.T) {
|
||||
r := someRecords()
|
||||
r.Assigned["one"] = append(r.Assigned["one"],
|
||||
Declared{Module: "listener", NoAccount: true, Consumes: []string{"shop.order.placed"}},
|
||||
Declared{Module: RuntimeModule})
|
||||
r.Assigned["two"] = append(r.Assigned["two"],
|
||||
Declared{Module: "auditor", Consumes: []string{"shop.order.placed"}})
|
||||
users, err := Users(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]int{}
|
||||
for _, c := range ConsumersOf(users) {
|
||||
got[c.Node+"/"+c.Module]++
|
||||
}
|
||||
if got["one/listener"] != 1 || got["two/auditor"] != 1 || len(got) != 2 {
|
||||
t.Fatalf("consumers: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A seat's worker that changed from push to pull delivery strands a holder built for the new shape
|
||||
// (novox/hq issue 206): the server refuses a pull subscription on a push consumer, and the controller
|
||||
// that would redefine it was the build that nobody could take. The controller owns the worker's
|
||||
// shape, type included: on a work queue it re-makes one of the wrong type, losing nothing, and a
|
||||
// pull subscription then binds and takes what was pending.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAWorker
|
||||
func TestAWorkerOfTheWrongTypeIsRemadeOnAWorkQueueAndAPullThenBinds(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
const stream, worker, filter = "SEAT_T_SHELF", "SEAT_T_SHELF_worker", "mesh.seat.t-shelf.accept.>"
|
||||
_ = js.js.DeleteStream(stream)
|
||||
if _, err := js.js.AddStream(&nats.StreamConfig{
|
||||
Name: stream, Subjects: []string{filter}, Retention: nats.WorkQueuePolicy, Storage: nats.MemoryStorage,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.js.DeleteStream(stream) }()
|
||||
|
||||
// The worker as the previous controller defined it: push, in a queue group.
|
||||
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
|
||||
Durable: worker, AckPolicy: nats.AckExplicitPolicy, AckWait: 60 * time.Second, MaxDeliver: 5,
|
||||
FilterSubject: filter, DeliverSubject: "_DELIVER." + worker, DeliverGroup: "holders",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, body := range []string{"one", "two", "three"} {
|
||||
if _, err := js.js.Publish("mesh.seat.t-shelf.accept.build", []byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The old holder took and acknowledged the first ask, then went away.
|
||||
old, err := js.js.QueueSubscribeSync(filter, "holders", nats.Bind(stream, worker))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := old.NextMsg(twoSeconds)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(m.Data) != "one" {
|
||||
t.Fatalf("the first ask is %q", m.Data)
|
||||
}
|
||||
if err := m.AckSync(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := old.Unsubscribe(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The new controller asserts the worker as the mesh derives it now: pull.
|
||||
if err := js.EnsureConsumer(Consumer{
|
||||
Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60, MaxDeliver: 5,
|
||||
Why: "the test's worker",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have, err := js.js.ConsumerInfo(stream, worker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have.Config.DeliverSubject != "" || have.Config.DeliverGroup != "" {
|
||||
t.Fatalf("the worker is still push: %+v", have.Config)
|
||||
}
|
||||
|
||||
// A holder built for the new shape binds, and takes exactly what the old one left.
|
||||
sub, err := js.js.PullSubscribe(filter, worker, nats.Bind(stream, worker), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatalf("a pull subscription does not bind the re-made worker: %v", err)
|
||||
}
|
||||
got, err := sub.Fetch(3, nats.MaxWait(twoSeconds))
|
||||
if err != nil && len(got) == 0 {
|
||||
t.Fatalf("nothing pending was delivered: %v", err)
|
||||
}
|
||||
var bodies []string
|
||||
for _, g := range got {
|
||||
bodies = append(bodies, string(g.Data))
|
||||
_ = g.Ack()
|
||||
}
|
||||
if len(bodies) != 2 || bodies[0] != "two" || bodies[1] != "three" {
|
||||
t.Fatalf("the pending asks after the acknowledged one, in order: %v", bodies)
|
||||
}
|
||||
|
||||
// Asserted again, the pull worker is the no-op a restart depends on.
|
||||
if err := js.EnsureConsumer(Consumer{
|
||||
Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60, MaxDeliver: 5,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// On a stream that keeps its history, a worker of the wrong type is re-made to deliver from now on:
|
||||
// re-making it from the start would replay what it acknowledged (novox/hq issue 156), and leaving it
|
||||
// for a hand re-made it exactly that way on 2026-10-03 (issue 207).
|
||||
func TestAWorkerOfTheWrongTypeOnAHistoryStreamIsRemadeFromNowOn(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
const stream, worker, filter = "EVENTS_T", "EVENTS_T_reader", "mesh.t.event.>"
|
||||
_ = js.js.DeleteStream(stream)
|
||||
if _, err := js.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{filter}, Storage: nats.MemoryStorage}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.js.DeleteStream(stream) }()
|
||||
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
|
||||
Durable: worker, AckPolicy: nats.AckExplicitPolicy, AckWait: 60 * time.Second,
|
||||
FilterSubject: filter, DeliverSubject: "_DELIVER." + worker,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// History the old consumer would have acknowledged long ago, and must not come back.
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := js.js.Publish("mesh.t.event.old", []byte("old")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := js.EnsureConsumer(Consumer{Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have, err := js.js.ConsumerInfo(stream, worker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have.Config.DeliverSubject != "" {
|
||||
t.Fatal("a history stream's consumer of the wrong type was left as it was")
|
||||
}
|
||||
if have.Config.DeliverPolicy != nats.DeliverNewPolicy || have.NumPending != 0 {
|
||||
t.Fatalf("re-made consumer delivers %v with %d pending; it must deliver from now on with nothing of the past", have.Config.DeliverPolicy, have.NumPending)
|
||||
}
|
||||
// And what arrives from now on is delivered.
|
||||
if _, err := js.js.Publish("mesh.t.event.new", []byte("new")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sub, err := js.js.PullSubscribe(filter, worker, nats.Bind(stream, worker))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
|
||||
if err != nil || len(got) != 1 || string(got[0].Data) != "new" {
|
||||
t.Fatalf("the re-made consumer delivered %v, %v; want the one new message", got, err)
|
||||
}
|
||||
}
|
||||
+180
-3
@@ -215,7 +215,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
||||
for _, a := range artifacts {
|
||||
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, seatBases, say)
|
||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, say)
|
||||
if err != nil {
|
||||
say("artifact", "%s FAILED: %v", a.Name, err)
|
||||
return Result{}, err
|
||||
@@ -443,7 +443,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
|
||||
|
||||
func one(ctx context.Context, run Runner, publish Publisher,
|
||||
module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
|
||||
held map[string]string, npmrc string, seats map[string]string,
|
||||
held map[string]string, npmrc string, registry Npmrc, seats map[string]string,
|
||||
say func(step, format string, args ...any)) (catalogue.Built, error) {
|
||||
|
||||
switch a.Kind {
|
||||
@@ -582,11 +582,28 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
"holds no copy of it. Build %s first",
|
||||
module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
|
||||
}
|
||||
// The module's own packages first, where the compiler and the bundler resolve them from
|
||||
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
|
||||
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
|
||||
}
|
||||
say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base)
|
||||
compiled, err := compile(ctx, run, tree, chain, base, a)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
// **Every entrypoint the runtime may serve is executable** (novox/hq ADR 0193). The runtime
|
||||
// knows no language; for one that runs through an interpreter the build writes the launcher.
|
||||
launchers, err := writeLaunchers(compiled, chain, a)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: writing %s's launchers failed: %w", module, a.Name, err)
|
||||
}
|
||||
if chain.Bundler != "" {
|
||||
say("bundle", "bundling each entrypoint into one file")
|
||||
if compiled, err = bundled(ctx, run, tree, chain, base, a, launchers); err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: bundling %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
}
|
||||
say("bundle", "compiled, packing")
|
||||
body, err := pack(compiled)
|
||||
if err != nil {
|
||||
@@ -598,7 +615,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
if err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest, Launchers: launchers}, nil
|
||||
|
||||
case catalogue.ArtifactPackage:
|
||||
// Built and published on a public base, to the mesh's package registry, by version
|
||||
@@ -744,6 +761,9 @@ func Command(ctx context.Context, dir, name string, args ...string) (string, err
|
||||
tell("run", "$ (%s) %s %s", short(filepath.Base(dir)), name, strings.Join(args, " "))
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = dir
|
||||
// **Ended whole when the build is** (novox/hq ADR 0219): its own process group, killed as one
|
||||
// when the context ends, so a build killed by hand leaves no `git` or `docker` child running.
|
||||
inItsOwnGroup(cmd)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
tell("run", "! %s %s failed after %s", name, args[0], since(started))
|
||||
@@ -968,6 +988,26 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
|
||||
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if chain.Dependencies != "" && chain.Bundler == "" {
|
||||
// **What the bundle runs with, from the image it was compiled in** (Toolchain.Dependencies).
|
||||
// A second run in the same image rather than a shell wrapped around the compiler: the
|
||||
// compile line stays a plain command a reader can run by hand, and the copy is one more
|
||||
// plain command beside it. Refused by name when the image carries no such directory — an
|
||||
// older toolchain image — because a bundle packed without its dependencies starts nowhere
|
||||
// and says so three layers away from here.
|
||||
copying := []string{
|
||||
"run", "--rm",
|
||||
"--volume", tree + ":" + within,
|
||||
"--workdir", within,
|
||||
base,
|
||||
"sh", "-c",
|
||||
`test -d "$1" || { echo "the toolchain image carries no $1: it predates the mesh shipping a bundle's dependencies, rebuild $2 first" >&2; exit 1; }; cp -a "$1/." "$3/"`,
|
||||
"dependencies", chain.Dependencies, chain.Base, out,
|
||||
}
|
||||
if _, err := run(ctx, tree, "docker", copying...); err != nil {
|
||||
return "", fmt.Errorf("copying the %s dependencies a bundle runs with: %w", chain.Language, err)
|
||||
}
|
||||
}
|
||||
return filepath.Join(tree, out), nil
|
||||
}
|
||||
|
||||
@@ -1145,6 +1185,9 @@ func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
|
||||
// binaryName is what a compiled bundle's executable is called: what the artifact says, or the name of
|
||||
// the package it is built from, which is what a compiler would have chosen anyway.
|
||||
func binaryName(a catalogue.Artifact) string {
|
||||
if name := catalogue.BinaryOf(a); name != "" {
|
||||
return name
|
||||
}
|
||||
if name := strings.TrimSpace(a.Binary); name != "" {
|
||||
return name
|
||||
}
|
||||
@@ -1153,3 +1196,137 @@ func binaryName(a catalogue.Artifact) string {
|
||||
}
|
||||
return a.Name
|
||||
}
|
||||
|
||||
// launcherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is
|
||||
// started as index.serve.mjs (novox/hq ADR 0193). An ES module by its own extension, whatever the
|
||||
// bundle's package.json says.
|
||||
const launcherSuffix = ".serve.mjs"
|
||||
|
||||
// writeLaunchers writes, beside every entrypoint of a TypeScript bundle, an executable that
|
||||
// imports the entrypoint and serves what it registered over MCP on stdio — through the bundle's
|
||||
// own copy of the SDK, so registering and serving meet in one registry (novox/hq ADR 0193). Its
|
||||
// answer is each entrypoint's launcher, by entrypoint, relative to the bundle's root; nothing for
|
||||
// a language whose build is already executable.
|
||||
func writeLaunchers(root string, chain Toolchain, a catalogue.Artifact) (map[string]string, error) {
|
||||
if chain.Language != "typescript" {
|
||||
return nil, nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, entry := range a.Entrypoints {
|
||||
if !strings.HasSuffix(entry, ".js") {
|
||||
continue
|
||||
}
|
||||
launcher := strings.TrimSuffix(entry, ".js") + launcherSuffix
|
||||
body := "#!/usr/bin/env node\n" +
|
||||
"// Written by the mesh's builder (novox/hq ADR 0193): serve what " + entry + " registers,\n" +
|
||||
"// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.\n" +
|
||||
"import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\n" +
|
||||
"await import(\"./" + filepath.Base(entry) + "\");\n" +
|
||||
"await serveRegisteredOverStdio();\n"
|
||||
path := filepath.Join(root, filepath.FromSlash(launcher))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// WriteFile honours the umask; the mode a machine unpacks is the one packed, so it is set.
|
||||
if err := os.Chmod(path, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[entry] = launcher
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// bundledSuffix is where a bundle's one-file output is written, beside what the compiler wrote.
|
||||
const bundledSuffix = ".bundled"
|
||||
|
||||
// bundled makes every entrypoint and every launcher of a compiled bundle ONE file, in the toolchain
|
||||
// image's bundler, and answers the directory to pack (novox/hq ADR 0193).
|
||||
//
|
||||
// **What a launched bundle runs is what it imports, and nothing else.** Every served bundle is its
|
||||
// own process, so it carries its own copy of the SDK and its own dependencies inlined — the
|
||||
// toolchain's whole node_modules no longer travels in every bundle. An entrypoint a process runs by
|
||||
// name (`node daemon/index.js`) is bundled in place under its own name; a launcher keeps its name
|
||||
// and its first line, and stays executable. A package the bundler cannot inline is named by the
|
||||
// artifact (`external`), kept as an import, and only then is the toolchain's runtime directory
|
||||
// copied beside the files. CommonJS inlined into an ES module still finds `require`.
|
||||
func bundled(ctx context.Context, run Runner, tree string, chain Toolchain, base string,
|
||||
a catalogue.Artifact, launchers map[string]string) (string, error) {
|
||||
const within = "/app/modules/module"
|
||||
out, final := Out(a.Name), Out(a.Name)+bundledSuffix
|
||||
if err := os.RemoveAll(filepath.Join(tree, final)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(tree, final), 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22",
|
||||
"--outbase=" + out, "--outdir=" + final, "--log-level=warning",
|
||||
"--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);"}
|
||||
for _, x := range a.External {
|
||||
common = append(common, "--external:"+x)
|
||||
}
|
||||
var plain []string
|
||||
for _, e := range a.Entrypoints {
|
||||
if strings.HasSuffix(e, ".js") {
|
||||
plain = append(plain, out+"/"+e)
|
||||
}
|
||||
}
|
||||
var launch []string
|
||||
for _, l := range sortedValues(launchers) {
|
||||
launch = append(launch, out+"/"+l)
|
||||
}
|
||||
// Refused by name in an image that predates the bundler, as the dependencies copy is: a bundle
|
||||
// packed without it would carry nothing it imports. Run as itself: npm installs esbuild's native
|
||||
// binary in place of its script, which `node` cannot run.
|
||||
guard := `test -x "$0" || { echo "the toolchain image carries no bundler at $0: it predates one-file bundles, rebuild mesh-tools first" >&2; exit 1; }; exec "$0" "$@"`
|
||||
step := func(entries []string, extra ...string) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
invocation := []string{"run", "--rm", "--volume", tree + ":" + within, "--workdir", within, base,
|
||||
"sh", "-c", guard, chain.Bundler}
|
||||
invocation = append(invocation, entries...)
|
||||
invocation = append(invocation, common...)
|
||||
invocation = append(invocation, extra...)
|
||||
_, err := run(ctx, tree, "docker", invocation...)
|
||||
return err
|
||||
}
|
||||
if err := step(plain); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := step(launch, "--out-extension:.js=.mjs"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Plain `.js` output is an ES module; said once, as the runtime directory used to say it.
|
||||
if err := os.WriteFile(filepath.Join(tree, final, "package.json"), []byte(`{"type":"module","private":true}`+"\n"), 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, l := range launchers {
|
||||
path := filepath.Join(tree, final, filepath.FromSlash(l))
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
if err := os.Chmod(path, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(a.External) > 0 && chain.Dependencies != "" {
|
||||
copying := []string{"run", "--rm", "--volume", tree + ":" + within, "--workdir", within, base,
|
||||
"sh", "-c", `cp -a "$0/node_modules" "$1/"`, chain.Dependencies, final}
|
||||
if _, err := run(ctx, tree, "docker", copying...); err != nil {
|
||||
return "", fmt.Errorf("copying the packages %s keeps external: %w", a.Name, err)
|
||||
}
|
||||
}
|
||||
return filepath.Join(tree, final), nil
|
||||
}
|
||||
|
||||
func sortedValues(m map[string]string) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for _, v := range m {
|
||||
out = append(out, v)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -82,6 +82,65 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) {
|
||||
if !strings.HasPrefix(digest, "sha256:") {
|
||||
t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0])
|
||||
}
|
||||
|
||||
// **One file per entrypoint and launcher, in the toolchain's bundler** (novox/hq ADR 0193). A
|
||||
// second run in the same toolchain image bundles each into the artifact's bundled output, the SDK
|
||||
// inlined, refusing by name in an image that predates the bundler; and the toolchain's
|
||||
// node_modules is no longer copied into a bundle that keeps nothing external.
|
||||
var bundling []string
|
||||
for _, line := range r.ran {
|
||||
if strings.HasPrefix(line, "docker run") && strings.Contains(line, "esbuild") {
|
||||
bundling = append(bundling, line)
|
||||
}
|
||||
}
|
||||
if len(bundling) != 2 {
|
||||
t.Fatalf("want one bundling run for the entrypoints and one for the launchers:\n%s", strings.Join(r.ran, "\n"))
|
||||
}
|
||||
for _, want := range []string{"mesh-tools/build@sha256:", "predates one-file bundles", "--bundle", "--format=esm",
|
||||
"--platform=node", "--outdir=" + Out("code") + ".bundled", Out("code") + "/index.js"} {
|
||||
if !strings.Contains(bundling[0], want) {
|
||||
t.Errorf("the entrypoints' bundling lacks %q: %s", want, bundling[0])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(bundling[1], Out("code")+"/index.serve.mjs") || !strings.Contains(bundling[1], "--out-extension:.js=.mjs") {
|
||||
t.Errorf("the launcher is not bundled under its own name: %s", bundling[1])
|
||||
}
|
||||
if strings.Contains(strings.Join(r.ran, "\n"), "/app/runtime") {
|
||||
t.Errorf("the toolchain's node_modules was copied into a bundle that keeps nothing external:\n%s", strings.Join(r.ran, "\n"))
|
||||
}
|
||||
if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "esbuild") {
|
||||
t.Fatal("the bundler ran before the compile wrote its output")
|
||||
}
|
||||
}
|
||||
|
||||
// A bundle naming packages it keeps external is bundled with them as imports, and carries the
|
||||
// toolchain's node_modules for them — the one case it still does.
|
||||
func TestABundleKeepingAPackageExternalCarriesTheToolchainsModules(t *testing.T) {
|
||||
manifest := strings.Replace(aBundle, `"entrypoints":["index.js"]`, `"entrypoints":["index.js"],"external":["sharp"]`, 1)
|
||||
r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "console.log(1)"})
|
||||
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
|
||||
if _, err := Build(context.Background(), compiling{r}.run, r,
|
||||
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all := strings.Join(r.ran, "\n")
|
||||
if !strings.Contains(all, "--external:sharp") || !strings.Contains(all, "/app/runtime") {
|
||||
t.Errorf("an external package was not kept as an import with the toolchain's modules beside it:\n%s", all)
|
||||
}
|
||||
}
|
||||
|
||||
// A language whose bundle carries its own dependencies copies nothing in: a Go binary is static.
|
||||
func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) {
|
||||
ts, _ := ToolchainFor("typescript")
|
||||
if ts.Dependencies != "/app/runtime" {
|
||||
t.Fatalf("typescript bundles run with %q", ts.Dependencies)
|
||||
}
|
||||
for _, language := range []string{"go", "python"} {
|
||||
chain, _ := ToolchainFor(language)
|
||||
if chain.Dependencies != "" {
|
||||
t.Fatalf("%s copies %q into every bundle, and its bundles carry their own", language, chain.Dependencies)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Refused before anything is built, naming what to build first.** A base the mesh has not built
|
||||
@@ -145,9 +204,13 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) {
|
||||
t.Fatalf("a module with two bundles did not build: %v", err)
|
||||
}
|
||||
|
||||
// Compiled into two different places.
|
||||
// Compiled into two different places. Only the compile lines: the copy of each bundle's
|
||||
// dependencies names the same directory again, deliberately.
|
||||
var outputs []string
|
||||
for _, line := range r.ran {
|
||||
if !strings.Contains(line, "--outDir") {
|
||||
continue
|
||||
}
|
||||
for _, part := range strings.Fields(line) {
|
||||
if strings.HasPrefix(part, ".mesh-build/") {
|
||||
outputs = append(outputs, part)
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A module's own packages, installed before its bundle is compiled, so the bundler inlines them.
|
||||
//
|
||||
// **A bundle could only import what the toolchain happened to carry.** The compiler and the bundler
|
||||
// resolve an import by walking up from the module's source: the module's own directory first, then
|
||||
// the toolchain image's node_modules. Nothing ever put anything in the first, so a module needing a
|
||||
// database driver (`pg`, `mongodb`, `mssql`) could not be a bundle at all, and kept a container whose
|
||||
// recipe installed it by hand (novox/hq ADR 0198 §4: "the backend's own driver inside the bundle").
|
||||
// Now the module's `package.json` says what it depends on, as any Node package does, and the build
|
||||
// installs exactly that into the module's own directory before compiling.
|
||||
//
|
||||
// **The SDK the toolchain carries is the one a bundle is built with, whatever the module says**
|
||||
// (novox/hq issue 212: the toolchain is rebuilt on every SDK release and every bundle after it). A
|
||||
// module's `package.json` names `@novox/mesh-sdk` with a range — it has to, to type-check on a
|
||||
// workstation — and installing that range would shadow the toolchain's copy for this module alone:
|
||||
// one module compiled against an older SDK than its neighbours, chosen by a caret nobody re-reads.
|
||||
// So the SDK is taken out of what is installed (and never fetched), and any copy something else
|
||||
// pulls in is removed afterwards; every import of it resolves past the module's node_modules to the
|
||||
// toolchain's. A module therefore cannot pin a different SDK, by design: the toolchain is the pin.
|
||||
//
|
||||
// **Correctness before speed.** Every build installs afresh into a fresh clone, from the lockfile
|
||||
// when the module has one (`npm ci`, exact) and from its ranges otherwise; nothing installed is kept
|
||||
// between builds. What is shared is npm's own download cache, a named volume, which is
|
||||
// content-addressed and verified by integrity on every read — it saves the network, never the
|
||||
// install. Install scripts do not run: the build node runs nobody's postinstall, and what a script
|
||||
// would build natively could not be inlined into one file anyway.
|
||||
|
||||
// sdkPackage is the package a TypeScript bundle's launcher serves through, and the one package a
|
||||
// module's own dependencies never supply (above).
|
||||
const sdkPackage = "@novox/mesh-sdk"
|
||||
|
||||
// npmCache is the named volume npm's download cache lives in across builds on one build node.
|
||||
const npmCache = "mesh-builder-npm-cache"
|
||||
|
||||
// ownDependencies is what a module's package.json depends on beyond the SDK, sorted; nothing when
|
||||
// the module has no package.json or depends on nothing else — which builds exactly as before.
|
||||
func ownDependencies(tree string) ([]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(tree, "package.json"))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var p struct {
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
|
||||
}
|
||||
var names []string
|
||||
for name := range p.Dependencies {
|
||||
if name != sdkPackage {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// installSteps is the script run inside the toolchain image, from the module's own directory ($0).
|
||||
// It works in a scratch copy so the module's package.json and lockfile are never rewritten, takes
|
||||
// the SDK out of what is installed, installs production dependencies only, removes any copy of the
|
||||
// SDK something pulled in, and puts the result at the module's node_modules.
|
||||
const installSteps = `set -e
|
||||
work="$(mktemp -d)"
|
||||
cp "$0/package.json" "$work/"
|
||||
if [ -f "$0/package-lock.json" ]; then cp "$0/package-lock.json" "$work/"; fi
|
||||
cd "$work"
|
||||
node -e '
|
||||
const fs = require("fs"), sdk = process.argv[1];
|
||||
const p = JSON.parse(fs.readFileSync("package.json", "utf8"));
|
||||
for (const k of ["dependencies", "peerDependencies", "optionalDependencies"]) if (p[k]) delete p[k][sdk];
|
||||
delete p.devDependencies; delete p.scripts;
|
||||
fs.writeFileSync("package.json", JSON.stringify(p));
|
||||
' "$1"
|
||||
shift
|
||||
if [ -f package-lock.json ]; then
|
||||
npm ci --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund "$@"
|
||||
else
|
||||
npm install --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund --no-package-lock "$@"
|
||||
fi
|
||||
find node_modules -depth -type d -path "*/node_modules/@novox/mesh-sdk" -exec rm -rf {} +
|
||||
rm -rf "$0/node_modules"
|
||||
cp -a node_modules "$0/node_modules"
|
||||
`
|
||||
|
||||
// installOwn installs a TypeScript module's own production dependencies into its directory, in the
|
||||
// toolchain image, before the compile — or does nothing at all for a module that has none.
|
||||
func installOwn(ctx context.Context, run Runner, tree string, chain Toolchain, base string,
|
||||
registry Npmrc, say func(step, format string, args ...any)) error {
|
||||
if chain.Language != "typescript" {
|
||||
return nil
|
||||
}
|
||||
deps, err := ownDependencies(tree)
|
||||
if err != nil || len(deps) == 0 {
|
||||
return err
|
||||
}
|
||||
scoped := strings.TrimSpace(registry.Scope)
|
||||
if !registry.Enabled() {
|
||||
// **No registry, no scoped package.** Without the mesh's registry a scoped name resolves on
|
||||
// the public one, where anybody may have published it: a dependency that installs is not
|
||||
// the dependency the module meant.
|
||||
for _, d := range deps {
|
||||
if strings.HasPrefix(d, "@novox/") {
|
||||
return fmt.Errorf("the module depends on %s, and this build knows no package registry "+
|
||||
"for its scope; it would resolve from the public registry, which is not where the "+
|
||||
"mesh publishes it", d)
|
||||
}
|
||||
}
|
||||
}
|
||||
const within = "/app/modules/module"
|
||||
invocation := []string{"run", "--rm",
|
||||
"--volume", tree + ":" + within,
|
||||
"--volume", npmCache + ":/root/.npm",
|
||||
"--workdir", within}
|
||||
var flags []string
|
||||
if registry.Enabled() {
|
||||
// The registry is reached where the binding says it is, which may be this machine's own
|
||||
// loopback — the reason an image build that resolves packages runs on the host network too.
|
||||
invocation = append(invocation, "--network", "host")
|
||||
reg := strings.TrimSpace(registry.Registry)
|
||||
if !strings.HasSuffix(reg, "/") {
|
||||
reg += "/"
|
||||
}
|
||||
flags = append(flags, "--"+scoped+":registry="+reg)
|
||||
}
|
||||
invocation = append(invocation, base, "sh", "-c", installSteps, within, sdkPackage)
|
||||
invocation = append(invocation, flags...)
|
||||
say("bundle", "installing the module's own packages: %s", strings.Join(deps, ", "))
|
||||
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
|
||||
return fmt.Errorf("installing the module's own packages (%s): %w", strings.Join(deps, ", "), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module's own packages (dependencies.go): installed into its own directory, in the toolchain,
|
||||
// before the compile, so the bundler inlines them — the SDK always the toolchain's.
|
||||
|
||||
func buildWithPackageJSON(t *testing.T, pkg string, extra map[string]string, registry Npmrc) (*recorded, error) {
|
||||
t.Helper()
|
||||
files := map[string]string{"index.ts": "console.log(1)"}
|
||||
if pkg != "" {
|
||||
files["package.json"] = pkg
|
||||
}
|
||||
for k, v := range extra {
|
||||
files[k] = v
|
||||
}
|
||||
r, workspace := aRepository(t, aBundle, files)
|
||||
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
|
||||
_, err := Build(context.Background(), compiling{r}.run, r,
|
||||
"https://forge.invalid/greeter.git", "", "", workspace, held, registry, GitCredential{}, nil)
|
||||
return r, err
|
||||
}
|
||||
|
||||
func installs(r *recorded) []string {
|
||||
var out []string
|
||||
for _, line := range r.ran {
|
||||
if strings.HasPrefix(line, "docker run") && strings.Contains(line, "npm ci") {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func compileIndex(r *recorded) int {
|
||||
for i, line := range r.ran {
|
||||
if strings.Contains(line, "--outDir") {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func TestAModulesOwnPackagesAreInstalledInTheToolchainBeforeTheCompile(t *testing.T) {
|
||||
r, err := buildWithPackageJSON(t, `{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0","pg":"^8"},"devDependencies":{"typescript":"^5"}}`,
|
||||
nil, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := installs(r)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want one install of the module's own packages:\n%s", strings.Join(r.ran, "\n"))
|
||||
}
|
||||
line := got[0]
|
||||
for _, want := range []string{
|
||||
"mesh-tools/build@sha256:", // in the toolchain image
|
||||
":/app/modules/module", // into the module's own directory
|
||||
"--workdir /app/modules/module", //
|
||||
npmCache + ":/root/.npm", // npm's verified download cache, and only that
|
||||
"--omit=dev", "--ignore-scripts", // production packages, no build-node scripts
|
||||
"npm ci", "npm install", "--no-package-lock", // the lockfile when there is one, else the ranges
|
||||
"--@novox:registry=https://forge.invalid/api/packages/novox/npm/", // the scope from the mesh's registry
|
||||
"--network host",
|
||||
"@novox/mesh-sdk", // named, to be taken out of what is installed
|
||||
} {
|
||||
if !strings.Contains(line, want) {
|
||||
t.Errorf("the install lacks %q:\n%s", want, line)
|
||||
}
|
||||
}
|
||||
// The SDK is the toolchain's: never installed from the module's range, and any copy removed.
|
||||
if !strings.Contains(line, `delete p[k][sdk]`) || !strings.Contains(line, `-path "*/node_modules/@novox/mesh-sdk" -exec rm -rf`) {
|
||||
t.Errorf("the module's own SDK range could shadow the toolchain's SDK:\n%s", line)
|
||||
}
|
||||
if i, c := strings.Index(strings.Join(r.ran, "\n"), "npm ci"), compileIndex(r); c < 0 ||
|
||||
i > strings.Index(strings.Join(r.ran, "\n"), "--outDir") {
|
||||
t.Fatalf("the install did not run before the compile:\n%s", strings.Join(r.ran, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// **A module with nothing beyond the SDK builds exactly as before**: the same commands, no install.
|
||||
func TestAModuleDependingOnlyOnTheSDKBuildsExactlyAsBefore(t *testing.T) {
|
||||
without, err := buildWithPackageJSON(t, "", nil, Npmrc{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, pkg := range []string{
|
||||
`{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0"},"devDependencies":{"typescript":"^5"}}`,
|
||||
`{"type":"module"}`,
|
||||
} {
|
||||
with, err := buildWithPackageJSON(t, pkg, map[string]string{"package-lock.json": "{}"}, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/npm/"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.Join(with.ran, "\n"), "npm ") {
|
||||
t.Fatalf("a module depending on nothing but the SDK ran npm:\n%s", strings.Join(with.ran, "\n"))
|
||||
}
|
||||
if len(with.ran) != len(without.ran) {
|
||||
t.Fatalf("a module depending only on the SDK built differently from one with no package.json:\n%s\n---\n%s",
|
||||
strings.Join(with.ran, "\n"), strings.Join(without.ran, "\n"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Without the mesh's registry a scoped package would resolve on the public one: refused by name.
|
||||
func TestAScopedPackageWithNoRegistryIsRefused(t *testing.T) {
|
||||
r, err := buildWithPackageJSON(t, `{"dependencies":{"@novox/mesh-sdk":"^0.1.0","@novox/other":"^1"}}`, nil, Npmrc{})
|
||||
if err == nil || !strings.Contains(err.Error(), "@novox/other") {
|
||||
t.Fatalf("a scoped package was installed with no registry for its scope: %v", err)
|
||||
}
|
||||
if strings.Contains(strings.Join(r.ran, "\n"), "--outDir") {
|
||||
t.Fatal("the compile ran after the refusal")
|
||||
}
|
||||
// A public package installs without one, from the public registry and nothing else.
|
||||
r, err = buildWithPackageJSON(t, `{"dependencies":{"mssql":"^11"}}`, nil, Npmrc{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := installs(r); len(got) != 1 || strings.Contains(got[0], ":registry=") || strings.Contains(got[0], "--network host") {
|
||||
t.Fatalf("a public package's install: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadablePackageJSONIsRefusedByName(t *testing.T) {
|
||||
_, err := buildWithPackageJSON(t, `{"dependencies":`, nil, Npmrc{})
|
||||
if err == nil || !strings.Contains(err.Error(), "package.json") {
|
||||
t.Fatalf("a broken package.json was not refused by name: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A build killed by hand (novox/hq ADR 0219).
|
||||
//
|
||||
// A build is a tree of commands — git, then docker, and docker's own children — and a container the
|
||||
// docker client started keeps running when the client dies. So ending one by hand is three things:
|
||||
// the build's context is cancelled, which kills each command's whole process group rather than the
|
||||
// one process the context knows; every container the build started carries the build's id as a
|
||||
// label, so what outlived its client is found and removed by that label; and the holder announces
|
||||
// the outcome itself, since nothing else will.
|
||||
|
||||
// BuildLabel is the label every container a build starts carries, valued with the build's id.
|
||||
const BuildLabel = "mesh.build"
|
||||
|
||||
// KillWait is how long a command killed with its build may take to let go of its output before it
|
||||
// is abandoned: a grandchild holding the pipe open must not hold the build open with it.
|
||||
const KillWait = 10 * time.Second
|
||||
|
||||
// inItsOwnGroup makes a command the leader of its own process group, killed as a group when its
|
||||
// context ends.
|
||||
func inItsOwnGroup(cmd *exec.Cmd) {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmd.Cancel = func() error {
|
||||
if cmd.Process == nil {
|
||||
return nil
|
||||
}
|
||||
// The negative pid is the group: the command and everything it started.
|
||||
if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
|
||||
return cmd.Process.Kill()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
cmd.WaitDelay = KillWait
|
||||
}
|
||||
|
||||
// Labelled is a Runner that marks every container a build starts with the build's id, so a kill
|
||||
// finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
|
||||
// passes rather than at each `docker run` the builder composes: a run added later is labelled too.
|
||||
func Labelled(run Runner, id string) Runner {
|
||||
return func(ctx context.Context, dir string, name string, args ...string) (string, error) {
|
||||
return run(ctx, dir, name, LabelledArgs(name, args, id)...)
|
||||
}
|
||||
}
|
||||
|
||||
// LabelledArgs is a command's arguments with the build's label added, when it is a `docker run`.
|
||||
func LabelledArgs(name string, args []string, id string) []string {
|
||||
if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
|
||||
return args
|
||||
}
|
||||
out := make([]string, 0, len(args)+2)
|
||||
out = append(out, "run", "--label", BuildLabel+"="+id)
|
||||
return append(out, args[1:]...)
|
||||
}
|
||||
|
||||
// RemoveContainersOf removes every container labelled with the build's id, running or not, and
|
||||
// says how many. Run with a context of its own: the build's is the one that was just cancelled.
|
||||
func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error) {
|
||||
out, err := run(ctx, "", "docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ids := strings.Fields(out)
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), nil
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A build killed by hand (novox/hq ADR 0219) ends every command it started: the context's end kills
|
||||
// the command's whole process group, not the one process the context knows about.
|
||||
func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
child := filepath.Join(dir, "child")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
began := time.Now()
|
||||
go func() {
|
||||
// A shell that starts a grandchild and waits on it: killing the shell alone would leave the
|
||||
// grandchild running, holding the output open.
|
||||
_, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`)
|
||||
done <- err
|
||||
}()
|
||||
var pid int
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
|
||||
if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" {
|
||||
pid, _ = strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||
break
|
||||
}
|
||||
}
|
||||
if pid == 0 {
|
||||
t.Fatal("the command never started its child")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("a killed command reported success")
|
||||
}
|
||||
case <-time.After(KillWait + 5*time.Second):
|
||||
t.Fatal("the killed command never returned")
|
||||
}
|
||||
if took := time.Since(began); took > KillWait {
|
||||
t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took)
|
||||
}
|
||||
for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
|
||||
if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) {
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = syscall.Kill(pid, syscall.SIGKILL)
|
||||
t.Fatalf("the grandchild %d outlived the kill", pid)
|
||||
}
|
||||
|
||||
// Every `docker run` a build starts carries its id as a label; nothing else is touched.
|
||||
func TestEveryContainerABuildStartsCarriesItsID(t *testing.T) {
|
||||
got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
|
||||
if want := []string{"run", "--label", "mesh.build=build-1", "--rm", "img", "sh"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("docker run became %v", got)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
args []string
|
||||
}{{"docker", []string{"build", "."}}, {"git", []string{"run"}}, {"docker", nil}} {
|
||||
if got := LabelledArgs(c.name, c.args, "build-1"); !reflect.DeepEqual(got, c.args) {
|
||||
t.Errorf("%s %v became %v", c.name, c.args, got)
|
||||
}
|
||||
}
|
||||
var ran [][]string
|
||||
run := Labelled(func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
ran = append(ran, append([]string{name}, args...))
|
||||
return "", nil
|
||||
}, "build-2")
|
||||
_, _ = run(context.Background(), "", "docker", "run", "img")
|
||||
if want := [][]string{{"docker", "run", "--label", "mesh.build=build-2", "img"}}; !reflect.DeepEqual(ran, want) {
|
||||
t.Errorf("ran %v", ran)
|
||||
}
|
||||
}
|
||||
|
||||
// What a kill removes is found by the label, and only what it finds.
|
||||
func TestAKillRemovesTheContainersLabelledWithTheBuild(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "ps" {
|
||||
return "c1\nc2\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
n, err := RemoveContainersOf(context.Background(), run, "build-3")
|
||||
if err != nil || n != 2 {
|
||||
t.Fatalf("%d %v", n, err)
|
||||
}
|
||||
if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker rm -f c1 c2"}; !reflect.DeepEqual(ran, want) {
|
||||
t.Errorf("ran %v", ran)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0193: every entrypoint the runtime may serve is executable, and the runtime knows no
|
||||
// language — so a TypeScript bundle carries a launcher beside each entrypoint.
|
||||
func TestATypeScriptBundleCarriesAnExecutableLauncherBesideEachEntrypoint(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
chain, err := ToolchainFor("typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := writeLaunchers(root, chain, catalogue.Artifact{Name: "tools", Kind: catalogue.ArtifactBundle,
|
||||
Language: "typescript", Entrypoints: []string{"tools/index.js", "index.js"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["tools/index.js"] != "tools/index.serve.mjs" || got["index.js"] != "index.serve.mjs" {
|
||||
t.Fatalf("launchers: %v", got)
|
||||
}
|
||||
path := filepath.Join(root, "tools", "index.serve.mjs")
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o755 {
|
||||
t.Errorf("the launcher is %v, not executable 0755", info.Mode().Perm())
|
||||
}
|
||||
body, _ := os.ReadFile(path)
|
||||
for _, want := range []string{"#!/usr/bin/env node\n", `from "@novox/mesh-sdk/stdio"`, `await import("./index.js")`, "serveRegisteredOverStdio()"} {
|
||||
if !strings.Contains(string(body), want) {
|
||||
t.Errorf("the launcher lacks %q:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
|
||||
// A compiled language's build is executable already: no launcher.
|
||||
goChain, _ := ToolchainFor("go")
|
||||
none, err := writeLaunchers(t.TempDir(), goChain, catalogue.Artifact{Name: "tools", Kind: catalogue.ArtifactBundle, Language: "go"})
|
||||
if err != nil || len(none) != 0 {
|
||||
t.Errorf("a Go bundle was given launchers: %v %v", none, err)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
)
|
||||
|
||||
// Where built artifacts go.
|
||||
@@ -66,22 +68,32 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string)
|
||||
return pinned, nil
|
||||
}
|
||||
|
||||
// PublishArchive stores bytes as a blob and returns where to fetch them from.
|
||||
// PublishArchive stores bytes as a blob, holds it by a manifest, and returns where to fetch them
|
||||
// from.
|
||||
//
|
||||
// Two steps, which is the registry's own protocol: ask for somewhere to put it, then put it there
|
||||
// naming the digest. The registry verifies the digest itself, so a blob that arrived corrupted is
|
||||
// refused by the thing storing it rather than by the machine unpacking it a week later.
|
||||
// Two steps for the blob, which is the registry's own protocol: ask for somewhere to put it, then
|
||||
// put it there naming the digest. The registry verifies the digest itself, so a blob that arrived
|
||||
// corrupted is refused by the thing storing it rather than by the machine unpacking it a week
|
||||
// later.
|
||||
//
|
||||
// **Then a manifest that names it** (novox/hq issue 253, ADR 0189). The store's own collector
|
||||
// marks only from manifests, and a blob no manifest names is collected however much the mesh
|
||||
// means to keep it — so an archive published bare is an archive the first nightly collection
|
||||
// deletes. The holder is composed from the digest and size alone (artifacts.Holder), which is
|
||||
// what lets the sweep recompute it to backfill or let go without anything being recorded here.
|
||||
// What a machine is told to fetch is the blob, exactly as before.
|
||||
func (r Registry) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
||||
base := "http://" + r.Address + "/v2/" + repository
|
||||
final := base + "/blobs/" + digest
|
||||
|
||||
// Already there. Blobs are immutable and named by their content, so this is not an
|
||||
// optimisation — re-uploading would be asking the registry to store what it already has under
|
||||
// the name it already has.
|
||||
// the name it already has. It is still held: a blob published before holders existed is
|
||||
// exactly the one a rebuild of the same source finds already there.
|
||||
if there, err := r.has(ctx, final); err != nil {
|
||||
return "", err
|
||||
} else if there {
|
||||
return final, nil
|
||||
return final, r.hold(ctx, repository, digest, len(body))
|
||||
}
|
||||
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/blobs/uploads/", nil)
|
||||
@@ -119,7 +131,17 @@ func (r Registry) PublishArchive(ctx context.Context, repository string, body []
|
||||
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||
return "", fmt.Errorf("%s refused the blob: %s %s", base, done.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return final, nil
|
||||
return final, r.hold(ctx, repository, digest, len(body))
|
||||
}
|
||||
|
||||
// hold puts the manifest holding an archive beside it. A build whose archive could not be held is
|
||||
// a failed build: recorded as published, it would be an archive the store's collector takes.
|
||||
func (r Registry) hold(ctx context.Context, repository, digest string, size int) error {
|
||||
store := artifacts.Store{Address: r.Address, HTTP: r.client()}
|
||||
if _, err := store.HoldBlob(ctx, repository, digest, int64(size)); err != nil {
|
||||
return fmt.Errorf("published %s/blobs/%s and could not hold it by a manifest: %w", repository, digest, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// has is whether this registry already holds what is at that URL.
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
)
|
||||
|
||||
// An OCI registry as a content-addressed blob store, which is what it is.
|
||||
@@ -18,9 +20,10 @@ import (
|
||||
// there is not sent again, and that a tag is never accepted as a pin.
|
||||
|
||||
type fakeRegistry struct {
|
||||
blobs map[string][]byte
|
||||
uploads int
|
||||
location string
|
||||
blobs map[string][]byte
|
||||
manifests map[string][]byte // "<repository>@<digest>"
|
||||
puts map[string]int // blob uploads, by digest
|
||||
location string
|
||||
}
|
||||
|
||||
func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
|
||||
@@ -28,6 +31,8 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
|
||||
if f.blobs == nil {
|
||||
f.blobs = map[string][]byte{}
|
||||
}
|
||||
f.manifests = map[string][]byte{}
|
||||
f.puts = map[string]int{}
|
||||
mux := http.NewServeMux()
|
||||
server := httptest.NewServer(mux)
|
||||
mux.HandleFunc("/v2/", func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -39,8 +44,28 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case strings.Contains(r.URL.Path, "/manifests/"):
|
||||
// The archive's holder (novox/hq issue 253): asked for with an Accept, put by digest.
|
||||
repository, digest, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/v2/"), "/manifests/")
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
if _, ok := f.manifests[repository+"@"+digest]; ok &&
|
||||
strings.Contains(r.Header.Get("Accept"), "application/vnd.oci.image.manifest.v1+json") {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodPut:
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
sum := sha256.Sum256(body)
|
||||
if digest != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
f.manifests[repository+"@"+digest] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/blobs/uploads/"):
|
||||
f.uploads++
|
||||
where := f.location
|
||||
if where == "" {
|
||||
where = "/v2/upload/" + hex.EncodeToString([]byte("session"))
|
||||
@@ -58,6 +83,7 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
|
||||
return
|
||||
}
|
||||
f.blobs[digest] = body
|
||||
f.puts[digest]++
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
@@ -92,6 +118,57 @@ func TestAnArchiveIsStoredAndFetchableByItsDigest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArchiveIsPublishedWithAManifestHoldingIt(t *testing.T) {
|
||||
// The store's collector marks only from manifests, so a bare blob is one the first collection
|
||||
// deletes, kept or not (novox/hq issue 253, ADR 0189). Every archive goes out held, by the
|
||||
// holder the sweep can compute for itself from the digest and size.
|
||||
f := &fakeRegistry{}
|
||||
r := registryFor(t, f)
|
||||
body := []byte("a theme")
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
|
||||
where, err := r.PublishArchive(context.Background(), "shell/config", body, digest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasSuffix(where, "/v2/shell/config/blobs/"+digest) {
|
||||
t.Fatalf("a machine is told to fetch %q; the blob is still what is fetched", where)
|
||||
}
|
||||
manifest, holder := artifacts.Holder(digest, int64(len(body)))
|
||||
if got := f.manifests["shell/config@"+holder]; string(got) != string(manifest) {
|
||||
t.Fatalf("the store holds %v; want the holder %s in the archive's own repository", f.manifests, holder)
|
||||
}
|
||||
if !strings.Contains(string(manifest), `"digest":"`+digest+`"`) {
|
||||
t.Fatalf("the holder does not name the archive: %s", manifest)
|
||||
}
|
||||
empty := sha256.Sum256([]byte("{}"))
|
||||
if _, ok := f.blobs["sha256:"+hex.EncodeToString(empty[:])]; !ok {
|
||||
t.Fatal("the holder's empty config was never put, and a registry refuses a manifest without it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArchiveAlreadyThereIsStillHeld(t *testing.T) {
|
||||
// A rebuild of the same source finds its archive already there — often one published bare,
|
||||
// before holders. It is not sent again, and it is held.
|
||||
f := &fakeRegistry{}
|
||||
r := registryFor(t, f)
|
||||
body := []byte("published bare")
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
f.blobs[digest] = body
|
||||
|
||||
if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.puts[digest] != 0 {
|
||||
t.Fatal("a blob already there was sent again")
|
||||
}
|
||||
if _, holder := artifacts.Holder(digest, int64(len(body))); f.manifests["shell/config@"+holder] == nil {
|
||||
t.Fatal("a blob already there was left bare")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABlobAlreadyThereIsNotSentAgain(t *testing.T) {
|
||||
// Not an optimisation: blobs are named by their content, so re-uploading is asking the
|
||||
// registry to store what it already has under the name it already has.
|
||||
@@ -107,8 +184,11 @@ func TestABlobAlreadyThereIsNotSentAgain(t *testing.T) {
|
||||
if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.uploads != 1 {
|
||||
t.Fatalf("the blob was uploaded %d times", f.uploads)
|
||||
if f.puts[digest] != 1 {
|
||||
t.Fatalf("the blob was uploaded %d times", f.puts[digest])
|
||||
}
|
||||
if len(f.manifests) != 1 {
|
||||
t.Fatalf("publishing twice left %d holders; want the one", len(f.manifests))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -200,3 +200,23 @@ func TestWhatABuildReadIsTheRepositoriesItsRecipesName(t *testing.T) {
|
||||
t.Fatal("a module whose recipes name no other repository read one")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/212: a toolchain stands on the SDK's published package, and is built with the
|
||||
// exact version the mesh published — an argument that changes when the SDK does, so a rebuild after
|
||||
// a release never reuses an install of the version before it.
|
||||
func TestAPackageTheMeshPublishedIsPassedByItsExactVersion(t *testing.T) {
|
||||
manifest := catalogue.Manifest{
|
||||
Module: "mesh-tools",
|
||||
Build: &catalogue.Build{
|
||||
On: []catalogue.BuildsOn{{Arg: "MESH_SDK", Module: "mesh-sdk", Artifact: "lib"}},
|
||||
},
|
||||
}
|
||||
held := map[string]string{"mesh-sdk/lib": "@novox/mesh-sdk@0.1.6"}
|
||||
args, resolved, err := standingOn(context.Background(), manifest, held, noMirror)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(args) != "[--build-arg MESH_SDK=@novox/mesh-sdk@0.1.6]" || fmt.Sprint(resolved) != "[@novox/mesh-sdk@0.1.6]" {
|
||||
t.Errorf("the package was passed as %v, recorded as %v", args, resolved)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,33 @@ type Toolchain struct {
|
||||
// carrying its debug info. The mistake was believing a comment rather than reading the file it
|
||||
// produced (novox/hq 04-ISSUES/161).
|
||||
LinkerFlags []string
|
||||
// Dependencies is a directory inside the toolchain image whose contents a bundle in this
|
||||
// language runs with, copied whole into the compiled output's root after the compile.
|
||||
//
|
||||
// **A bundle that compiles is not yet a bundle that runs.** The compiler resolves `import
|
||||
// "nats"` from the toolchain image's own node_modules and the pack takes only what the compiler
|
||||
// wrote, so what a machine unpacked could not find a single dependency — and no TypeScript bundle
|
||||
// had ever run live to show it (novox/hq to-be 38 WP3). For TypeScript the directory holds a
|
||||
// `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a
|
||||
// bundle with its dependencies and without that line still fails to start — and the pruned,
|
||||
// production-only node_modules the runtime itself ships with: the SDK's and the runtime's
|
||||
// dependencies, and nothing module-specific (a module's own npm dependencies are installed into
|
||||
// its own directory before the compile and inlined by the bundler: dependencies.go). Empty for a
|
||||
// language whose bundle carries its own —
|
||||
// a Go binary is static, a Python bundle is installed with its dependencies.
|
||||
//
|
||||
// A toolchain image without the directory fails the build by name rather than packing a bundle
|
||||
// that starts nowhere: the image predates this and must be rebuilt first.
|
||||
//
|
||||
// *Since the bundler (below):* copied only for a bundle that names packages it keeps external,
|
||||
// which cannot be inlined; a bundle with none carries no node_modules at all.
|
||||
Dependencies string
|
||||
// Bundler is the bundler inside the toolchain image that makes each compiled entrypoint and each
|
||||
// launcher ONE self-contained file (novox/hq ADR 0193): every served bundle is its own process
|
||||
// now, so each carries its own copy of what it imports — the SDK included — and nothing else.
|
||||
// A bundle shrinks from the toolchain's whole node_modules to the code it runs. Empty for a
|
||||
// language whose build is already one file.
|
||||
Bundler string
|
||||
// SystemStamp is the variable this language's linker fills with the artifact's declared system,
|
||||
// for a language whose binaries are pinned to one at link time (novox/hq ADR 0005).
|
||||
//
|
||||
@@ -107,14 +134,22 @@ var toolchains = []Toolchain{
|
||||
// symlinks to a launcher that requires its library relatively — and the base image's own
|
||||
// assembly resolves them away, leaving a launcher whose relative require points nowhere.
|
||||
// Every module's hand-written Dockerfile had to know this. Now none of them does.
|
||||
// **Rooted at the module, so an entrypoint lands where it is named.** Without a root the
|
||||
// compiler takes the common directory of the files it is given: a module compiling only
|
||||
// `tools/index.ts` had its output at `index.js`, and the entrypoint it declared —
|
||||
// `tools/index.js`, "named as it will be found" — named a file the bundle did not
|
||||
// contain. The runtime that loads bundles by their declared entrypoints (novox/hq ADR
|
||||
// 0175) is what made this visible.
|
||||
Compile: []string{
|
||||
"node", "/app/node_modules/typescript/bin/tsc",
|
||||
"--module", "NodeNext", "--moduleResolution", "NodeNext",
|
||||
"--target", "ES2022",
|
||||
"--target", "ES2022", "--rootDir", ".",
|
||||
},
|
||||
OutputFlag: "--outDir",
|
||||
Unit: UnitSources,
|
||||
SourceExt: ".ts",
|
||||
OutputFlag: "--outDir",
|
||||
Unit: UnitSources,
|
||||
SourceExt: ".ts",
|
||||
Dependencies: "/app/runtime",
|
||||
Bundler: "/app/node_modules/esbuild/bin/esbuild",
|
||||
},
|
||||
{
|
||||
Language: "go",
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq issue 213 (ADR 0188 §1, §3): a module's own Go service is a bundle the host runs as a
|
||||
// process, not an image. Each test holds one thing that had to change in the composer for the
|
||||
// controller to be declared that way.
|
||||
|
||||
const aServiceDigest = "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
||||
|
||||
// aServiceModule is the controller's shape in miniature: it answers tools of its own, its code is a
|
||||
// Go bundle a process runs as an account it declares, its secrets belong to that account, it
|
||||
// prepares its state, and its process replaces the container it used to run as.
|
||||
func aServiceModule(t *testing.T) Manifest {
|
||||
t.Helper()
|
||||
raw := `{
|
||||
"module": "svc", "version": "1", "tools": ["status"], "prepares": true,
|
||||
"own-secrets": {"store": "${dir:state}/store"},
|
||||
"secrets-owner": "svc",
|
||||
"resources": [
|
||||
{"id": "state", "type": "directory", "mode": "0700", "place": "mesh", "owner": "svc"},
|
||||
{"id": "service", "type": "process", "name": "svc", "artifact": "code",
|
||||
"run": ["./svc", "serve"], "user": "svc", "replaces": ["server"],
|
||||
"env": {"SVC_STORE_FILE": "${dir:state}/store", "SVC_STORE_PORT": "${seat:mesh-store:5432}"}},
|
||||
{"id": "account", "type": "user", "name": "svc", "shell": "/usr/bin/nologin", "home": "/var/lib/svc"}
|
||||
],
|
||||
"build": {"artifacts": [{"name": "code", "kind": "bundle", "language": "go", "system": "arch",
|
||||
"from": "cmd/svc", "binary": "svc"}]}
|
||||
}`
|
||||
m, err := ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("the service's manifest is refused: %v", err)
|
||||
}
|
||||
resolved, err := m.Resolve([]Built{{Name: "code", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "svc/code@" + aServiceDigest, Digest: aServiceDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func composeTheService(t *testing.T, with Rendering) []map[string]any {
|
||||
t.Helper()
|
||||
with.Needed = map[string]map[string]string{"svc": {"store": "sealed-store"}}
|
||||
with.ArtifactStore = "anchor.internal:5100"
|
||||
out, err := Resolution{Node: "anchor", Modules: []Manifest{aServiceModule(t)}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatalf("the service does not compose: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func indexOf(out []map[string]any, id string) int {
|
||||
for i, r := range out {
|
||||
if r["id"] == id {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// A module that declares tools has every bundle served by the node's runtime unless it says
|
||||
// otherwise — and the controller declares the verbs it answers as tools. Its service bundle is run
|
||||
// by its own process; launched a second time by the runtime it would be a second controller
|
||||
// pretending to be an MCP server.
|
||||
func TestABundleItsOwnProcessRunsIsNotServedByTheRuntime(t *testing.T) {
|
||||
m := aServiceModule(t)
|
||||
if len(m.Bundles) != 1 {
|
||||
t.Fatalf("the service's bundle was not kept: %+v", m.Bundles)
|
||||
}
|
||||
if loads := m.Bundles[0].Loads; len(loads) != 0 {
|
||||
t.Fatalf("the runtime would launch the service's own bundle as tools: %v", loads)
|
||||
}
|
||||
// And a bundle no resource runs still is served, as a module declaring tools always had it.
|
||||
tools := Manifest{Module: "t", Version: "1", Tools: []string{"x"},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "tools", Kind: ArtifactBundle, Language: "go",
|
||||
System: "arch", From: "cmd/t"}}}}
|
||||
resolved, err := tools.Resolve([]Built{{Name: "tools", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "t/tools@" + aServiceDigest, Digest: aServiceDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loads := resolved.Bundles[0].Loads; len(loads) != 1 || loads[0] != "t" {
|
||||
t.Fatalf("a tools bundle nothing runs is no longer served: %v", loads)
|
||||
}
|
||||
}
|
||||
|
||||
// The account is created before anything is given to it. Its secrets are mesh-computed and so
|
||||
// placed before the module's own resources; given to a user the machine did not have yet, they were
|
||||
// refused on the first apply and the process started without them.
|
||||
func TestAModulesAccountComesBeforeWhatBelongsToIt(t *testing.T) {
|
||||
out := composeTheService(t, Rendering{})
|
||||
account, secret := indexOf(out, "svc.account"), indexOf(out, "svc."+NeedID("store"))
|
||||
if account < 0 || secret < 0 {
|
||||
t.Fatalf("the account or the secret is missing: %v", out)
|
||||
}
|
||||
if account > secret {
|
||||
t.Fatalf("the secret owned by svc is written before svc exists: account at %d, secret at %d",
|
||||
account, secret)
|
||||
}
|
||||
if owner := out[secret]["owner"]; owner != "svc" {
|
||||
t.Errorf("the secret belongs to %v, not the account its process runs as", owner)
|
||||
}
|
||||
}
|
||||
|
||||
// The process is the module's program; its preparation is the same program asked to prepare, as a
|
||||
// step before it — with the same account and environment, and handing nothing over.
|
||||
func TestAProcessIsPreparedByItsOwnProgram(t *testing.T) {
|
||||
out := composeTheService(t, Rendering{})
|
||||
step, process := indexOf(out, "svc.service-prepare"), indexOf(out, "svc.service")
|
||||
if step < 0 || process < 0 || step > process {
|
||||
t.Fatalf("the preparation is not a step before the process (%d, %d): %v", step, process, out)
|
||||
}
|
||||
s := out[step]
|
||||
if s["type"] != "process" || s["run-once"] != true || s["name"] != "svc-prepare" {
|
||||
t.Errorf("the preparation is not a run-once process: %v", s)
|
||||
}
|
||||
if run, _ := json.Marshal(s["run"]); string(run) != `["./svc","prepare"]` {
|
||||
t.Errorf("the preparation runs %s", run)
|
||||
}
|
||||
if s["user"] != "svc" || s["source"] != out[process]["source"] {
|
||||
t.Errorf("the preparation does not run the same bundle as the same account: %v", s)
|
||||
}
|
||||
if env, _ := s["env"].(map[string]any); env["SVC_STORE_FILE"] == nil {
|
||||
t.Errorf("the preparation is not given the process's environment: %v", s["env"])
|
||||
}
|
||||
if _, has := s["replaces"]; has {
|
||||
t.Errorf("the preparation would hand over what the process replaces: %v", s)
|
||||
}
|
||||
if _, has := s["args"]; has {
|
||||
t.Errorf("the preparation carries a container's args: %v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// What the process replaces is named as the host recorded it, `<module>.<id>`; unprefixed, the host
|
||||
// matches nothing and removes the container first, as before.
|
||||
func TestWhatAProcessReplacesIsNamedAsTheHostRecordedIt(t *testing.T) {
|
||||
out := composeTheService(t, Rendering{})
|
||||
p := out[indexOf(out, "svc.service")]
|
||||
if got, _ := json.Marshal(p["replaces"]); string(got) != `["svc.server"]` {
|
||||
t.Fatalf("the process replaces %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatReplacesMayNameIsRefusedNearItsAuthor(t *testing.T) {
|
||||
for what, resource := range map[string]string{
|
||||
"a container": `{"id":"c","type":"container","name":"c","image":"x@` + aServiceDigest + `","replaces":["old"]}`,
|
||||
"a step": `{"id":"p","type":"process","name":"p","run":["./p"],"run-once":true,"replaces":["old"]}`,
|
||||
"something declared": `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["p"]}`,
|
||||
"another module's": `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["other.old"]}`,
|
||||
"not a list": `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":"old"}`,
|
||||
} {
|
||||
raw := `{"module":"m","version":"1","resources":[` + resource + `]}`
|
||||
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "replace") {
|
||||
t.Errorf("replaces on %s was accepted: %v", what, err)
|
||||
}
|
||||
}
|
||||
ok := `{"module":"m","version":"1","resources":[{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["old"]}]}`
|
||||
if _, err := ParseManifest([]byte(ok)); err != nil {
|
||||
t.Errorf("a process replacing what its module no longer declares was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A store provider says how its data is backed up (novox/hq ADR 0214); a provider of something that
|
||||
// holds nothing does not have to.
|
||||
func TestAStoreProviderWithoutABackupIsRefusedByTheCheck(t *testing.T) {
|
||||
bare, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
||||
"provides":[{"name":"postgres-database","scope":"mesh"}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parsing refused it, and the rule is the check's: %v", err)
|
||||
}
|
||||
if problems := CheckBackup(bare); len(problems) != 1 || !strings.Contains(problems[0], "node-backup") {
|
||||
t.Fatalf("a store with no backup passed the check: %v", problems)
|
||||
}
|
||||
backed, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
||||
"provides":[{"name":"postgres-database","scope":"mesh"}],
|
||||
"resources":[{"id":"dumps","type":"directory","mode":"0700"}],
|
||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:dumps}"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if problems := CheckBackup(backed); len(problems) != 0 {
|
||||
t.Fatalf("a store that contributes a backup was refused: %v", problems)
|
||||
}
|
||||
route, _ := ParseManifest([]byte(`{"module":"r","version":"1","provides":[{"name":"route","scope":"mesh"}]}`))
|
||||
if problems := CheckBackup(route); len(problems) != 0 {
|
||||
t.Fatalf("a route was asked for a backup: %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
// A backup contribution names its module's own directories; one it does not declare is refused
|
||||
// where it is written rather than reaching the holder as the literal text.
|
||||
func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "nowhere") {
|
||||
t.Fatalf("a backup of an undeclared directory was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The holder receives every module's backup lines with each module's own directories filled, each
|
||||
// module's under a comment naming it — and a kind written in a shell's grammar is left untouched.
|
||||
func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) {
|
||||
pg, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
||||
"resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}],
|
||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mail, err := ParseManifest([]byte(`{"module":"mail","version":"1",
|
||||
"resources":[{"id":"spool","type":"directory","mode":"0700"}],
|
||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:spool}"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"# pg\nrun pg-dump-all\npath /srv/pg/dumps\n", "# mail\npath /var/lib/mail/spool\n"} {
|
||||
if !strings.Contains(placed, want) {
|
||||
t.Errorf("placed contributions lack %q:\n%s", want, placed)
|
||||
}
|
||||
}
|
||||
if strings.Contains(placed, "${dir:") {
|
||||
t.Errorf("a directory reached the holder unfilled:\n%s", placed)
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,9 @@ import (
|
||||
// formats and gains no fields.
|
||||
//
|
||||
// **It stays name-agnostic** ([ADR 0027]). The mesh does not learn what a `postgres-database` is:
|
||||
// `at`, `as` and `from` are facts about any provision at all, and everything else comes from what
|
||||
// the provider said it serves — whose keys are agreed by the requirement's name, not by this file.
|
||||
// `at`, `as`, `from` and `address` (the providing machine's private address, novox/hq ADR 0194) are
|
||||
// facts about any provision at all, and everything else comes from what the provider said it
|
||||
// serves — whose keys are agreed by the requirement's name, not by this file.
|
||||
|
||||
// bound is where a module says a value from one of its bindings belongs:
|
||||
// ${bound:<provision>.<key>}.
|
||||
@@ -46,7 +47,7 @@ func boundUsed(content string) [][2]string {
|
||||
// Three facts the mesh states about any provision, plus whatever the provider said it serves. A
|
||||
// module may not reach a binding it does not have — the same boundary as a secret, for the same
|
||||
// reason.
|
||||
func knownFor(m Manifest, needs []Needed, node string) map[string]map[string]string {
|
||||
func knownFor(m Manifest, needs []Needed, node string) (map[string]map[string]string, error) {
|
||||
out := map[string]map[string]string{}
|
||||
for _, want := range m.Wants() {
|
||||
for i := range needs {
|
||||
@@ -54,12 +55,20 @@ func knownFor(m Manifest, needs []Needed, node string) map[string]map[string]str
|
||||
if n.Name != want || n.For != m.Module {
|
||||
continue
|
||||
}
|
||||
as := ConsumerIdentity(node, IdentitySource(m.Slug, m.Module))
|
||||
values := map[string]string{
|
||||
"at": n.At,
|
||||
"from": n.From,
|
||||
"as": ConsumerIdentity(node, IdentitySource(m.Slug, m.Module)),
|
||||
"as": as,
|
||||
}
|
||||
for key, value := range n.Serves {
|
||||
// What the provider derives for this consumer rather than for all of them
|
||||
// (novox/hq ADR 0201). Filled here, the one place a provision and the module
|
||||
// requiring it are both in hand.
|
||||
served, err := ServedTo(n.Serves, as)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s requires %s: %w", m.Module, want, err)
|
||||
}
|
||||
for key, value := range served {
|
||||
// The provider's own vocabulary. Rendered plainly: a port is 5432, not 5432.000000,
|
||||
// which is what a float would write and what a connection string would refuse.
|
||||
values[key] = plainly(value)
|
||||
@@ -67,7 +76,7 @@ func knownFor(m Manifest, needs []Needed, node string) map[string]map[string]str
|
||||
out[want] = values
|
||||
}
|
||||
}
|
||||
return out
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// withOwnNames adds a module's own composed names to what it may name from one binding:
|
||||
|
||||
@@ -232,3 +232,34 @@ func TestTwoModulesOnOneNodeAreTwoIdentities(t *testing.T) {
|
||||
t.Fatal("one module on two machines shares an identity")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's resolver configuration must name its resolver by address — it cannot resolve the name
|
||||
// of the thing it resolves names with (novox/hq ADR 0194). So a binding offers the providing
|
||||
// machine's private address beside its name, and only when the machine has one.
|
||||
func TestABindingOffersTheProvidersAddress(t *testing.T) {
|
||||
consumer := func() Resolution {
|
||||
return Resolution{
|
||||
Node: "workstation",
|
||||
Modules: []Manifest{{
|
||||
Module: "resolv-conf",
|
||||
Requires: []string{"wildcard-resolution"},
|
||||
Resources: []map[string]any{{
|
||||
"id": "resolv", "type": "file", "path": "/etc/resolv.conf", "mode": "0644",
|
||||
"content": "nameserver ${bound:wildcard-resolution:address}\n",
|
||||
}},
|
||||
}},
|
||||
Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", At: "anchor.internal", For: "resolv-conf"}},
|
||||
}
|
||||
}
|
||||
out, err := consumer().Declaration(Rendering{Machines: map[string]string{"anchor.internal": "10.77.0.1"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "resolv-conf.resolv")["content"]; got != "nameserver 10.77.0.1\n" {
|
||||
t.Fatalf("the resolver is not named by its address: %q", got)
|
||||
}
|
||||
// A machine with no address yet: refused, never written with a blank where the address belongs.
|
||||
if _, err := consumer().Declaration(Rendering{}); err == nil {
|
||||
t.Fatal("a file naming an address the mesh does not have was composed")
|
||||
}
|
||||
}
|
||||
|
||||
+181
-1
@@ -2,6 +2,7 @@ package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
@@ -28,6 +29,9 @@ type Built struct {
|
||||
Reference string
|
||||
// Digest is "sha256:<hex>", for an archive. An image reference already ends in one.
|
||||
Digest string
|
||||
// Launchers are, for a bundle in an interpreted language, the executable the build wrote beside
|
||||
// each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it.
|
||||
Launchers map[string]string
|
||||
}
|
||||
|
||||
// Resolve fills a manifest's resources in from what was built.
|
||||
@@ -67,6 +71,49 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
|
||||
out := m
|
||||
out.Build = nil
|
||||
out.Resources = nil
|
||||
// What the build compiled, kept on the resolved manifest (novox/hq ADR 0175): a tools bundle is
|
||||
// named by no resource of the module's own — the node's runtime loads it — so this is the only
|
||||
// place the mesh would otherwise not have it. In artifact order, so two resolutions of one
|
||||
// build compare equal.
|
||||
out.Bundles = nil
|
||||
if m.Build != nil {
|
||||
run := runByAResource(m)
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != ArtifactBundle {
|
||||
continue
|
||||
}
|
||||
made := by[a.Name]
|
||||
// What the runtime loads: what the artifact said, else every entrypoint of a module
|
||||
// that declares tools, else nothing (the field's own rule; see Artifact.Loads).
|
||||
//
|
||||
// **Never, unasked, a bundle one of the module's own resources runs** (novox/hq issue 213).
|
||||
// A process the host runs is the module's service, not its tools: the controller declares
|
||||
// the verbs it answers as `tools` and serves them itself, and its bundle would otherwise
|
||||
// have been launched a second time by the node's runtime, as an MCP child it is not.
|
||||
loads := append([]string(nil), a.Loads...)
|
||||
if a.Loads == nil && len(m.Tools) > 0 && !run[a.Name] {
|
||||
loads = append([]string(nil), a.Entrypoints...)
|
||||
// A bundle compiled to a binary has no entrypoints: the binary is what it is, and what
|
||||
// the runtime starts to serve it (novox/hq ADR 0193). So a Go tools bundle is served
|
||||
// as Go — the runtime execs it — exactly as a TypeScript one is through its launcher.
|
||||
if bin := BinaryOf(a); bin != "" {
|
||||
loads = []string{bin}
|
||||
}
|
||||
}
|
||||
// **Kept, never routed** (ADR 0155): the builder publishes to the store at the address
|
||||
// it reached it by, and a manifest carrying that address names an installation —
|
||||
// registration refused node-tools for exactly this on 2026-10-02. The build record
|
||||
// already keeps the store-relative form; the resolved manifest keeps the same, and
|
||||
// composition routes it through the store a machine reaches (Routed).
|
||||
out.Bundles = append(out.Bundles, Bundle{
|
||||
Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest,
|
||||
Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...),
|
||||
Loads: loads, Env: copyWords(a.Env), Launchers: copyWords(made.Launchers),
|
||||
Binary: BinaryOf(a),
|
||||
})
|
||||
}
|
||||
sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name })
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
named, _ := r["artifact"].(string)
|
||||
if named == "" {
|
||||
@@ -110,7 +157,8 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
|
||||
// The same on the wire: both are bytes fetched by digest and unpacked. They differ in
|
||||
// how they were made — one packed as it stood, the other compiled first — and a
|
||||
// machine has no reason to care which.
|
||||
filled["source"] = artifact.Reference
|
||||
// Kept, not routed, for the reason the bundles above are (ADR 0155).
|
||||
filled["source"] = Recorded(artifact.Reference)
|
||||
filled["digest"] = artifact.Digest
|
||||
// **And `${version}`, so a resource can name a place that is this build's alone**
|
||||
// (novox/hq ADR 0141, 04-ISSUES/142). A component is unpacked into a directory named
|
||||
@@ -172,6 +220,17 @@ func (b *Build) problems(module string) []string {
|
||||
// A bundle's source is the module's own directory by definition, and what it needs to say
|
||||
// is which compiler — because the mesh chooses that, and cannot choose for a module that
|
||||
// has not said.
|
||||
if len(a.External) > 0 && (a.Kind != ArtifactBundle || a.Language != "typescript") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q names packages it keeps external, and only a TypeScript bundle is bundled into "+
|
||||
"one file with some kept out (novox/hq ADR 0193)", module, a.Name))
|
||||
}
|
||||
if len(a.Env) > 0 && a.Kind != ArtifactBundle {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+
|
||||
"serves is given words (novox/hq ADR 0192); a container says its own environment",
|
||||
module, a.Name, a.Kind))
|
||||
}
|
||||
if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage {
|
||||
// **Except for a language that compiles to a binary, where it names which one**
|
||||
// (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own
|
||||
@@ -191,6 +250,26 @@ func (b *Build) problems(module string) []string {
|
||||
"%s: %q is a bundle and says no language, so nothing can choose a compiler "+
|
||||
"for it", module, a.Name))
|
||||
}
|
||||
problems = append(problems, bundleEnvProblems(module, a)...)
|
||||
// What the runtime loads is among what was compiled (ADR 0175): a name here that is
|
||||
// not an entrypoint is a file the bundle does not contain, and the runtime would
|
||||
// fail to import it on every machine rather than here.
|
||||
for _, load := range a.Loads {
|
||||
found := false
|
||||
for _, e := range a.Entrypoints {
|
||||
found = found || e == load
|
||||
}
|
||||
// A bundle compiled to a binary is one executable: the runtime loads that or nothing
|
||||
// (novox/hq ADR 0193).
|
||||
if bin := BinaryOf(a); bin != "" {
|
||||
found = load == bin
|
||||
}
|
||||
if !found {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q says the runtime loads %q, which is not among its entrypoints — "+
|
||||
"what is loaded is compiled, so it is named there too", module, a.Name, load))
|
||||
}
|
||||
}
|
||||
// **A system, for a language that compiles to a binary** (novox/hq ADR 0142). A binary
|
||||
// is pinned to one operating system at link time so a host refuses to touch a machine
|
||||
// it was not built for (novox/hq ADR 0005); an artifact that says nothing would be
|
||||
@@ -315,3 +394,104 @@ func versionOf(digest string) string {
|
||||
}
|
||||
return hex
|
||||
}
|
||||
|
||||
// bundleEnvWords are the words the runtime sets for itself; a bundle that named one would be
|
||||
// telling the runtime what it is, which is the mesh's to say (novox/hq ADR 0192).
|
||||
var bundleEnvWords = map[string]bool{
|
||||
RuntimeToolModules: true, RuntimeBrokerFile: true, RuntimeOperatorAccount: true,
|
||||
RuntimeOperatorHome: true, RuntimeToolEnv: true,
|
||||
}
|
||||
|
||||
// bundleEnvProblems says what is wrong with what a bundle says it is given (novox/hq ADR 0192):
|
||||
// a value is a path or a constant written with the references a container's environment may use
|
||||
// for a place or a port, and never a secret's content or another module's binding — a secret
|
||||
// reaches a tool as a file whose path is named.
|
||||
func bundleEnvProblems(module string, a Artifact) []string {
|
||||
if len(a.Env) == 0 {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
for _, word := range sortedKeys(a.Env) {
|
||||
value := a.Env[word]
|
||||
if bundleEnvWords[word] {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q gives itself %s, which the node's runtime sets for itself; a bundle is "+
|
||||
"given its own words beside the runtime's, never in place of them (novox/hq ADR 0192)",
|
||||
module, a.Name, word))
|
||||
}
|
||||
rest := ofPort.ReplaceAllString(dirRef.ReplaceAllString(value, ""), "")
|
||||
if strings.Contains(rest, "${") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q gives %s the value %q. A bundle's word is a path or a constant, written with "+
|
||||
"${dir:…} and ${port:…} only; a secret reaches a tool as a file the mesh places, "+
|
||||
"named by its path, never as its content (novox/hq ADR 0192)",
|
||||
module, a.Name, word, value))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func copyWords(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BinaryOf is what a bundle compiled to a binary is called once built: what the artifact names, else
|
||||
// the package it is built from, else the artifact's own name (novox/hq 04-ISSUES/142). Empty for a
|
||||
// language that does not compile to one. The builder writes the binary under this name, and the
|
||||
// composer runs it by it, so both ask here.
|
||||
func BinaryOf(a Artifact) string {
|
||||
if !compilesToABinary(a.Language) {
|
||||
return ""
|
||||
}
|
||||
if name := strings.TrimSpace(a.Binary); name != "" {
|
||||
return name
|
||||
}
|
||||
if from := strings.Trim(a.From, "./"); from != "" {
|
||||
return path.Base(from)
|
||||
}
|
||||
return a.Name
|
||||
}
|
||||
|
||||
// runByAResource is the artifacts one of a module's own resources names — a process that runs it,
|
||||
// a step, an archive that unpacks it — by name.
|
||||
func runByAResource(m Manifest) map[string]bool {
|
||||
named := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if a, ok := r["artifact"].(string); ok && a != "" {
|
||||
named[a] = true
|
||||
}
|
||||
}
|
||||
return named
|
||||
}
|
||||
|
||||
// undeliveredBundles says which of a module's bundles nothing would ever put on a machine (novox/hq
|
||||
// 04-ISSUES/216). A bundle reaches a machine three ways: the node's runtime serves it (it says
|
||||
// `loads`, or its module declares `tools`), a resource names it (a process, a step, an archive), or
|
||||
// it is the runtime itself. One reached by none of them was built, recorded and pushed as success,
|
||||
// and was simply absent — seven modules' tools went missing that way on 2026-10-03. Refused here,
|
||||
// naming the field that would deliver it.
|
||||
func undeliveredBundles(m Manifest) []string {
|
||||
if m.Build == nil || m.Module == RuntimeModule {
|
||||
return nil
|
||||
}
|
||||
named := runByAResource(m)
|
||||
var problems []string
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != ArtifactBundle || named[a.Name] || len(a.Loads) > 0 || len(m.Tools) > 0 {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: the bundle %q would be built and never reach a machine: nothing loads it, runs it or "+
|
||||
"unpacks it. A tools bundle says `loads` (the entrypoints the node's runtime serves) or its "+
|
||||
"module lists its `tools`; a daemon or a step is a resource naming it (novox/hq 04-ISSUES/216)",
|
||||
m.Module, a.Name))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -86,3 +86,36 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
|
||||
t.Fatalf("%d value(s) name an installation:\n %s", len(named), strings.Join(named, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// TestEveryCatalogueStoreSaysHowItIsBackedUp is ADR 0214's check over the real catalogue: a module
|
||||
// providing a store contributes a backup to node-backup, so a store added is a store backed up.
|
||||
func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
|
||||
root := catalogueRoot(t)
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil || len(found) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
stores := 0
|
||||
for _, p := range found {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
continue // TestEveryCatalogueManifestParses says why
|
||||
}
|
||||
for _, o := range m.Provides {
|
||||
if storeProvisions[o.Name] {
|
||||
stores++
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, problem := range CheckBackup(m) {
|
||||
t.Error(problem)
|
||||
}
|
||||
}
|
||||
if stores == 0 {
|
||||
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What a provider derives for one consumer, said once in the provider's definition and delivered
|
||||
// to both ends (novox/hq ADR 0201, issue 124).
|
||||
//
|
||||
// A `serves` block is otherwise literal: the same values for every consumer. Where the provider
|
||||
// *names the resource* — a bucket, a database, a vhost — the name is derived from who is asking,
|
||||
// and before this the mesh had no channel for it. The provider recomputed it in its own code and
|
||||
// every consumer transcribed it into its own definition by hand, which is a copy of somebody
|
||||
// else's rule kept in agreement by nobody. One of three transcriptions was wrong for months.
|
||||
//
|
||||
// **The mesh learns no protocol here; it spells its own name in an alphabet it already knows.**
|
||||
// The only fact a served value may name is the identity the mesh itself minted for the consumer,
|
||||
// in one of two alphabets: as it was minted, and as a DNS label. Everything a provider wants
|
||||
// around it — a prefix, a suffix, a separator — it writes around the placeholder, because a
|
||||
// served value is a string.
|
||||
|
||||
// consumerFact is `${consumer:<fact>}` or `${consumer:<fact>:<alphabet>}`.
|
||||
var consumerFact = regexp.MustCompile(`\$\{consumer:([a-z][a-z0-9-]*)(?::([a-z][a-z0-9-]*))?\}`)
|
||||
|
||||
// consumerFacts are what a served value may name about the consumer it is being derived for.
|
||||
// One entry, deliberately: the identity is the one thing about a consumer the mesh itself chose,
|
||||
// so it is the one thing the mesh can hand to a provider without either end guessing.
|
||||
var consumerFacts = []string{"as"}
|
||||
|
||||
// consumerAlphabets are the ways the mesh will write that identity. `dns` is the mesh's own
|
||||
// identifier with its separator written `-` instead of `_` — the whole of the difference between
|
||||
// the alphabet the mesh mints in and the one buckets, vhosts and hostnames accept.
|
||||
var consumerAlphabets = []string{"dns"}
|
||||
|
||||
// ServedTo fills a provider's served values for one consumer.
|
||||
//
|
||||
// `as` is the identity the mesh minted for that consumer — the same string it is told to present
|
||||
// as a login. Values with no placeholder are returned exactly as they were, and a block with no
|
||||
// placeholder at all is returned unchanged, so this costs nothing for the providers that derive
|
||||
// nothing.
|
||||
//
|
||||
// Only strings carry placeholders. A number, a boolean or a nested object is a value the provider
|
||||
// stated outright, and is left alone.
|
||||
func ServedTo(serves map[string]any, as string) (map[string]any, error) {
|
||||
if len(serves) == 0 {
|
||||
return serves, nil
|
||||
}
|
||||
var out map[string]any
|
||||
for _, key := range sortedAnyKeys(serves) {
|
||||
text, ok := serves[key].(string)
|
||||
if !ok || !strings.Contains(text, "${consumer:") {
|
||||
continue
|
||||
}
|
||||
filled, err := consumerInto(text, as)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the value served as %q: %w", key, err)
|
||||
}
|
||||
if out == nil {
|
||||
// Copied only once something actually changes: the caller's map is the manifest's,
|
||||
// and a provider that derives nothing must not have it rewritten underneath it.
|
||||
out = make(map[string]any, len(serves))
|
||||
for k, v := range serves {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
out[key] = filled
|
||||
}
|
||||
if out == nil {
|
||||
return serves, nil
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// consumerInto replaces every `${consumer:…}` in one value.
|
||||
//
|
||||
// **A fact or an alphabet the mesh does not have is refused, not left standing.** Written through,
|
||||
// the literal `${consumer:as}` would reach a configuration file and be read as a bucket name,
|
||||
// failing somewhere that names neither the module nor the mesh — the same reasoning `${bound:…}`
|
||||
// is refused by (boundInto).
|
||||
func consumerInto(value, as string) (string, error) {
|
||||
var failed error
|
||||
out := consumerFact.ReplaceAllStringFunc(value, func(match string) string {
|
||||
parts := consumerFact.FindStringSubmatch(match)
|
||||
fact, alphabet := parts[1], parts[2]
|
||||
if fact != "as" {
|
||||
if failed == nil {
|
||||
failed = fmt.Errorf(
|
||||
"says %s, and the mesh states %s about a consumer", match, orNothing(consumerFacts))
|
||||
}
|
||||
return match
|
||||
}
|
||||
switch alphabet {
|
||||
case "":
|
||||
return as
|
||||
case "dns":
|
||||
return asDNSLabel(as)
|
||||
default:
|
||||
if failed == nil {
|
||||
failed = fmt.Errorf(
|
||||
"says %s, and the mesh writes an identity as %s", match, orNothing(consumerAlphabets))
|
||||
}
|
||||
return match
|
||||
}
|
||||
})
|
||||
if failed != nil {
|
||||
return "", failed
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// asDNSLabel writes a minted identity as a DNS label.
|
||||
//
|
||||
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
|
||||
// already short enough for the tightest backend they reach (CheckIdentity, twenty characters). So
|
||||
// this is the separator and nothing else — no lower-casing of what is already lower case, no
|
||||
// truncation to a limit the identity is already inside, no padding of a name that is already long
|
||||
// enough. Each of those would be the mesh guessing at a rule it has not been given.
|
||||
func asDNSLabel(as string) string {
|
||||
return strings.ReplaceAll(as, "_", "-")
|
||||
}
|
||||
|
||||
// CheckServes refuses a `serves` block that names a consumer fact or an alphabet the mesh does not
|
||||
// have, when the definition is parsed rather than when a consumer is resolved.
|
||||
//
|
||||
// A provision nobody consumes yet still has its rule read: a definition that would be refused the
|
||||
// first time somebody required it is a definition that is wrong now.
|
||||
func CheckServes(m Manifest) []string {
|
||||
var problems []string
|
||||
for _, provision := range sortedServes(m.Serves) {
|
||||
for _, key := range sortedAnyKeys(m.Serves[provision]) {
|
||||
text, ok := m.Serves[provision][key].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// A probe identity, because what is checked is the shape of the statement and not
|
||||
// what any consumer is called.
|
||||
if _, err := consumerInto(text, "mesh_node_module"); err != nil {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func sortedServes(serves map[string]map[string]any) []string {
|
||||
out := make([]string, 0, len(serves))
|
||||
for k := range serves {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedAnyKeys(values map[string]any) []string {
|
||||
out := make([]string, 0, len(values))
|
||||
for k := range values {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// derivedFor is what the provider on this machine derives for one consumer of one provision
|
||||
// (novox/hq ADR 0201).
|
||||
//
|
||||
// Settled first, then derived: an operator may set a prefix on what the provider serves and the
|
||||
// mesh still fills the consumer's half of it ([ADR 0174]). Only the keys that actually name the
|
||||
// consumer are returned — the rest of a `serves` block is the same for every consumer and is
|
||||
// already in the provider's own definition, so repeating it here would be a second copy to go
|
||||
// stale.
|
||||
//
|
||||
// The first module in the resolved order that says it serves the provision answers, which is the
|
||||
// choice servedOnThisMachine makes for the consumer's half. Nothing serving it on this machine is
|
||||
// not an error: a contribution can reach a machine whose provider is a record or an adapter, and
|
||||
// then there is nothing derived to tell.
|
||||
func (r Resolution) derivedFor(provision, as, consumer, local string, settings SettingsBy) (map[string]any, error) {
|
||||
for _, m := range r.Modules {
|
||||
serves, said := m.Serves[provision]
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
var names map[string]any
|
||||
for key, value := range serves {
|
||||
if text, ok := value.(string); ok && strings.Contains(text, "${consumer:") {
|
||||
if names == nil {
|
||||
names = map[string]any{}
|
||||
}
|
||||
names[key] = value
|
||||
}
|
||||
}
|
||||
if names == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// **A consumer that keeps several holders of this provision is refused** — this is issue
|
||||
// 124's own failure one case to the side, and it would be just as quiet.
|
||||
//
|
||||
// Each holder gets its own login, `…_<local>` (ADR 0094), and a provider derives from the
|
||||
// login, so it would make one resource per holder. The consumer's side has no such
|
||||
// dimension: one binding file per provision, one `${bound:<provision>:<key>}`, both
|
||||
// derived from the un-suffixed identity. So the provider would create the holder's
|
||||
// resource and the consumer would be configured against a name nothing made — it would
|
||||
// authenticate successfully and be refused on every object, which reads like a credential
|
||||
// fault and is not one.
|
||||
//
|
||||
// Lifting this means giving the consumer's side a local dimension. That is a decision,
|
||||
// not an omission, and until it is taken the mesh says so rather than guessing.
|
||||
if local != "" {
|
||||
return nil, fmt.Errorf(
|
||||
"%s keeps several holders of %s (this one is %q), and %s derives %s for each "+
|
||||
"consumer from the login the mesh minted. Each holder has its own login, and a "+
|
||||
"consumer is told one value per requirement — so the two ends would name "+
|
||||
"different things and nothing would compare them (novox/hq ADR 0201)",
|
||||
consumer, local, provision, m.Module, orNothing(sortedAnyKeys(names)))
|
||||
}
|
||||
settled, err := Settle(names, settings[m.Module])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
|
||||
}
|
||||
derived, err := ServedTo(settled, as)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s serving %s to %s: %w", m.Module, provision, as, err)
|
||||
}
|
||||
return derived, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// notTranscribed refuses a consumer's file that writes out the value its provider derives for it,
|
||||
// instead of asking for it (novox/hq ADR 0201, issue 124).
|
||||
//
|
||||
// **What would have caught the one wrong instance.** The object store's three consumers each wrote
|
||||
// their bucket into their own configuration by hand. One of them named a predecessor's bucket, and
|
||||
// nothing compared it to what the provider would actually create: the module would have
|
||||
// authenticated successfully and been refused on every object, which reads like a credential fault
|
||||
// and is not one. It looked authoritative for months.
|
||||
//
|
||||
// The test is exact and costs one string search: a definition whose file already contains the
|
||||
// value the mesh is about to derive for it has written down somebody else's rule. It cannot be a
|
||||
// coincidence — a derived value carries the identity the mesh minted for this very consumer on
|
||||
// this very machine, which nothing else would spell out — and it cannot be checked afterwards,
|
||||
// because after substitution every consumer's file contains it legitimately.
|
||||
//
|
||||
// Only values that actually name the consumer are judged. A provider that serves a constant under
|
||||
// the same key serves the same constant to everyone, and a consumer repeating it is redundant
|
||||
// rather than wrong.
|
||||
func notTranscribed(resource map[string]any, known map[string]map[string]string, module string) error {
|
||||
if fmt.Sprint(resource["type"]) != "file" {
|
||||
return nil
|
||||
}
|
||||
content, ok := resource["content"].(string)
|
||||
if !ok || content == "" {
|
||||
return nil
|
||||
}
|
||||
for _, provision := range sortedKnown(known) {
|
||||
values := known[provision]
|
||||
identity := values["as"]
|
||||
if identity == "" {
|
||||
continue
|
||||
}
|
||||
for _, key := range sortedStringKeys(values) {
|
||||
if key == "as" {
|
||||
// The login is not derived from itself, and a consumer that must present it in a
|
||||
// connection string legitimately has it from `${bound:…}` — which is what it will
|
||||
// be after substitution, so this would judge the substitution, not the module.
|
||||
continue
|
||||
}
|
||||
value := values[key]
|
||||
if value == "" || !namesTheConsumer(value, identity) {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(content, value) {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"%s writes %q into %v, and that is exactly what %s derives for it — a definition "+
|
||||
"keeping its own copy of somebody else's naming rule is one that can disagree "+
|
||||
"with it, silently. Say ${bound:%s:%s} and be told",
|
||||
module, value, resource["id"], provision, provision, key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// namesTheConsumer is whether a derived value was built from this consumer's identity — in the
|
||||
// alphabet it was minted in, or as a DNS label. A value that does not contain it was not derived
|
||||
// from it, whatever else it may be.
|
||||
func namesTheConsumer(value, identity string) bool {
|
||||
return strings.Contains(value, identity) || strings.Contains(value, asDNSLabel(identity))
|
||||
}
|
||||
|
||||
func sortedKnown(known map[string]map[string]string) []string {
|
||||
out := make([]string, 0, len(known))
|
||||
for k := range known {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedStringKeys(values map[string]string) []string {
|
||||
out := make([]string, 0, len(values))
|
||||
for k := range values {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0210 §3: a contribution is a dependency on the seat that receives it.
|
||||
|
||||
func TestAContributionDependsOnTheSeatThatPlacesIt(t *testing.T) {
|
||||
env := mod("theme", nil, nil, nil)
|
||||
env.Environment = &Environment{Variables: map[string]string{"GTK_THEME": "Adwaita:dark"}}
|
||||
path := mod("toolchain", nil, nil, nil)
|
||||
path.Environment = &Environment{Path: []PathEntry{{Entry: "/opt/x/bin"}}}
|
||||
shell := mod("prompt", nil, nil, nil)
|
||||
shell.Shell = []ShellCode{{For: "zsh", Slot: "first", Code: "true"}}
|
||||
session := mod("wallpaper", nil, nil, nil)
|
||||
session.Shell = []ShellCode{{For: "xinitrc", Slot: "normal", Code: "true"}}
|
||||
resources := mod("bar", nil, nil, nil)
|
||||
resources.Shell = []ShellCode{{For: "xresources", Slot: "normal", Code: "x: y"}}
|
||||
empty := mod("nothing", nil, nil, nil)
|
||||
empty.Environment = &Environment{}
|
||||
|
||||
cases := map[string]struct {
|
||||
m Manifest
|
||||
want []string
|
||||
}{
|
||||
"a variable": {env, []string{EnvironmentSeat}},
|
||||
"a path entry": {path, []string{EnvironmentSeat}},
|
||||
"shell code": {shell, []string{LoginShellSeat}},
|
||||
"the session's start": {session, []string{DisplayServerSeat}},
|
||||
"the session's X resources": {resources, []string{DisplayServerSeat}},
|
||||
"an empty environment": {empty, nil},
|
||||
}
|
||||
for name, c := range cases {
|
||||
got := DependsOn(c.m)
|
||||
if len(got) == 0 {
|
||||
got = nil
|
||||
}
|
||||
if !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s depends on %v, want %v", name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAContributionIsMetByAHolderOnTheNodeAndRefusedWithout(t *testing.T) {
|
||||
holder := mod("node-env", nil, nil, nil, Claim{Name: EnvironmentSeat})
|
||||
contributor := mod("theme", nil, nil, nil)
|
||||
contributor.Environment = &Environment{Variables: map[string]string{"GTK_THEME": "Adwaita:dark"}}
|
||||
catalogue := map[string]Manifest{"node-env": holder, "theme": contributor}
|
||||
|
||||
if _, err := AssignRefusal(catalogue, "laptop", []string{"node-env"}, []string{"theme"}); err != nil {
|
||||
t.Fatalf("a contributor beside the holder is refused: %v", err)
|
||||
}
|
||||
_, err := AssignRefusal(catalogue, "laptop", nil, []string{"theme"})
|
||||
if err == nil {
|
||||
t.Fatal("a contributor on a node without the holder was accepted, and its contribution would be written nowhere")
|
||||
}
|
||||
if !strings.Contains(err.Error(), EnvironmentSeat) || !strings.Contains(err.Error(), "node-env") {
|
||||
t.Errorf("the refusal names neither the seat nor its holder: %v", err)
|
||||
}
|
||||
if _, err := AssignRefusal(catalogue, "laptop", nil, []string{"theme", "node-env"}); err != nil {
|
||||
t.Errorf("the contributor and the holder assigned together are refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHolderMeetsItsOwnContribution(t *testing.T) {
|
||||
// The display server's module contributes nothing to its own seat today, but the zsh module's
|
||||
// environment contributions do go to another seat: a claim meets only the seat it names.
|
||||
zsh := mod("zsh", nil, nil, nil, Claim{Name: LoginShellSeat})
|
||||
zsh.Shell = []ShellCode{{For: "zsh", Slot: "normal", Code: "true"}}
|
||||
zsh.Environment = &Environment{Variables: map[string]string{"EDITOR": "vim"}}
|
||||
catalogue := map[string]Manifest{"zsh": zsh,
|
||||
"node-env": mod("node-env", nil, nil, nil, Claim{Name: EnvironmentSeat})}
|
||||
unheld := UnheldDependencies(catalogue, "laptop", []Manifest{zsh}, nil)
|
||||
if len(unheld) != 1 || unheld[0].Seat != EnvironmentSeat {
|
||||
t.Errorf("zsh alone: unheld %v, want only %s (its shell code is its own seat's)", unheld, EnvironmentSeat)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoModulesDeclaringOnePackageAreRefusedBeforeAnythingIsRecorded(t *testing.T) {
|
||||
pacman := withResources(mod("pacman", nil, nil, nil, Claim{Name: PackageManagerSeat}),
|
||||
res("package", "pacman-contrib"))
|
||||
bar := withResources(mod("bar", nil, nil, nil), res("package", "bar"), res("package", "pacman-contrib"))
|
||||
other := withResources(mod("other", nil, nil, nil), res("package", "other"))
|
||||
catalogue := map[string]Manifest{"pacman": pacman, "bar": bar, "other": other}
|
||||
|
||||
err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"bar"})
|
||||
if err == nil || !strings.Contains(err.Error(), "pacman-contrib") || !strings.Contains(err.Error(), "bar") {
|
||||
t.Fatalf("the second owner of a package was not refused by name: %v", err)
|
||||
}
|
||||
if err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"other"}); err != nil {
|
||||
t.Errorf("a module declaring nothing shared is refused: %v", err)
|
||||
}
|
||||
// A collision already on the node is status's, not a reason to refuse an unrelated assignment.
|
||||
if err := CollisionRefusal(catalogue, "laptop", []string{"pacman", "bar"}, []string{"other"}); err != nil {
|
||||
t.Errorf("an unrelated assignment is refused for a collision already there: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodeForAPowerMomentDependsOnThePowerSeat(t *testing.T) {
|
||||
for _, moment := range powerMoments {
|
||||
m := mod("laptop", nil, nil, nil)
|
||||
m.Shell = []ShellCode{{For: moment, Slot: "normal", Code: "true"}}
|
||||
if got := DependsOn(m); !reflect.DeepEqual(got, []string{PowerSeat}) {
|
||||
t.Errorf("code for %s depends on %v, want %s", moment, got, PowerSeat)
|
||||
}
|
||||
if p := m.shellProblems(); len(p) != 0 {
|
||||
t.Errorf("code for %s is refused: %v", moment, p)
|
||||
}
|
||||
}
|
||||
m := mod("laptop", nil, nil, nil)
|
||||
m.Shell = []ShellCode{{For: "after-lunch", Slot: "normal", Code: "true"}}
|
||||
if p := m.shellProblems(); len(p) == 0 {
|
||||
t.Error("code for a moment that does not exist was accepted")
|
||||
}
|
||||
if s, ok := SeatNamed(PowerSeat); !ok || s.Scope != ScopeNode {
|
||||
t.Errorf("%s is not a node seat of the mesh's own: %+v %v", PowerSeat, s, ok)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq issue 213: the controller is a Go program and was the one piece of the mesh's own Go
|
||||
// code still shipped as an image (ADR 0188 §1). Its own manifest, composed for the machine that runs
|
||||
// it, is a Go bundle run by the host as a process — and no container.
|
||||
func TestTheControllerIsAProcessAndNoContainer(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("the controller's own manifest does not parse:\n%v", err)
|
||||
}
|
||||
if m.Build == nil || len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("the controller builds %+v; it is one bundle", m.Build)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if a.Kind != ArtifactBundle || a.Language != "go" || a.System == "" || BinaryOf(a) != "mesh-controller" {
|
||||
t.Fatalf("the controller's artifact is %+v, not a Go bundle naming its system and binary", a)
|
||||
}
|
||||
for _, c := range m.Capabilities {
|
||||
if c == "container-runtime" {
|
||||
t.Error("the controller still requires a container runtime on its machine")
|
||||
}
|
||||
}
|
||||
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
control, err := m.Resolve([]Built{{Name: a.Name, Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "mesh-controller/" + a.Name + "@" + digest, Digest: digest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The node's runtime does not launch it: it serves its seat's verbs itself.
|
||||
if loads := control.Bundles[0].Loads; len(loads) != 0 {
|
||||
t.Errorf("the node's runtime would launch the controller as a tools bundle: %v", loads)
|
||||
}
|
||||
|
||||
needed := map[string]map[string]string{"mesh-controller": {}}
|
||||
for name := range m.OwnSecrets {
|
||||
needed["mesh-controller"][name] = "sealed-" + name
|
||||
}
|
||||
out, err := Resolution{Node: "anchor", Modules: []Manifest{control}}.Declaration(Rendering{
|
||||
Needed: needed, ArtifactStore: "anchor.internal:5100",
|
||||
Seats: map[string]map[int]int{"mesh-store": {5432: 6852}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("the controller does not compose: %v", err)
|
||||
}
|
||||
|
||||
var process, step map[string]any
|
||||
account, firstSecret := -1, -1
|
||||
for i, r := range out {
|
||||
switch {
|
||||
case r["type"] == "container":
|
||||
t.Errorf("the controller's declaration still runs a container: %v", r)
|
||||
case r["id"] == "mesh-controller.controller":
|
||||
process = r
|
||||
case r["id"] == "mesh-controller.controller-prepare":
|
||||
step = r
|
||||
if process != nil {
|
||||
t.Error("the controller's preparation is placed after the process it prepares for")
|
||||
}
|
||||
case r["type"] == "user" && r["name"] == "mesh-controller":
|
||||
account = i
|
||||
case strings.HasPrefix(fmt.Sprint(r["id"]), "mesh-controller.needs-") && firstSecret < 0:
|
||||
firstSecret = i
|
||||
}
|
||||
}
|
||||
if process == nil {
|
||||
t.Fatalf("the controller's process is not in its declaration: %v", out)
|
||||
}
|
||||
if run, _ := json.Marshal(process["run"]); string(run) != `["./mesh-controller","serve"]` {
|
||||
t.Errorf("the controller is run as %s, not its own bundle's binary", run)
|
||||
}
|
||||
if process["source"] != "anchor.internal:5100/mesh-controller/"+a.Name+"@"+digest || process["digest"] != digest {
|
||||
t.Errorf("the controller's bundle is fetched from %v (%v)", process["source"], process["digest"])
|
||||
}
|
||||
// The user: an account the host declares, which owns what the process reads.
|
||||
if process["user"] != "mesh-controller" || account < 0 {
|
||||
t.Errorf("the controller runs as %v, and the account declared is at %d", process["user"], account)
|
||||
}
|
||||
if firstSecret >= 0 && account > firstSecret {
|
||||
t.Error("the controller's secrets are written before the account they belong to exists")
|
||||
}
|
||||
for _, r := range out {
|
||||
if strings.HasPrefix(fmt.Sprint(r["id"]), "mesh-controller.needs-") && r["owner"] != "mesh-controller" {
|
||||
t.Errorf("%v belongs to %v, which the controller's process cannot read", r["id"], r["owner"])
|
||||
}
|
||||
}
|
||||
if dir := fileNamed(out, "mesh-controller.mesh-state"); dir == nil || dir["owner"] != "mesh-controller" {
|
||||
t.Errorf("the controller's state directory is not its account's to enter: %v", dir)
|
||||
}
|
||||
// Each mount became a path the process reads: nothing it is told is a path inside a container.
|
||||
state := fmt.Sprint(fileNamed(out, "mesh-controller.mesh-state")["path"])
|
||||
env, _ := process["env"].(map[string]any)
|
||||
for key, value := range env {
|
||||
v := fmt.Sprint(value)
|
||||
if strings.HasPrefix(v, "/run/secrets") || strings.HasPrefix(v, "/broker-tls") {
|
||||
t.Errorf("%s=%s is a path inside the container the controller no longer runs in", key, v)
|
||||
}
|
||||
if strings.HasSuffix(key, "_FILE") && !strings.HasPrefix(v, state+"/") {
|
||||
t.Errorf("%s=%s is not one of the files the mesh places for it", key, v)
|
||||
}
|
||||
}
|
||||
if env["MESH_BROKER_CERTIFICATE"] != "/var/lib/mesh-broker-tls/tls.crt" {
|
||||
t.Errorf("the controller reads the broker's certificate from %v", env["MESH_BROKER_CERTIFICATE"])
|
||||
}
|
||||
if env["MESH_STORE_INVENTORY_PORT"] != "6852" {
|
||||
t.Errorf("the controller is told the store is on %v; the node put it on 6852", env["MESH_STORE_INVENTORY_PORT"])
|
||||
}
|
||||
// The handover: the container it ran as goes only once this is running.
|
||||
if got, _ := json.Marshal(process["replaces"]); string(got) != `["mesh-controller.server"]` {
|
||||
t.Errorf("the controller's process replaces %s, not the container it ran as", got)
|
||||
}
|
||||
// And its state is prepared first, by the same program as the same account.
|
||||
if step == nil || step["run-once"] != true || step["user"] != "mesh-controller" {
|
||||
t.Fatalf("the controller's preparation is %v", step)
|
||||
}
|
||||
if run, _ := json.Marshal(step["run"]); string(run) != `["./mesh-controller","prepare"]` {
|
||||
t.Errorf("the controller's preparation runs %s", run)
|
||||
}
|
||||
}
|
||||
@@ -155,6 +155,10 @@ type Rendering struct {
|
||||
// standing beside the machines and looking as real as they do.
|
||||
Machines map[string]string
|
||||
|
||||
// Zones is every zone a module in the mesh answers itself, where it is answered (novox/hq ADR
|
||||
// 0199): the mesh's resolver forwards each one there.
|
||||
Zones []ZoneAt
|
||||
|
||||
Settings SettingsBy
|
||||
Generators map[string]Generator
|
||||
// Grants are the credentials this node must create, for the provisions it offers. Passed in
|
||||
@@ -415,6 +419,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation,
|
||||
with.OutwardLinks, with.TunnelInterface)
|
||||
|
||||
// **A variable two modules set is refused whether or not anything places it** (novox/hq ADR
|
||||
// 0203 §5): the account has one environment, and a machine whose holder arrives later should not
|
||||
// be the moment two modules are found to disagree about it.
|
||||
if err := variablesSetOnce(r.Modules); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out []map[string]any
|
||||
for _, m := range r.Modules {
|
||||
if with.Adopted && m.Filtering != nil {
|
||||
@@ -508,10 +519,27 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
return nil, fmt.Errorf(
|
||||
"%s needs a secret called %q and none was made for it", m.Module, name)
|
||||
}
|
||||
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
|
||||
// The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175,
|
||||
// to-be 38 WP3): its process is composed `user: <account>` where the node has one, and a
|
||||
// root-owned 0600 file is one that process cannot read. Composed here rather than said in
|
||||
// the manifest, because a manifest cannot say ${machine:account} safely — a node with no
|
||||
// account has nothing to resolve it to, and then the runtime runs as root and the file
|
||||
// stays root's.
|
||||
owner := m.SecretsOwner
|
||||
if m.Module == RuntimeModule && r.Account != "" {
|
||||
owner = r.Account
|
||||
}
|
||||
first = append(first, ownedBy(owner, map[string]any{
|
||||
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed,
|
||||
}))
|
||||
}
|
||||
// This module's tools bundles, where the machine runs the node's tool runtime (novox/hq
|
||||
// ADR 0175, to-be 38 WP2). Mesh-computed like everything above it, and before the module's
|
||||
// own resources for the same reason: the runtime's process names the files inside these
|
||||
// and is restarted when one changes, so they are on the machine before it is.
|
||||
if r.runtimeHere() {
|
||||
first = append(first, bundleArchives(m)...)
|
||||
}
|
||||
// Operator-owned paths this module is granted use of (novox/hq ADR 0051). Written before
|
||||
// the module's own resources, and so before the container that mounts them: the host must
|
||||
// find each present — refusing clearly if the operator has not provided it — before it
|
||||
@@ -586,14 +614,14 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// and nothing would say so.
|
||||
continue
|
||||
}
|
||||
first = append(first, map[string]any{
|
||||
first = append(first, ownedBy(r.provisionsAs(m), map[string]any{
|
||||
// One file per holder — the consumer's module with its local name after it
|
||||
// where it keeps several (ADR 0094); the lab found two files with one id.
|
||||
"id": GrantID(to, g.Consumer+"."+holderAs(g.From, g.Local)),
|
||||
"type": "file",
|
||||
"path": grantPath(m.Grants[to], g.Consumer, holderAs(g.From, g.Local)),
|
||||
"sealed": g.Sealed,
|
||||
})
|
||||
}))
|
||||
}
|
||||
}
|
||||
for _, to := range sortedKeys(m.Binds) {
|
||||
@@ -628,7 +656,16 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file, err := boundFile(*found, m.Binds[to], ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module)), own)
|
||||
as := ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module))
|
||||
// What the provider derives for THIS consumer, filled here where the consumer is
|
||||
// known (novox/hq ADR 0201). The same fill knownFor does below, so the binding file
|
||||
// and the module's `${bound:…}` substitutions cannot say different things.
|
||||
told := *found
|
||||
told.Serves, err = ServedTo(told.Serves, as)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s is told about %s: %w", m.Module, to, err)
|
||||
}
|
||||
file, err := boundFile(told, m.Binds[to], as, own)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -676,7 +713,15 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
|
||||
// Now, and not before: a module whose resources are computed replaces them wholesale, and
|
||||
// merging earlier would throw away the files it still needs.
|
||||
resources = append(append([]map[string]any{}, first...), resources...)
|
||||
//
|
||||
// **Except the module's own accounts, which go before even those** (novox/hq issue 213). What
|
||||
// the mesh computes may belong to one: a module whose code runs as an account it declares has
|
||||
// its secrets written owned by that account, and a file given to a user the machine does not
|
||||
// have yet fails — so on the first apply the secrets were refused, the process started without
|
||||
// them, and the second apply healed it, which is the fault the paragraph above describes.
|
||||
// An account depends on nothing the mesh computes.
|
||||
accounts, rest := accountsFirst(resources)
|
||||
resources = append(append(accounts, first...), rest...)
|
||||
|
||||
// No container is given the mesh's names (novox/hq ADR 0148). It used to be: every
|
||||
// container got the whole roster as `--add-host` entries at creation, and a name that
|
||||
@@ -694,7 +739,10 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
return nil, err
|
||||
}
|
||||
// And what its bindings say, for the half of a connection that is not secret.
|
||||
known := knownFor(m, r.Needs, r.Node)
|
||||
known, err := knownFor(m, r.Needs, r.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A requirement answered on this same machine is not in r.Needs — its binding file is
|
||||
// written from `here` (above) — and so `${bound:…}` could not name it, though the file
|
||||
// beside it said the same facts. Filled from the same answer, so the two cannot disagree.
|
||||
@@ -711,10 +759,24 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
}
|
||||
local := *answered
|
||||
local.For = m.Module
|
||||
for provision, values := range knownFor(m, []Needed{local}, r.Node) {
|
||||
here, err := knownFor(m, []Needed{local}, r.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for provision, values := range here {
|
||||
known[provision] = values
|
||||
}
|
||||
}
|
||||
// And the providing machine's private address, beside its name (novox/hq ADR 0194). A name is
|
||||
// what nearly every consumer wants; the one that cannot use it is a machine's resolver
|
||||
// configuration, which must reach the resolver before it can resolve anything — the resolver's
|
||||
// own name included. Absent when the machine has no address yet, so a file naming it is refused
|
||||
// rather than written with a blank where an address belongs.
|
||||
for _, values := range known {
|
||||
if address := with.Machines[values["at"]]; address != "" {
|
||||
values["address"] = address
|
||||
}
|
||||
}
|
||||
// And what the module is called through each requirement it contributes to (novox/hq
|
||||
// 04-ISSUES/122) — the same composition its binding file carries.
|
||||
for provision, values := range known {
|
||||
@@ -736,6 +798,17 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// And the machine underneath, which no binding of its own can tell it.
|
||||
thisMachine := machineFacts(r, with.Names, with.MeshRange)
|
||||
|
||||
// **A definition that already holds the answer transcribed it** (novox/hq ADR 0201).
|
||||
// Judged over what the module itself declares, and before anything is substituted: the
|
||||
// mesh's own generated files — the binding, the contributions — legitimately carry the
|
||||
// derived value, and after substitution so does every consumer's file, so this is the one
|
||||
// moment the two can be told apart.
|
||||
for _, own := range m.Resources {
|
||||
if err := notTranscribed(own, known, m.Module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Which of this module's files carry a secret, for the rule that a container may not read
|
||||
// one of them as its environment without saying so (ADR 0086, issue 041).
|
||||
secretFiles := secretFilesOf(resources)
|
||||
@@ -837,6 +910,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
return nil, err
|
||||
}
|
||||
publishedOn(copied, m.Module, with)
|
||||
// The account's environment and every module's shell code, where this module holds the
|
||||
// seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on
|
||||
// the node, as the jails are, and **last of every placeholder pass**: shell code is a
|
||||
// shell's own syntax, full of `${…}` no pass above should ever be shown.
|
||||
if err := contributionsInto(copied, m, r.Modules, thisMachine, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
|
||||
// A service saying what it reflects names resources within its own module, so those
|
||||
// are prefixed too or they would point at nothing.
|
||||
@@ -855,6 +935,30 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil {
|
||||
copied["reload-on"] = renamed
|
||||
}
|
||||
// And which of its module's containers a scheduled step holds still (novox/hq ADR 0189).
|
||||
// **The loudest of the three when it is missed.** An unprefixed `restart-on` matches
|
||||
// nothing and a service quietly never restarts; an unprefixed `while-stopped` names a
|
||||
// container the declaration does not contain, and the host refuses the whole
|
||||
// declaration — so the machine takes nothing at all, for every push, until this is
|
||||
// right. That is what it did on the control node (2026-10-04).
|
||||
if renamed := reflectsRenamed(m.Module, resource[WhileStopped]); renamed != nil {
|
||||
copied[WhileStopped] = renamed
|
||||
}
|
||||
// And what a process replaces (novox/hq issue 213): a resource of this module's that it
|
||||
// no longer declares, named as the host recorded it, or the host hands nothing over and
|
||||
// removes it first.
|
||||
if renamed := reflectsRenamed(m.Module, resource["replaces"]); renamed != nil {
|
||||
copied["replaces"] = renamed
|
||||
}
|
||||
// **What reads one of this module's own secrets is restarted when it changes** (novox/hq
|
||||
// issue 203, issue 206). A credential is re-issued by the mesh, and a container that
|
||||
// mounted the old file keeps the old one open: the build machine ran for an hour on a
|
||||
// credential the mesh had replaced, because its manifest restarted it on its
|
||||
// environment file and nobody had thought to name the credential too. Composed here so
|
||||
// no manifest has to say it, for a container or a daemon that names the secret's path.
|
||||
if reads := secretsReadBy(copied, m); len(reads) > 0 {
|
||||
copied["restart-on"] = withRestartOn(copied["restart-on"], reads)
|
||||
}
|
||||
// **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the
|
||||
// module's own resource rather than declared beside it: what prepares the state is the
|
||||
// module's own code, so what it is given has to be what that code is given — and a
|
||||
@@ -875,7 +979,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
||||
// this module's name, so they are applied, reported and removed exactly as anything else
|
||||
// it declares.
|
||||
given, err := FactsInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix)
|
||||
given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -885,6 +989,41 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
out = append(out, fact)
|
||||
}
|
||||
}
|
||||
// The node's tool runtime, last (novox/hq ADR 0175, to-be 38 WP2.3): one process loading every
|
||||
// bundle delivered above and holding the credential sealed above, so both exist before it starts
|
||||
// — the order written here is the order the machine applies.
|
||||
if r.runtimeHere() {
|
||||
process, err := r.runtimeProcess(with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owner[fmt.Sprint(process["id"])] = RuntimeModule
|
||||
out = append(out, process)
|
||||
// What each module's bundles are given is read as the account the runtime runs as.
|
||||
words := map[string]map[string]string{}
|
||||
for _, m := range r.Modules {
|
||||
w, err := bundleWords(m, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
words[m.Module] = w
|
||||
}
|
||||
// And a file a module's words name is one the runtime is restarted for when it changes.
|
||||
if named := givenTo(out, owner, words, r.Account); len(named) > 0 {
|
||||
restarts, _ := process["restart-on"].([]any)
|
||||
seen := map[string]bool{}
|
||||
for _, id := range restarts {
|
||||
seen[fmt.Sprint(id)] = true
|
||||
}
|
||||
for _, id := range named {
|
||||
if !seen[id] {
|
||||
restarts = append(restarts, id)
|
||||
seen[id] = true
|
||||
}
|
||||
}
|
||||
process["restart-on"] = restarts
|
||||
}
|
||||
}
|
||||
if with.Adopted {
|
||||
// First, before anything a module declares: what the mesh needs reachable, then its guard.
|
||||
// The order a machine applies is the order written here.
|
||||
@@ -1100,6 +1239,41 @@ type Contribution struct {
|
||||
// requirement's name — everything providing `reverse-proxy` understands the same shape, which
|
||||
// is what makes swapping one for another cost nothing.
|
||||
Values map[string]any `json:"values"`
|
||||
// Derived is what this provider's own definition said it derives for this consumer, already
|
||||
// derived (novox/hq ADR 0201).
|
||||
//
|
||||
// **The provider is told, rather than recomputing it.** A served value may name the consumer's
|
||||
// identity — a bucket named for who is asking, a database prefixed with it — and before this
|
||||
// the rule lived twice: once in the provisioner's code, once transcribed into every consumer's
|
||||
// definition. The mesh fills the provider's own statement here and delivers the same filled
|
||||
// value to the consumer, so the two cannot disagree: there is no second computation to
|
||||
// disagree with.
|
||||
//
|
||||
// Only the keys that are per-consumer. The rest of what the provider serves is the same for
|
||||
// everyone and is in its own definition, where it already is.
|
||||
Derived map[string]any `json:"derived,omitempty"`
|
||||
}
|
||||
|
||||
// provisionsAs is the account that reads what the mesh writes for this provider: the one secret
|
||||
// per consumer it must open to set that consumer's password (novox/hq issue 225).
|
||||
//
|
||||
// **A root-owned 0600 file is one that process cannot read**, which is the same sentence already
|
||||
// written above for a module's own secrets — and the grant secret is the other kind of secret
|
||||
// the mesh writes for a module, so it is the same rule.
|
||||
//
|
||||
// Which account depends on where the module's code runs. A module whose code is a bundle is run
|
||||
// by the node's tool runtime, as the node's account ([ADR 0198](0198)); one still in a container
|
||||
// is whatever it declares as its secrets owner. Nothing names these paths, so the rule that
|
||||
// claims a bundle's other files by the words that name them (givenTo) cannot reach them: the
|
||||
// harness composes a grant secret's path from the contributions file, not from a word.
|
||||
//
|
||||
// Empty is root, which is what it was and what a module with no bundle and no declared owner
|
||||
// still wants.
|
||||
func (r Resolution) provisionsAs(m Manifest) string {
|
||||
if len(m.Bundles) > 0 && r.Account != "" {
|
||||
return r.Account
|
||||
}
|
||||
return m.SecretsOwner
|
||||
}
|
||||
|
||||
// grantPath is where one consumer's sealed credential lands on the providing machine.
|
||||
@@ -1191,12 +1365,17 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
|
||||
// told about it and withdraws the login on its next pass.
|
||||
continue
|
||||
}
|
||||
as := holderAs(ConsumerIdentity(g.Consumer, IdentitySource(g.Slug, g.From)), g.Local)
|
||||
derived, err := r.derivedFor(g.Provision, as, g.From, g.Local, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[g.Provision] = append(out[g.Provision], Contribution{
|
||||
From: g.From, Node: g.Consumer, At: g.At, Values: g.Values,
|
||||
From: g.From, Node: g.Consumer, At: g.At, Values: g.Values, Derived: derived,
|
||||
// One holder per local name: the identity the consumer is known by, and the local name
|
||||
// after it where the module keeps several (ADR 0094). Not a login any backend checks —
|
||||
// a secret is not a login — so the identity limit does not apply to the suffix.
|
||||
As: holderAs(ConsumerIdentity(g.Consumer, IdentitySource(g.Slug, g.From)), g.Local),
|
||||
As: as,
|
||||
Secret: grantPath(directories[g.Provision], g.Consumer, holderAs(g.From, g.Local)),
|
||||
})
|
||||
if granted[g.Provision] == nil {
|
||||
@@ -1949,12 +2128,18 @@ func preparationTarget(m Manifest) string {
|
||||
return ""
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "container" || !ownArtifact(r, m.Module) {
|
||||
// A container, or a process the host runs from a bundle the module built (novox/hq issue
|
||||
// 213): the same program in the same context, hosted as a unit rather than a container.
|
||||
kind := fmt.Sprint(r["type"])
|
||||
if (kind != "container" && kind != "process") || !ownArtifact(r, m.Module) {
|
||||
continue
|
||||
}
|
||||
if once, _ := r["run-once"].(bool); once {
|
||||
continue
|
||||
}
|
||||
if r["schedule"] != nil {
|
||||
continue
|
||||
}
|
||||
return fmt.Sprint(r["id"])
|
||||
}
|
||||
return ""
|
||||
@@ -1968,7 +2153,10 @@ func ownArtifact(resource map[string]any, module string) bool {
|
||||
return true
|
||||
}
|
||||
image, _ := resource["image"].(string)
|
||||
return strings.HasPrefix(image, ArtifactStoreScheme+module+"/")
|
||||
// A process or an archive carries what was built as its source (novox/hq issue 213).
|
||||
source, _ := resource["source"].(string)
|
||||
return strings.HasPrefix(image, ArtifactStoreScheme+module+"/") ||
|
||||
strings.HasPrefix(source, ArtifactStoreScheme+module+"/")
|
||||
}
|
||||
|
||||
// prepared is the module's own resource as the step that prepares its state: the same image, the same
|
||||
@@ -1990,7 +2178,19 @@ func prepared(from map[string]any) map[string]any {
|
||||
step["id"] = fmt.Sprint(from["id"]) + "-prepare"
|
||||
step["name"] = fmt.Sprint(from["name"]) + "-prepare"
|
||||
step["run-once"] = true
|
||||
step["args"] = []any{PreparationArgument}
|
||||
if fmt.Sprint(from["type"]) == "process" {
|
||||
// A process says its whole command: the program, then its arguments. The step is the same
|
||||
// program asked to prepare (novox/hq issue 213). It replaces nothing — what the process
|
||||
// replaces is handed over to the process, never to the step that runs before it — and a
|
||||
// step is not restarted, it runs again when what it reads changed, which `restart-on` says.
|
||||
run := stringsIn(from["run"])
|
||||
if len(run) > 0 {
|
||||
step["run"] = []any{run[0], PreparationArgument}
|
||||
}
|
||||
delete(step, "replaces")
|
||||
} else {
|
||||
step["args"] = []any{PreparationArgument}
|
||||
}
|
||||
delete(step, "ports")
|
||||
delete(step, "ip")
|
||||
delete(step, "schedule")
|
||||
@@ -2051,3 +2251,100 @@ func portOfEndpoint(values map[string]any, ports map[string]int) {
|
||||
values["port"] = port
|
||||
}
|
||||
}
|
||||
|
||||
// secretsReadBy is the file resources of this module's own secrets that a container or a daemon reads
|
||||
// — named in its volumes, its environment or its env-files by the secret's placed path — as
|
||||
// restart-on ids. Nothing for other shapes, and nothing for a scheduled or run-once process, which
|
||||
// the host refuses a restart-on for (it runs again anyway, and reads the file afresh).
|
||||
func secretsReadBy(resource map[string]any, m Manifest) []string {
|
||||
kind := fmt.Sprint(resource["type"])
|
||||
if kind != "container" && kind != "process" {
|
||||
return nil
|
||||
}
|
||||
if resource["schedule"] != nil || resource["run-once"] == true {
|
||||
return nil
|
||||
}
|
||||
var mentioned []string
|
||||
for _, key := range []string{"volumes", "env", "env-file"} {
|
||||
mentioned = append(mentioned, stringsIn(resource[key])...)
|
||||
}
|
||||
var out []string
|
||||
for _, name := range sortedKeys(m.OwnSecrets) {
|
||||
path := m.OwnSecrets[name].Path
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
for _, s := range mentioned {
|
||||
// A volume is `source:destination[:mode]`; an env value or an env-file is the path itself.
|
||||
if s == path || strings.HasPrefix(s, path+":") {
|
||||
out = append(out, m.Module+"."+NeedID(name))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stringsIn is every string in a list or a map's values; nothing for anything else.
|
||||
func stringsIn(v any) []string {
|
||||
switch x := v.(type) {
|
||||
case []any:
|
||||
var out []string
|
||||
for _, item := range x {
|
||||
if s, ok := item.(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
return x
|
||||
case map[string]any:
|
||||
var out []string
|
||||
for _, k := range sortedKeys(x) {
|
||||
if s, ok := x[k].(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case map[string]string:
|
||||
var out []string
|
||||
for _, k := range sortedKeys(x) {
|
||||
out = append(out, x[k])
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// withRestartOn is a resource's restart-on list with these ids added once each.
|
||||
func withRestartOn(have any, add []string) []any {
|
||||
var out []any
|
||||
seen := map[string]bool{}
|
||||
for _, id := range reflectsRenamed("", have) {
|
||||
s := fmt.Sprint(id)
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
for _, id := range add {
|
||||
if !seen[id] {
|
||||
seen[id] = true
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// accountsFirst splits a module's resources into its accounts and everything else, each in the order
|
||||
// written.
|
||||
func accountsFirst(resources []map[string]any) (accounts, rest []map[string]any) {
|
||||
for _, r := range resources {
|
||||
if fmt.Sprint(r["type"]) == "user" {
|
||||
accounts = append(accounts, r)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, r)
|
||||
}
|
||||
return accounts, rest
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user