Compare commits
127
Commits
e11caecdad
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9cf47f4429 | ||
|
|
8ddc019cd2 | ||
|
|
fab6b0059e | ||
|
|
e1f5d4fdf0 | ||
|
|
bb1607e424 | ||
|
|
cf4834a36c | ||
|
|
9d8cbe7b81 | ||
|
|
e74c32ed50 | ||
|
|
146c48fd96 | ||
|
|
1f3abd3e0e | ||
|
|
6d620f77c3 | ||
|
|
f8286c063d | ||
|
|
e096b4595a | ||
|
|
09c0c6b367 | ||
|
|
d1fc25f682 | ||
|
|
eda457f415 | ||
|
|
59f4d486b1 | ||
|
|
801552c0eb | ||
|
|
ede9bce6ef | ||
|
|
0f0028785c | ||
|
|
6fdcfad8d3 | ||
|
|
8d9d33ae85 | ||
|
|
cc25baa563 | ||
|
|
68a2ebdcc3 | ||
|
|
69b99eec68 | ||
|
|
09bd0eec4f | ||
|
|
222a38e050 | ||
|
|
ee99a24f77 | ||
|
|
f68521da28 | ||
|
|
296064c799 | ||
|
|
df9231c734 | ||
|
|
843b709b59 | ||
|
|
f506fb34ec | ||
|
|
c34b937dd3 | ||
|
|
d84c9699b3 | ||
|
|
002d5e578c | ||
|
|
2421b82ad2 | ||
|
|
0ebd48a6a8 | ||
|
|
67e291c02a | ||
|
|
8a400d165e | ||
|
|
53d3cd7ce9 | ||
|
|
6648e4a5c8 | ||
|
|
7fa2568ce7 | ||
|
|
4b382ceffd | ||
|
|
ce86d09d22 | ||
|
|
853be00ebe | ||
|
|
e0ce2236dd | ||
|
|
9873b3bf13 | ||
|
|
541603c15c | ||
|
|
106507b1d3 | ||
|
|
e610f2d92c | ||
|
|
f80b6cdbd1 | ||
|
|
5dcf33db45 | ||
|
|
6abce7e956 | ||
|
|
0d2b304f08 | ||
|
|
bdfbd0254f | ||
|
|
16c5e78fa8 | ||
|
|
ed90771382 | ||
|
|
672d1f4ca1 | ||
|
|
208901c6cc | ||
|
|
4ac5cfe3a7 | ||
|
|
22660dc274 | ||
|
|
d7f359c498 | ||
|
|
ed25fd68e2 | ||
|
|
01c5ab2aab | ||
|
|
bb3cd6437b | ||
|
|
0b07e68cb8 | ||
|
|
625d02862c | ||
|
|
e8478e208b | ||
|
|
5761737687 | ||
|
|
89ec48b9a9 | ||
|
|
869fb6d6bf | ||
|
|
d25b69178b | ||
|
|
a7bb1e0b1c | ||
|
|
5eaed84271 | ||
|
|
326b1aec14 | ||
|
|
134d039ff8 | ||
|
|
d90c6ab93a | ||
|
|
6b7d2ec49b | ||
|
|
4ed1057df3 | ||
|
|
1f4c67a01b | ||
|
|
5474ea2d41 | ||
|
|
b7912172af | ||
|
|
b36babcd7d | ||
|
|
49cf0aa562 | ||
|
|
5a4f73f761 | ||
|
|
6af891e358 | ||
|
|
568f55fd2e | ||
|
|
35314175f2 | ||
|
|
ec2e6255a9 | ||
|
|
f3f34a170e | ||
|
|
e2622fd031 | ||
|
|
3ee32970ef | ||
|
|
11b654499b | ||
|
|
d69e19103c | ||
|
|
10f948e970 | ||
|
|
f421d4588c | ||
|
|
74b0dab34c | ||
|
|
41b20b2782 | ||
|
|
912e9f4e85 | ||
|
|
babd7b2f47 | ||
|
|
892dfd1d08 | ||
|
|
cfac579392 | ||
|
|
fe0d295490 | ||
|
|
d8a0238e02 | ||
|
|
dcf710a8d5 | ||
|
|
a2003ab616 | ||
|
|
1363a2fe27 | ||
|
|
f19a2254ac | ||
|
|
7d46e48b26 | ||
|
|
78915f9f7a | ||
|
|
17b8f14fe1 | ||
|
|
42c394acc2 | ||
|
|
b9ad7a2948 | ||
|
|
cde22ff627 | ||
|
|
79993fb498 | ||
|
|
aec55b7072 | ||
|
|
c7884f5a72 | ||
|
|
580c4d66a7 | ||
|
|
63bfc5fda5 | ||
|
|
02e3482eb5 | ||
|
|
294e83dab1 | ||
|
|
b5438bb331 | ||
|
|
c23be73d4d | ||
|
|
d2d171f2d2 | ||
|
|
73fa64ea68 | ||
|
|
1a13dbeb17 |
+13
-5
@@ -1,5 +1,11 @@
|
|||||||
ARG GO_BASE=golang:1.25-alpine
|
# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq
|
||||||
# The control plane's image.
|
# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how
|
||||||
|
# `make image` broke once before (issue 146).
|
||||||
|
ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||||
|
# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go
|
||||||
|
# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it.
|
||||||
|
# Genesis builds this file and raises it as the container the process replaces on the first push
|
||||||
|
# (mesh-host internal/bootstrap, novox/hq issue 223).
|
||||||
#
|
#
|
||||||
# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
|
# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
|
||||||
# machine where no mesh exists yet, and run before there is anything to check it against. So it
|
# machine where no mesh exists yet, and run before there is anything to check it against. So it
|
||||||
@@ -15,13 +21,15 @@ ARG GO_BASE=golang:1.25-alpine
|
|||||||
FROM ${GO_BASE} AS build
|
FROM ${GO_BASE} AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
# Dependencies first, so a change to the source does not refetch them.
|
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
|
||||||
|
# the image builds from this repository alone — the host's validator among them, whose module no
|
||||||
|
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
COPY vendor/ vendor/
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
ARG VERSION=development
|
ARG VERSION=development
|
||||||
RUN CGO_ENABLED=0 go build -trimpath \
|
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||||
-o /mesh-controller ./cmd/mesh-controller
|
-o /mesh-controller ./cmd/mesh-controller
|
||||||
|
|
||||||
|
|||||||
@@ -27,17 +27,21 @@ build:
|
|||||||
IMAGE ?= mesh-controller:$(VERSION)
|
IMAGE ?= mesh-controller:$(VERSION)
|
||||||
DEV_TAG ?= mesh-controller:development
|
DEV_TAG ?= mesh-controller:development
|
||||||
|
|
||||||
# The base the module declares, read from the manifest rather than written here twice.
|
# The Go base the image is built on.
|
||||||
#
|
#
|
||||||
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
|
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
|
||||||
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
|
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
|
||||||
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
|
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
|
||||||
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
|
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
|
||||||
# what it cost).
|
# what it cost).
|
||||||
GO_BASE ?= $(shell python3 -c "import json;print(next(o['image'] for o in json.load(open('module.json'))['build']['on'] if o['arg']=='GO_BASE'))" 2>/dev/null)
|
#
|
||||||
|
# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds
|
||||||
|
# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab —
|
||||||
|
# still needs a Go base. The digest is the one the manifest declared until then.
|
||||||
|
GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||||
|
|
||||||
image:
|
image:
|
||||||
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
|
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||||
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
||||||
@echo
|
@echo
|
||||||
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
@@ -48,7 +52,7 @@ BUILDER_IMAGE ?= mesh-builder:$(VERSION)
|
|||||||
BUILDER_DEV_TAG ?= mesh-builder:development
|
BUILDER_DEV_TAG ?= mesh-builder:development
|
||||||
|
|
||||||
builder-image:
|
builder-image:
|
||||||
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
|
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||||
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
||||||
@echo
|
@echo
|
||||||
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|||||||
@@ -193,6 +193,13 @@ passes every check that only looks at the message.
|
|||||||
|
|
||||||
## The image
|
## The image
|
||||||
|
|
||||||
|
**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go
|
||||||
|
bundle, `controller`, which the host on the controller's machine unpacks and runs as the process
|
||||||
|
`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what
|
||||||
|
still runs a container of the controller: genesis, which raises the first controller from it and
|
||||||
|
installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the
|
||||||
|
mesh's own build any more — `make image` builds it.
|
||||||
|
|
||||||
`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
|
`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
|
||||||
manager, no libc, no CA certificates.
|
manager, no libc, no CA certificates.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,386 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/micro"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
|
||||||
|
//
|
||||||
|
// **The queue is the controller's; the build running here is this machine's.** The controller can
|
||||||
|
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
|
||||||
|
// on this machine and containers in this machine's runtime, and only this machine can end them. So
|
||||||
|
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
|
||||||
|
// it, pause, resume.
|
||||||
|
//
|
||||||
|
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
|
||||||
|
// set per build — so "the build running here" is one or none, and kill names it by id so a call
|
||||||
|
// that arrives as one build ends and the next begins cannot end the wrong one.
|
||||||
|
|
||||||
|
// holder is this machine's state as a holder of the build seat.
|
||||||
|
type holder struct {
|
||||||
|
on, seat string
|
||||||
|
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
|
||||||
|
// rather than silently taking work again.
|
||||||
|
workspace string
|
||||||
|
// say publishes this machine's state: whether it takes work (link.HolderState).
|
||||||
|
say func(link.HolderState) error
|
||||||
|
// remove runs what a kill needs outside the build: the containers left behind.
|
||||||
|
remove builder.Runner
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
paused bool
|
||||||
|
running *running
|
||||||
|
}
|
||||||
|
|
||||||
|
// running is the build this machine is doing.
|
||||||
|
type running struct {
|
||||||
|
request link.BuildRequest
|
||||||
|
step string
|
||||||
|
started time.Time
|
||||||
|
cancel context.CancelFunc
|
||||||
|
killed bool
|
||||||
|
// returned is the build's own work having ended, before a kill or not; outcome is what was then
|
||||||
|
// announced, and announced whether it went out.
|
||||||
|
returned bool
|
||||||
|
outcome string
|
||||||
|
announced bool
|
||||||
|
done chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pausedFile is where the flag lives in the workspace.
|
||||||
|
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
|
||||||
|
|
||||||
|
// newHolder reads the paused flag the workspace keeps.
|
||||||
|
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
|
||||||
|
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
|
||||||
|
if _, err := os.Stat(pausedFile(workspace)); err == nil {
|
||||||
|
h.paused = true
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// Paused is asked by the taking loop before every fetch.
|
||||||
|
func (h *holder) Paused() bool {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return h.paused
|
||||||
|
}
|
||||||
|
|
||||||
|
// setPaused records the flag in the workspace first and then in memory, so what this holder says
|
||||||
|
// it is and what it would be after a restart never differ.
|
||||||
|
func (h *holder) setPaused(paused bool) error {
|
||||||
|
path := pausedFile(h.workspace)
|
||||||
|
if paused {
|
||||||
|
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||||
|
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
|
||||||
|
}
|
||||||
|
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
h.paused = paused
|
||||||
|
h.mu.Unlock()
|
||||||
|
h.announce()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
|
||||||
|
// controller reading an older state is a plan read as late rather than a build lost.
|
||||||
|
func (h *holder) announce() {
|
||||||
|
if h.say == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
|
||||||
|
// a pause outlives the events stream's retention.
|
||||||
|
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||||
|
h.announce()
|
||||||
|
tick := time.NewTicker(every)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
if h.Paused() {
|
||||||
|
h.announce()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// begin records the build this machine took, with the cancel that ends it.
|
||||||
|
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||||
|
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||||
|
h.mu.Lock()
|
||||||
|
h.running = r
|
||||||
|
h.mu.Unlock()
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// end clears it, and lets a kill waiting on it know it is over.
|
||||||
|
func (h *holder) end(r *running) {
|
||||||
|
h.mu.Lock()
|
||||||
|
if h.running == r {
|
||||||
|
h.running = nil
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
close(r.done)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepped records the step a build is at, for `current`.
|
||||||
|
func (h *holder) stepped(r *running, step string) {
|
||||||
|
h.mu.Lock()
|
||||||
|
r.step = step
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// currentBuild is what `current` answers.
|
||||||
|
type currentBuild struct {
|
||||||
|
On string `json:"on"`
|
||||||
|
Paused bool `json:"paused"`
|
||||||
|
Running *struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
Step string `json:"step,omitempty"`
|
||||||
|
Started string `json:"started"`
|
||||||
|
Elapsed string `json:"elapsed"`
|
||||||
|
} `json:"running,omitempty"`
|
||||||
|
Said string `json:"said"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *holder) current() currentBuild {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
out := currentBuild{On: h.on, Paused: h.paused}
|
||||||
|
taking := "taking builds"
|
||||||
|
if h.paused {
|
||||||
|
taking = "paused, taking no new build"
|
||||||
|
}
|
||||||
|
if h.running == nil {
|
||||||
|
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
r := h.running
|
||||||
|
elapsed := time.Since(r.started).Round(time.Second)
|
||||||
|
out.Running = &struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
Step string `json:"step,omitempty"`
|
||||||
|
Started string `json:"started"`
|
||||||
|
Elapsed string `json:"elapsed"`
|
||||||
|
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
|
||||||
|
r.started.UTC().Format(time.RFC3339), elapsed.String()}
|
||||||
|
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
|
||||||
|
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
|
||||||
|
// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
|
||||||
|
// (killAnswer in the controller's queue.go, 75s).
|
||||||
|
var (
|
||||||
|
killRemoves = 15 * time.Second
|
||||||
|
killWaits = 20 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// kill ends the build with this id, if it is the one running here and still working: its context
|
||||||
|
// cancelled — which kills each command's process group — and the containers it started removed by
|
||||||
|
// their label, once at once and again after the build has ended, so one created while it was being
|
||||||
|
// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
|
||||||
|
// ask so it is not redelivered; the answer says whether that happened.
|
||||||
|
func (h *holder) kill(id string) (string, error) {
|
||||||
|
h.mu.Lock()
|
||||||
|
r := h.running
|
||||||
|
if r == nil || r.request.ID != id {
|
||||||
|
doing := "nothing"
|
||||||
|
if r != nil {
|
||||||
|
doing = r.request.ID
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
|
||||||
|
}
|
||||||
|
if r.returned {
|
||||||
|
h.mu.Unlock()
|
||||||
|
return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
|
||||||
|
}
|
||||||
|
r.killed = true
|
||||||
|
cancel, done := r.cancel, r.done
|
||||||
|
h.mu.Unlock()
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
removed, removeErr := h.removeContainers(id)
|
||||||
|
ended := false
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
ended = true
|
||||||
|
case <-time.After(killWaits):
|
||||||
|
}
|
||||||
|
again, againErr := h.removeContainers(id)
|
||||||
|
removed += again
|
||||||
|
if removeErr == nil {
|
||||||
|
removeErr = againErr
|
||||||
|
}
|
||||||
|
containers := fmt.Sprintf("%d container(s) it started removed", removed)
|
||||||
|
if removeErr != nil {
|
||||||
|
containers = "its containers could not all be listed or removed: " + removeErr.Error()
|
||||||
|
}
|
||||||
|
if !ended {
|
||||||
|
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
|
||||||
|
"its outcome is in", id, h.on, containers), nil
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
outcome, announced := r.outcome, r.announced
|
||||||
|
h.mu.Unlock()
|
||||||
|
if !announced {
|
||||||
|
return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
|
||||||
|
"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
|
||||||
|
containers), nil
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
|
||||||
|
"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeContainers is one pass of removing what the build left, bounded.
|
||||||
|
func (h *holder) removeContainers(id string) (int, error) {
|
||||||
|
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
||||||
|
defer stop()
|
||||||
|
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// returned records that the build's work ended, and says whether a kill came first — only then is
|
||||||
|
// the build killed; an error it ended with on its own is its own outcome.
|
||||||
|
func (h *holder) returned(r *running) bool {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
r.returned = true
|
||||||
|
return r.killed
|
||||||
|
}
|
||||||
|
|
||||||
|
// said records the outcome announced, and whether it went out, for a kill to answer with.
|
||||||
|
func (h *holder) said(r *running, outcome string, announced bool) {
|
||||||
|
h.mu.Lock()
|
||||||
|
r.outcome, r.announced = outcome, announced
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlers are the seat's verbs, as this machine answers them.
|
||||||
|
func (h *holder) handlers() map[string]link.ToolHandler {
|
||||||
|
return map[string]link.ToolHandler{
|
||||||
|
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
|
||||||
|
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||||
|
var args struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
|
||||||
|
return nil, errors.New("kill needs the build's id")
|
||||||
|
}
|
||||||
|
said, err := h.kill(strings.TrimSpace(args.ID))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"said": said}, nil
|
||||||
|
},
|
||||||
|
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
if err := h.setPaused(true); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
said := h.on + " is paused: it takes no new build until resumed"
|
||||||
|
if c := h.current(); c.Running != nil {
|
||||||
|
said += "; " + c.Running.ID + " runs on and finishes"
|
||||||
|
}
|
||||||
|
return map[string]any{"said": said, "paused": true}, nil
|
||||||
|
},
|
||||||
|
"resume": func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
if err := h.setPaused(false); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNothing(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "its start"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
|
||||||
|
// log it would be is the one being ended.
|
||||||
|
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||||
|
cmd := exec.CommandContext(ctx, name, args...)
|
||||||
|
cmd.Dir = dir
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
return string(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
|
||||||
|
// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
|
||||||
|
// the module answering, the seat, scope node, the machine, its description and argument schema — so
|
||||||
|
// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
|
||||||
|
//
|
||||||
|
// One service per machine, named for the seat and identified by the machine, so the answer is this
|
||||||
|
// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
|
||||||
|
// store, and what it serves is what this binary was built to serve.
|
||||||
|
func announcement(seat, module, node string) micro.Info {
|
||||||
|
s, _ := catalogue.SeatNamed(seat)
|
||||||
|
var endpoints []micro.EndpointInfo
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
schema, _ := json.Marshal(v.Input)
|
||||||
|
endpoints = append(endpoints, micro.EndpointInfo{
|
||||||
|
Name: seat + "__" + v.Name,
|
||||||
|
Subject: link.NodeSeatToolSubject(seat, v.Name, node),
|
||||||
|
// The queue group the verbs are served in, as every runtime announces its own.
|
||||||
|
QueueGroup: "seat." + seat,
|
||||||
|
Metadata: map[string]string{
|
||||||
|
"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
|
||||||
|
"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return micro.Info{
|
||||||
|
ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
|
||||||
|
Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
|
||||||
|
Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
|
||||||
|
Endpoints: endpoints,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
|
||||||
|
func moduleOf(user string) string {
|
||||||
|
if _, module, ok := strings.Cut(user, "."); ok && module != "" {
|
||||||
|
return module
|
||||||
|
}
|
||||||
|
return "build-agent"
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
||||||
|
// 0219), and what it says about itself follows.
|
||||||
|
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
var said []link.HolderState
|
||||||
|
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
||||||
|
said = append(said, s)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if h.Paused() {
|
||||||
|
t.Fatal("a new holder starts paused")
|
||||||
|
}
|
||||||
|
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
||||||
|
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
||||||
|
}
|
||||||
|
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
||||||
|
if !again.Paused() {
|
||||||
|
t.Fatal("restarted, the holder forgot it was paused")
|
||||||
|
}
|
||||||
|
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
||||||
|
t.Fatal("resumed, the holder came back paused")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
||||||
|
// label, and refuses an id it is not building.
|
||||||
|
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||||
|
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||||
|
var removed []string
|
||||||
|
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||||
|
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||||
|
if args[0] == "ps" {
|
||||||
|
return "c1\n", nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
kill := h.handlers()["kill"]
|
||||||
|
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
||||||
|
t.Fatal("killed a build while none ran")
|
||||||
|
}
|
||||||
|
|
||||||
|
building, cancel := context.WithCancel(context.Background())
|
||||||
|
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
||||||
|
h.stepped(r, "image")
|
||||||
|
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
||||||
|
t.Fatalf("current says %+v", c)
|
||||||
|
}
|
||||||
|
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "build-1") {
|
||||||
|
t.Fatalf("killed another id: %v", err)
|
||||||
|
}
|
||||||
|
// The build's own goroutine: it ends when its context does, as a build's commands do, and
|
||||||
|
// announces what came of it.
|
||||||
|
go func() {
|
||||||
|
<-building.Done()
|
||||||
|
if h.returned(r) {
|
||||||
|
h.said(r, link.KilledByHand, true)
|
||||||
|
}
|
||||||
|
h.end(r)
|
||||||
|
}()
|
||||||
|
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if building.Err() == nil {
|
||||||
|
t.Fatal("the build's context was not cancelled")
|
||||||
|
}
|
||||||
|
if !r.killed {
|
||||||
|
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
||||||
|
}
|
||||||
|
said := answer.(map[string]any)["said"].(string)
|
||||||
|
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
|
||||||
|
t.Errorf("kill said %q", said)
|
||||||
|
}
|
||||||
|
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
||||||
|
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
||||||
|
t.Errorf("removed %v", removed)
|
||||||
|
}
|
||||||
|
if c := h.current(); c.Running != nil {
|
||||||
|
t.Errorf("after the kill current says %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
|
||||||
|
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
|
||||||
|
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
|
||||||
|
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
|
||||||
|
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
|
||||||
|
}
|
||||||
|
kill := info.Endpoints[1]
|
||||||
|
md := kill.Metadata
|
||||||
|
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
|
||||||
|
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
|
||||||
|
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
|
||||||
|
t.Fatalf("kill is announced as %+v", kill)
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(info)
|
||||||
|
if err != nil || len(body) > 8*1024 {
|
||||||
|
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
|
||||||
|
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
|
||||||
|
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
|
||||||
|
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||||
|
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
|
||||||
|
_, cancel := context.WithCancel(context.Background())
|
||||||
|
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
|
||||||
|
if killed := h.returned(r); killed {
|
||||||
|
t.Fatal("a build nobody killed reads as killed")
|
||||||
|
}
|
||||||
|
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
|
||||||
|
t.Fatalf("killed a build that had ended: %v", err)
|
||||||
|
}
|
||||||
|
h.end(r)
|
||||||
|
|
||||||
|
building, cancel := context.WithCancel(context.Background())
|
||||||
|
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
|
||||||
|
go func() {
|
||||||
|
<-building.Done()
|
||||||
|
if h.returned(r) {
|
||||||
|
h.said(r, link.KilledByHand, false)
|
||||||
|
}
|
||||||
|
h.end(r)
|
||||||
|
}()
|
||||||
|
said, err := h.kill("build-2")
|
||||||
|
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
|
||||||
|
t.Fatalf("kill said %q (%v)", said, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+92
-12
@@ -19,11 +19,13 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
@@ -107,48 +109,96 @@ func run() error {
|
|||||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
machine, err := takeWorkFrom(credential, on)
|
js, seat, err := dialFor(credential)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer js.Close()
|
||||||
|
|
||||||
|
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
|
||||||
|
// paused flag is read from the workspace before anything is taken, so a holder restarted while
|
||||||
|
// paused takes nothing.
|
||||||
|
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
|
||||||
|
body, err := json.Marshal(state)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if h.Paused() {
|
||||||
|
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||||
|
}
|
||||||
|
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||||
defer machine.Close()
|
defer machine.Close()
|
||||||
|
|
||||||
|
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||||
|
// one serves none, and a subscription its holder has no grant for would be refused for ever.
|
||||||
|
if seat == link.TheBuildMachine {
|
||||||
|
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
|
||||||
|
log.New(os.Stderr, "", 0))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer stopServing()
|
||||||
|
// And says so, for the console to find (novox/hq ADR 0197).
|
||||||
|
stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
|
||||||
|
announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer stopAnnouncing()
|
||||||
|
go h.stateWhilePaused(ctx, time.Hour)
|
||||||
|
}
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||||
|
|
||||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||||
answer(ctx, publisher, on, workspace, work)
|
answer(ctx, publisher, on, workspace, work, h)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
|
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
|
||||||
|
// holds.
|
||||||
//
|
//
|
||||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||||
// machine told about both would take work from one and answer on the other, and every log line would
|
// machine told about both would take work from one and answer on the other, and every log line would
|
||||||
// say it was fine.
|
// say it was fine.
|
||||||
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
|
func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||||
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
||||||
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
||||||
// and that is enough to dial it, pinned.
|
// and that is enough to dial it, pinned.
|
||||||
if !credential.onTheNewBus() {
|
if !credential.onTheNewBus() {
|
||||||
return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||||
}
|
}
|
||||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
||||||
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
||||||
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
||||||
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
||||||
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
||||||
return link.MachineOverNATSOn(js, on, seat), nil
|
return js, seat, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// answer does one build and says what happened, whichever way it went.
|
// answer does one build and says what happened, whichever way it went.
|
||||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
|
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
|
||||||
request := work.Request()
|
request := work.Request()
|
||||||
|
|
||||||
|
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
|
||||||
|
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
|
||||||
|
// reaches it through the parent, and that keeps today's meaning below.
|
||||||
|
building, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
var mine *running
|
||||||
|
if h != nil {
|
||||||
|
mine = h.begin(request, cancel)
|
||||||
|
defer h.end(mine)
|
||||||
|
}
|
||||||
|
|
||||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||||
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||||
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||||
@@ -162,6 +212,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
say := func(step, message string) {
|
say := func(step, message string) {
|
||||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||||
work.Say(step, message)
|
work.Say(step, message)
|
||||||
|
if mine != nil && step != "run" && step != "output" {
|
||||||
|
h.stepped(mine, step)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
builder.Said = say
|
builder.Said = say
|
||||||
defer func() { builder.Said = nil }()
|
defer func() { builder.Said = nil }()
|
||||||
@@ -171,7 +224,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
|
|
||||||
result := link.BuildResult{
|
result := link.BuildResult{
|
||||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||||
Ref: request.Ref, On: on, Source: request.Source,
|
Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
|
||||||
}
|
}
|
||||||
what := "building " + request.Repository
|
what := "building " + request.Repository
|
||||||
if request.Path != "" {
|
if request.Path != "" {
|
||||||
@@ -188,11 +241,24 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||||
// after a clone that then fails at npm ci.
|
// after a clone that then fails at npm ci.
|
||||||
built, err = builder.Build(ctx, builder.Command, publisher,
|
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||||
|
// docker client (ADR 0219).
|
||||||
|
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||||
forgeFrom(), say, request.Seats)
|
forgeFrom(), say, request.Seats)
|
||||||
}
|
}
|
||||||
if err != nil {
|
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
||||||
|
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
||||||
|
killed := false
|
||||||
|
if mine != nil {
|
||||||
|
killed = h.returned(mine) && err != nil
|
||||||
|
}
|
||||||
|
if killed {
|
||||||
|
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||||
|
// error it ended with is the kill's consequence, not a fault of the source.
|
||||||
|
result.Failed = link.KilledByHand
|
||||||
|
say("failed", link.KilledByHand)
|
||||||
|
} else if err != nil {
|
||||||
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
||||||
// builder that is not running, and those want completely different responses.
|
// builder that is not running, and those want completely different responses.
|
||||||
result.Failed = err.Error()
|
result.Failed = err.Error()
|
||||||
@@ -217,7 +283,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := work.Announce(ctx, result); err != nil {
|
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
|
||||||
|
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
|
||||||
|
// to be built again. A machine being stopped is the other case and keeps its meaning: the
|
||||||
|
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
|
||||||
|
announcing := ctx
|
||||||
|
if killed {
|
||||||
|
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer stop()
|
||||||
|
announcing = fresh
|
||||||
|
}
|
||||||
|
announceErr := work.Announce(announcing, result)
|
||||||
|
if mine != nil {
|
||||||
|
h.said(mine, result.Failed, announceErr == nil)
|
||||||
|
}
|
||||||
|
if err := announceErr; err != nil {
|
||||||
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||||
// failed is a lie about work that was done — and the request stays unsettled below only if
|
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||||
// nothing was said at all, so another machine can try.
|
// nothing was said at all, so another machine can try.
|
||||||
|
|||||||
+147
-22
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -39,7 +40,10 @@ import (
|
|||||||
//
|
//
|
||||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||||
// machines this just blocked is worth more than the milliseconds.
|
// machines this just blocked is worth more than the milliseconds.
|
||||||
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
|
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||||
|
if len(modules) == 0 {
|
||||||
|
return "", fmt.Errorf("assign %s names no module", node)
|
||||||
|
}
|
||||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||||
@@ -47,6 +51,16 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
|
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
||||||
|
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
||||||
|
// resources are applied through a seat nothing on the node holds is refused, because that order
|
||||||
|
// — the service manager, the package manager and the runtime before anything that installs,
|
||||||
|
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
|
||||||
|
// holders that depend on each other go on in one command.
|
||||||
|
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
||||||
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
||||||
// assignment resolves against a record rather than against a coincidence.
|
// assignment resolves against a record rather than against a coincidence.
|
||||||
@@ -54,65 +68,176 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
var lines []string
|
||||||
if err != nil {
|
var added []string
|
||||||
return "", err
|
for _, module := range modules {
|
||||||
|
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||||
|
if err != nil {
|
||||||
|
return strings.Join(lines, "\n"), err
|
||||||
|
}
|
||||||
|
if !fresh {
|
||||||
|
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||||
|
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||||
|
lines = append(lines, fmt.Sprintf(
|
||||||
|
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
added = append(added, module)
|
||||||
|
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
|
||||||
}
|
}
|
||||||
if !fresh {
|
if len(added) == 0 {
|
||||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
return strings.Join(lines, "\n"), nil
|
||||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
|
||||||
return fmt.Sprintf("%s already runs %s — one node runs one of each (ADR 0115); nothing changed",
|
|
||||||
node, module), nil
|
|
||||||
}
|
}
|
||||||
said := fmt.Sprintf("%s is assigned %s", node, module)
|
answer := strings.Join(lines, "\n")
|
||||||
for _, line := range settled {
|
for _, line := range settled {
|
||||||
said += "\n " + line
|
answer += "\n " + line
|
||||||
|
}
|
||||||
|
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||||
|
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||||
|
// node's list, and every other node's, is `status`'s.
|
||||||
|
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||||
|
answer += "\n " + line
|
||||||
}
|
}
|
||||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
for _, module := range added {
|
||||||
said += "\n " + line
|
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||||
|
answer += "\n " + line
|
||||||
|
}
|
||||||
}
|
}
|
||||||
plan, _, err := planFor(ctx, open, node)
|
plan, _, err := planFor(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||||
return said + blockedElsewhere(ctx, open, node), err
|
return answer + blockedElsewhere(ctx, open, node), err
|
||||||
}
|
}
|
||||||
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
||||||
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
||||||
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
||||||
|
isAdded := map[string]bool{}
|
||||||
|
for _, m := range added {
|
||||||
|
isAdded[m] = true
|
||||||
|
}
|
||||||
for _, u := range plan.Unhostable {
|
for _, u := range plan.Unhostable {
|
||||||
if u.Module != module {
|
if !isAdded[u.Module] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, c := range u.Missing {
|
for _, c := range u.Missing {
|
||||||
said += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return said + fmt.Sprintf("\n run `push %s` to send it", node) +
|
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||||
blockedElsewhere(ctx, open, node), nil
|
blockedElsewhere(ctx, open, node), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// unassign takes a module off a node. What it leaves behind is the host's business: a directory
|
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
||||||
|
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
|
||||||
|
// and are not judged again.
|
||||||
|
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
|
||||||
|
map[string]catalogue.Manifest, []string, error) {
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
assigned, err := open.inventory.Assigned(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
already := map[string]bool{}
|
||||||
|
for _, a := range assigned {
|
||||||
|
already[a] = true
|
||||||
|
}
|
||||||
|
var adding []string
|
||||||
|
for _, m := range modules {
|
||||||
|
if !already[m] {
|
||||||
|
adding = append(adding, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
|
||||||
|
// with everything else this act changed, so they are not said twice.
|
||||||
|
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
// Two modules declaring one package, path or unit is refused before anything is recorded
|
||||||
|
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
||||||
|
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return shelf, assigned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// unassign takes modules off a node. What they leave behind is the host's business: a directory
|
||||||
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
||||||
//
|
//
|
||||||
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
||||||
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
||||||
// about the command's own output would ever have said so.
|
// about the command's own output would ever have said so.
|
||||||
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
|
//
|
||||||
|
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
|
||||||
|
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||||
|
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||||
|
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||||
|
if len(modules) == 0 {
|
||||||
|
return "", fmt.Errorf("unassign %s names no module", node)
|
||||||
|
}
|
||||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
assigned, err := open.inventory.Assigned(ctx, node)
|
||||||
node, module, node) + blockedElsewhere(ctx, open, node), nil
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
// Every one checked before any is taken off, so a refusal leaves the node as it was.
|
||||||
|
runs := map[string]bool{}
|
||||||
|
for _, a := range assigned {
|
||||||
|
runs[a] = true
|
||||||
|
}
|
||||||
|
for _, module := range modules {
|
||||||
|
if !runs[module] {
|
||||||
|
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
for _, module := range modules {
|
||||||
|
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||||
|
node, strings.Join(modules, ", "), node)
|
||||||
|
var left []string
|
||||||
|
for _, a := range assigned {
|
||||||
|
if !slices.Contains(modules, a) {
|
||||||
|
left = append(left, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||||
|
answer += "\n " + line
|
||||||
|
}
|
||||||
|
return answer + blockedElsewhere(ctx, open, node), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||||
|
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||||
|
// command API and the controller seat's verbs as they do from the command line.
|
||||||
|
func splitModules(field string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, m := range strings.Split(field, ",") {
|
||||||
|
if m = strings.TrimSpace(m); m != "" {
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
||||||
|
|||||||
@@ -111,6 +111,14 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
|||||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||||
|
// asks where this machine reaches the store, as the docker module does.
|
||||||
|
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||||
|
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||||
|
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
on := map[string]bool{"anchor": true, "laptop": true}
|
on := map[string]bool{"anchor": true, "laptop": true}
|
||||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||||
@@ -145,7 +153,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
|||||||
//
|
//
|
||||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||||
// container is told so beside the sealed connection genesis wrote.
|
// process is told so beside the sealed connection genesis wrote.
|
||||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -157,8 +165,8 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
|
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||||
Reference: "registry.example/control@" + aDigest}})
|
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -200,12 +208,12 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
|||||||
|
|
||||||
var env map[string]any
|
var env map[string]any
|
||||||
for _, r := range composed(t, open, "anchor").Resources {
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
if r["id"] == "mesh-controller.server" {
|
if r["id"] == "mesh-controller.controller" {
|
||||||
env, _ = r["env"].(map[string]any)
|
env, _ = r["env"].(map[string]any)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if env == nil {
|
if env == nil {
|
||||||
t.Fatal("the control plane's container is not in its own node's declaration")
|
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||||
}
|
}
|
||||||
for key, want := range map[string]string{
|
for key, want := range map[string]string{
|
||||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||||
@@ -238,7 +246,7 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
|||||||
out := m
|
out := m
|
||||||
out.Resources = nil
|
out.Resources = nil
|
||||||
for _, r := range m.Resources {
|
for _, r := range m.Resources {
|
||||||
if r["type"] != "container" {
|
if r["type"] != "container" && r["type"] != "process" {
|
||||||
out.Resources = append(out.Resources, r)
|
out.Resources = append(out.Resources, r)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||||
|
|||||||
@@ -95,10 +95,10 @@ func commands(who Authenticator) http.Handler {
|
|||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
return assign(ctx, open, in.Node, in.Module)
|
return assign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||||
}))
|
}))
|
||||||
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
return unassign(ctx, open, in.Node, in.Module)
|
return unassign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||||
}))
|
}))
|
||||||
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
||||||
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -392,22 +393,34 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
|||||||
//
|
//
|
||||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||||
|
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
||||||
|
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
||||||
|
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
||||||
|
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
||||||
|
dryRun bool) (string, error) {
|
||||||
|
if dryRun && wait != 0 {
|
||||||
|
return "", errors.New("a dry run waited for is `build --dry-run`")
|
||||||
|
}
|
||||||
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
||||||
// the reason, rather than sent to a machine to fail at `git clone`.
|
// the reason, rather than sent to a machine to fail at `git clone`.
|
||||||
repository, err := cloneFrom(ctx, source)
|
repository, err := cloneFrom(ctx, source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
ident, err := openIdentity(ctx)
|
ident, err := openIdentity(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
defer ident.Close()
|
defer ident.Close()
|
||||||
|
|
||||||
server, err := connectLink(ctx, nil, nil, nil)
|
server, err := connectLink(ctx, nil, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
@@ -420,6 +433,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
|||||||
Ref: ref,
|
Ref: ref,
|
||||||
Held: heldBy(ctx),
|
Held: heldBy(ctx),
|
||||||
Seats: seatBases(ctx),
|
Seats: seatBases(ctx),
|
||||||
|
DryRun: dryRun,
|
||||||
}
|
}
|
||||||
fmt.Printf("asked for %s", source)
|
fmt.Printf("asked for %s", source)
|
||||||
if source.Seat != "" {
|
if source.Seat != "" {
|
||||||
@@ -438,7 +452,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
|||||||
seat := buildSeatHeld(ctx)
|
seat := buildSeatHeld(ctx)
|
||||||
ask, err := askOverOn(seat)
|
ask, err := askOverOn(seat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
defer ask.Close()
|
defer ask.Close()
|
||||||
fmt.Printf(" of %s\n", seat)
|
fmt.Printf(" of %s\n", seat)
|
||||||
@@ -449,26 +463,31 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
|||||||
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
|
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
|
||||||
// follows the build by its id instead.
|
// follows the build by its id instead.
|
||||||
if err := ask.Ask(ctx, request); err != nil {
|
if err := ask.Ask(ctx, request); err != nil {
|
||||||
return err
|
return "", err
|
||||||
|
}
|
||||||
|
if dryRun {
|
||||||
|
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
||||||
|
"its outcome is not taken in\n", request.ID)
|
||||||
|
return request.ID, nil
|
||||||
}
|
}
|
||||||
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
|
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
|
||||||
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
|
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
|
||||||
return nil
|
return request.ID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := ask.Submit(ctx, request, wait)
|
result, err := ask.Submit(ctx, request, wait)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return request.ID, err
|
||||||
}
|
}
|
||||||
|
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return request.ID, err
|
||||||
}
|
}
|
||||||
defer open.Close()
|
defer open.Close()
|
||||||
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return request.ID, err
|
||||||
}
|
}
|
||||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||||
for _, made := range kept.Made {
|
for _, made := range kept.Made {
|
||||||
@@ -478,7 +497,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
|||||||
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||||
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
|
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
|
||||||
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
||||||
return nil
|
return request.ID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
|
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
|
||||||
@@ -558,6 +577,10 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
|||||||
}
|
}
|
||||||
return manifest, kept, err
|
return manifest, kept, err
|
||||||
}
|
}
|
||||||
|
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||||
|
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||||
|
// worked and the module is registered.
|
||||||
|
collect(ctx, inv)
|
||||||
return manifest, kept, nil
|
return manifest, kept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -588,6 +611,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
|||||||
ID: link.NewBuildID(time.Now()),
|
ID: link.NewBuildID(time.Now()),
|
||||||
Repository: repository, Path: path, Ref: ref,
|
Repository: repository, Path: path, Ref: ref,
|
||||||
Held: heldBy(ctx), Seats: seatBases(ctx),
|
Held: heldBy(ctx), Seats: seatBases(ctx),
|
||||||
|
DryRun: true,
|
||||||
}, wait)
|
}, wait)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -623,6 +647,8 @@ type answers struct {
|
|||||||
reported []inventory.Reported
|
reported []inventory.Reported
|
||||||
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
||||||
plans []inventory.Plan
|
plans []inventory.Plan
|
||||||
|
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||||
|
paused pauseView
|
||||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||||
// other answer here: those are about a machine that was told something, and this is about one
|
// other answer here: those are about a machine that was told something, and this is about one
|
||||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||||
@@ -646,6 +672,25 @@ type answers struct {
|
|||||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||||
untaken map[string]map[string]int
|
untaken map[string]map[string]int
|
||||||
|
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||||
|
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||||
|
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||||
|
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||||
|
unheld []catalogue.Unheld
|
||||||
|
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
|
||||||
|
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
|
||||||
|
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
|
||||||
|
// not be read when they could not — never read as none.
|
||||||
|
conditions []conditions.Condition
|
||||||
|
conditionsUnread string
|
||||||
|
// overflowing is every module whose identity overflows the bound of a provision it requires
|
||||||
|
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||||
|
// this is the one place it is said across the mesh. Not well while there is any.
|
||||||
|
overflowing []catalogue.Overflow
|
||||||
|
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||||
|
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||||
|
handActs *int
|
||||||
|
handActsUnread string
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
|
||||||
|
// D6 and D7).
|
||||||
|
//
|
||||||
|
// **What the controller asserts at its start and what the self-check expects to find are one
|
||||||
|
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
|
||||||
|
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
|
||||||
|
// the drift rather than the fault.
|
||||||
|
|
||||||
|
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
|
||||||
|
// can now hear a declaration.
|
||||||
|
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(nodes))
|
||||||
|
for _, n := range nodes {
|
||||||
|
names = append(names, n.Name)
|
||||||
|
}
|
||||||
|
if err := broker.Raise(r, names); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||||
|
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||||
|
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||||
|
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||||
|
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||||
|
// consumer nothing had created (2026-09-28).
|
||||||
|
holders, err := seatHolders(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||||
|
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||||
|
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||||
|
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||||
|
consumers, err := moduleConsumers(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, c := range consumers {
|
||||||
|
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||||
|
return nil, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||||
|
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return broker.ConsumersOf(users), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||||
|
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||||
|
consumers, err := moduleConsumers(ctx, inv)
|
||||||
|
return len(consumers), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
|
||||||
|
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||||
|
var rec recordingRaiser
|
||||||
|
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
|
||||||
|
}
|
||||||
|
return rec.streams, rec.consumers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordingRaiser keeps what it was asked to assert and asserts nothing.
|
||||||
|
type recordingRaiser struct {
|
||||||
|
streams []broker.Stream
|
||||||
|
consumers []broker.Consumer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
|
||||||
|
r.streams = append(r.streams, s)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||||
|
r.consumers = append(r.consumers, c)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
|
||||||
|
// RequestTimeout) — the shortest wait of a caller the mesh ships.
|
||||||
|
const consoleWaits = 30 * time.Second
|
||||||
|
|
||||||
|
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
|
||||||
|
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
|
||||||
|
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
|
||||||
|
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
||||||
|
if link.AnswerWithin >= consoleWaits/2 {
|
||||||
|
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
|
||||||
|
}
|
||||||
|
if link.AnswerWithin >= broker.ResponseTTL {
|
||||||
|
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
||||||
|
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
||||||
|
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||||
|
{"push", map[string]any{"why": "w"}, true},
|
||||||
|
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||||
|
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||||
|
{"command", map[string]any{"command": "builds"}, false},
|
||||||
|
{"status", map[string]any{}, false},
|
||||||
|
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||||
|
} {
|
||||||
|
argv, err := argvFor(c.verb, c.args)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
||||||
|
}
|
||||||
|
if got := answersFirst(argv); got != c.want {
|
||||||
|
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||||
|
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||||
|
handlers, behind, err := seatToolHandlers()
|
||||||
|
if err != nil || len(behind) != 0 {
|
||||||
|
t.Fatalf("%v %v", behind, err)
|
||||||
|
}
|
||||||
|
calls, ok := handlers["calls"]
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("calls is not served")
|
||||||
|
}
|
||||||
|
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), `"node"`) {
|
||||||
|
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "not across a restart") {
|
||||||
|
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||||
|
}
|
||||||
|
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||||
|
t.Errorf("calls answered %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
)
|
)
|
||||||
@@ -24,7 +25,17 @@ import (
|
|||||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||||
// refused what it could not see would teach people to ignore it.
|
// refused what it could not see would teach people to ignore it.
|
||||||
|
//
|
||||||
|
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
||||||
|
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
||||||
|
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||||
|
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||||
|
// provider's machine when a real machine's name first meets the module's.
|
||||||
func moduleCheck(paths []string, out io.Writer) error {
|
func moduleCheck(paths []string, out io.Writer) error {
|
||||||
|
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||||
if len(paths) == 0 {
|
if len(paths) == 0 {
|
||||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||||
"manifest of a repository together so the rules between them are checked too")
|
"manifest of a repository together so the rules between them are checked too")
|
||||||
@@ -73,6 +84,15 @@ func moduleCheck(paths []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
failed += len(problems)
|
failed += len(problems)
|
||||||
|
|
||||||
|
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||||
|
// only the provider's manifest states.
|
||||||
|
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
||||||
|
sort.Strings(identities)
|
||||||
|
for _, p := range identities {
|
||||||
|
fmt.Fprintln(out, p)
|
||||||
|
}
|
||||||
|
failed += len(identities)
|
||||||
|
|
||||||
var names []string
|
var names []string
|
||||||
for name := range shelf {
|
for name := range shelf {
|
||||||
names = append(names, name)
|
names = append(names, name)
|
||||||
@@ -90,6 +110,17 @@ func moduleCheck(paths []string, out io.Writer) error {
|
|||||||
if len(m.Invokes) > 0 {
|
if len(m.Invokes) > 0 {
|
||||||
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
||||||
}
|
}
|
||||||
|
// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
|
||||||
|
if len(m.State) > 0 {
|
||||||
|
kept := make([]string, 0, len(m.State))
|
||||||
|
for _, s := range m.State {
|
||||||
|
kept = append(kept, s.Name)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||||
|
}
|
||||||
|
if len(m.Reads) > 0 {
|
||||||
|
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||||
|
}
|
||||||
fmt.Fprintln(out)
|
fmt.Fprintln(out)
|
||||||
}
|
}
|
||||||
if failed > 0 {
|
if failed > 0 {
|
||||||
@@ -97,7 +128,8 @@ func moduleCheck(paths []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||||
"module not given here reads as unknown\n", len(paths))
|
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
||||||
|
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,175 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Letting the artifact store go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||||
|
//
|
||||||
|
// **Run where the records change.** A build is the moment new bytes landed in the store and the
|
||||||
|
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
||||||
|
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
||||||
|
//
|
||||||
|
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
||||||
|
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
||||||
|
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
||||||
|
// having failed.
|
||||||
|
|
||||||
|
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
||||||
|
// what it let go of. Says what it did and what it could not; returns nothing, because nothing
|
||||||
|
// upstream should branch on it.
|
||||||
|
func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||||
|
references, err := inv.ToCollect(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
kept, err := inv.KeptArchives(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(references) == 0 && len(kept) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read the catalogue to find the artifact store: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// As the mesh reaches it from the network. Empty means the store is not on the network — on a
|
||||||
|
// mesh being raised it is not yet, and there the store holds one build of anything and has
|
||||||
|
// nothing to collect.
|
||||||
|
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil || address == "" {
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not find the artifact store to collect from: %v\n", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
|
||||||
|
// never collected has the whole history to get through, and a person waiting on `build` should
|
||||||
|
// not pay for it. Two bounds, and what is left over is simply offered again next time —
|
||||||
|
// builds are frequent, and the point is that the store stops growing, not that it empties
|
||||||
|
// tonight.
|
||||||
|
within, stop := context.WithTimeout(ctx, sweepBudget)
|
||||||
|
defer stop()
|
||||||
|
store := artifacts.Store{Address: address}
|
||||||
|
|
||||||
|
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||||
|
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||||
|
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||||
|
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||||
|
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||||
|
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||||
|
wrote, missing, err := holdKept(within, store, kept)
|
||||||
|
if wrote > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
|
||||||
|
}
|
||||||
|
if missing > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||||
|
"`collection` lists them\n", missing)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var done []string
|
||||||
|
var left, skipped int
|
||||||
|
for i, reference := range references {
|
||||||
|
if i >= mostPerSweep || within.Err() != nil {
|
||||||
|
left = len(references) - i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
err := store.LetGo(within, reference)
|
||||||
|
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||||
|
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||||
|
// again for ever.
|
||||||
|
done = append(done, reference)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||||
|
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||||
|
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||||
|
// reason to stop, because the store was never asked. One of these at the front of
|
||||||
|
// the oldest-first order ended every sweep until this.
|
||||||
|
skipped++
|
||||||
|
if skipped == 1 {
|
||||||
|
fmt.Fprintf(os.Stderr,
|
||||||
|
"the sweep will not address %s and went on: %v\n", reference, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||||
|
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||||
|
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||||
|
// front of whoever was building something.
|
||||||
|
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
|
||||||
|
reference, err)
|
||||||
|
left = len(references) - i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(done) > 0 {
|
||||||
|
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||||
|
// the sweep ran out of budget would mean asking about them again for ever.
|
||||||
|
if err := inv.MarkCollected(ctx, done); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
|
||||||
|
len(done), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
|
||||||
|
len(done))
|
||||||
|
}
|
||||||
|
if left > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
|
||||||
|
}
|
||||||
|
if skipped > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
|
||||||
|
// Answers how many holders it wrote and how many kept archives the store does not have.
|
||||||
|
//
|
||||||
|
// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
|
||||||
|
// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
|
||||||
|
// reference the store cannot be asked about is skipped as the deletion loop skips one
|
||||||
|
// (novox/hq issue 226).
|
||||||
|
func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
|
||||||
|
for _, reference := range kept {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
|
||||||
|
}
|
||||||
|
did, err := store.Hold(ctx, reference)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
if did {
|
||||||
|
wrote++
|
||||||
|
}
|
||||||
|
case errors.Is(err, artifacts.Gone):
|
||||||
|
missing++
|
||||||
|
case errors.Is(err, artifacts.ErrNotOurs):
|
||||||
|
default:
|
||||||
|
return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return wrote, missing, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
|
||||||
|
// several times a day converges within days of this landing; small enough that no single build
|
||||||
|
// waits on the whole backlog.
|
||||||
|
const mostPerSweep = 200
|
||||||
|
|
||||||
|
// sweepBudget is the longest a sweep will keep a build waiting.
|
||||||
|
const sweepBudget = 60 * time.Second
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
|
||||||
|
// (novox/hq issue 253, ADR 0189).
|
||||||
|
//
|
||||||
|
// **The question to answer before the store's collector runs for real.** The collector deletes
|
||||||
|
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
|
||||||
|
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
|
||||||
|
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
|
||||||
|
// dry run go.
|
||||||
|
//
|
||||||
|
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
|
||||||
|
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
|
||||||
|
// new verb in the seat's row.
|
||||||
|
|
||||||
|
type collectionReport struct {
|
||||||
|
// Store is the artifact store as this machine reached it; empty when it is not on the network.
|
||||||
|
Store string `json:"store"`
|
||||||
|
// KeptArchives is how many archives the mesh keeps, for either reason.
|
||||||
|
KeptArchives int `json:"kept_archives"`
|
||||||
|
// Held is how many of them the store holds by their manifest.
|
||||||
|
Held int `json:"held"`
|
||||||
|
// Unheld are the kept archives the collector would delete tonight if it ran for real.
|
||||||
|
Unheld []string `json:"unheld"`
|
||||||
|
// Missing are kept archives the store does not have at all.
|
||||||
|
Missing []string `json:"missing"`
|
||||||
|
// Unasked is how many could not be asked about, and why the asking stopped.
|
||||||
|
Unasked int `json:"unasked"`
|
||||||
|
Stopped string `json:"stopped,omitempty"`
|
||||||
|
// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
|
||||||
|
// collected — split by kind.
|
||||||
|
Eligible int `json:"eligible"`
|
||||||
|
EligibleImages int `json:"eligible_images"`
|
||||||
|
EligibleArchives int `json:"eligible_archives"`
|
||||||
|
// SafeToCollect is whether every kept archive was asked about and every one is held.
|
||||||
|
SafeToCollect bool `json:"safe_to_collect"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectionCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("collection", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "answer as JSON")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 0 {
|
||||||
|
return errors.New("collection [--json]")
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
kept, err := inv.KeptArchives(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
eligible, err := inv.ToCollect(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
|
||||||
|
for _, reference := range eligible {
|
||||||
|
if strings.Contains(reference, "/blobs/") {
|
||||||
|
report.EligibleArchives++
|
||||||
|
} else {
|
||||||
|
report.EligibleImages++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if report.Store == "" {
|
||||||
|
report.Unasked = len(kept)
|
||||||
|
report.Stopped = "this mesh has no artifact store on its network"
|
||||||
|
} else {
|
||||||
|
report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
|
||||||
|
}
|
||||||
|
report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
|
||||||
|
|
||||||
|
if *asJSON {
|
||||||
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
|
encoder.SetIndent("", " ")
|
||||||
|
return encoder.Encode(report)
|
||||||
|
}
|
||||||
|
printCollection(report)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// askHeld asks the store about each kept archive, stopping at the first answer that is not about
|
||||||
|
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
|
||||||
|
// identical failures says less than one line.
|
||||||
|
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
|
||||||
|
for i, reference := range kept {
|
||||||
|
held, err := store.Held(ctx, reference)
|
||||||
|
switch {
|
||||||
|
case err == nil && held:
|
||||||
|
report.Held++
|
||||||
|
case err == nil:
|
||||||
|
report.Unheld = append(report.Unheld, reference)
|
||||||
|
case errors.Is(err, artifacts.Gone):
|
||||||
|
report.Missing = append(report.Missing, reference)
|
||||||
|
case errors.Is(err, artifacts.ErrNotOurs):
|
||||||
|
// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
|
||||||
|
report.Unasked++
|
||||||
|
default:
|
||||||
|
return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return report.Unasked, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func printCollection(r collectionReport) {
|
||||||
|
store := r.Store
|
||||||
|
if store == "" {
|
||||||
|
store = "(not on the network)"
|
||||||
|
}
|
||||||
|
fmt.Printf("artifact store %s\n", store)
|
||||||
|
fmt.Printf("kept archives %d\n", r.KeptArchives)
|
||||||
|
fmt.Printf(" held %d\n", r.Held)
|
||||||
|
fmt.Printf(" unheld %d\n", len(r.Unheld))
|
||||||
|
fmt.Printf(" missing %d\n", len(r.Missing))
|
||||||
|
if r.Unasked > 0 {
|
||||||
|
fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped)
|
||||||
|
}
|
||||||
|
fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
|
||||||
|
if len(r.Unheld) > 0 {
|
||||||
|
fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
|
||||||
|
for _, reference := range r.Unheld {
|
||||||
|
fmt.Printf(" %s\n", reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(r.Missing) > 0 {
|
||||||
|
fmt.Println("\nmissing — kept by the mesh, not in the store:")
|
||||||
|
for _, reference := range r.Missing {
|
||||||
|
fmt.Printf(" %s\n", reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
if r.SafeToCollect {
|
||||||
|
fmt.Println("every kept archive is held: the store's collector may run for real")
|
||||||
|
} else {
|
||||||
|
fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,431 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
||||||
|
//
|
||||||
|
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||||
|
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||||
|
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||||
|
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||||
|
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||||
|
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||||
|
// while, with a reason, and that is recorded as a hand act.
|
||||||
|
|
||||||
|
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||||
|
var conditionsFrom *conditions.Keeper
|
||||||
|
|
||||||
|
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||||
|
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||||
|
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||||
|
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||||
|
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||||
|
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||||
|
// does nothing).
|
||||||
|
Changed: statusFrom.nudge}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
||||||
|
// it was given before it returns.
|
||||||
|
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
return f(conditionsFrom)
|
||||||
|
}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
k, err := keeperOn(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
k.Close(flushing)
|
||||||
|
}()
|
||||||
|
return f(k)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
||||||
|
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
||||||
|
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
return conditionsFrom.Open(ctx)
|
||||||
|
}
|
||||||
|
if _, err := broker.BusAddress(); err != nil {
|
||||||
|
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||||
|
}
|
||||||
|
var out []conditions.Condition
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
out, err = conditions.Read(reading, store)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// conditionsUsage is how the verb is typed.
|
||||||
|
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
||||||
|
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
||||||
|
"conditions history [--days N] [--key K] [--json]"
|
||||||
|
|
||||||
|
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
||||||
|
func conditionsCommand(ctx context.Context, args []string) error {
|
||||||
|
sub := "list"
|
||||||
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
sub, args = args[0], args[1:]
|
||||||
|
}
|
||||||
|
switch sub {
|
||||||
|
case "list":
|
||||||
|
return listConditions(ctx, args)
|
||||||
|
case "show":
|
||||||
|
return showCondition(ctx, args)
|
||||||
|
case "silence":
|
||||||
|
return silenceCondition(ctx, args)
|
||||||
|
case "history":
|
||||||
|
return conditionHistory(ctx, args)
|
||||||
|
}
|
||||||
|
return errors.New(conditionsUsage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func listConditions(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||||
|
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||||
|
severity := set.String("severity", "", "only urgent, or only warning")
|
||||||
|
machine := set.String("machine", "", "only those about this machine")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New(conditionsUsage)
|
||||||
|
}
|
||||||
|
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
||||||
|
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
||||||
|
}
|
||||||
|
open, err := openConditions(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var out []conditions.Condition
|
||||||
|
for _, c := range open {
|
||||||
|
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
||||||
|
(*machine == "" || concerns(c, *machine)) {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
if out == nil {
|
||||||
|
out = []conditions.Condition{}
|
||||||
|
}
|
||||||
|
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||||
|
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
if len(open) == 0 {
|
||||||
|
fmt.Println("no open conditions")
|
||||||
|
} else {
|
||||||
|
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, line := range conditionLines(out, time.Now()) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||||
|
func concerns(c conditions.Condition, machine string) bool {
|
||||||
|
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, part := range strings.Split(c.Key, ".") {
|
||||||
|
if part == machine {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
||||||
|
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||||
|
var out []string
|
||||||
|
for _, c := range list {
|
||||||
|
times := ""
|
||||||
|
if c.Count > 1 {
|
||||||
|
times = fmt.Sprintf(", raised %d times", c.Count)
|
||||||
|
}
|
||||||
|
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
||||||
|
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
||||||
|
if c.SilencedAt(now) {
|
||||||
|
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
||||||
|
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func showCondition(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
rest, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 1 {
|
||||||
|
return errors.New("conditions show <key>")
|
||||||
|
}
|
||||||
|
key := rest[0]
|
||||||
|
var c conditions.Condition
|
||||||
|
var found bool
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
c, found, err = conditionsFrom.Get(ctx, key)
|
||||||
|
} else {
|
||||||
|
err = onTheBus(func(conn *nats.Conn) error {
|
||||||
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
||||||
|
"history --key %s` what became of it", key, key)
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return printJSON(c)
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||||
|
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||||
|
if c.Subject.Machine != "" {
|
||||||
|
fmt.Printf(", on %s", c.Subject.Machine)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||||
|
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||||
|
now.Sub(c.LastObserved).Round(time.Second))
|
||||||
|
if c.Count > 1 {
|
||||||
|
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||||
|
}
|
||||||
|
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||||
|
if c.Silenced != nil {
|
||||||
|
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||||
|
c.Silenced.By, c.Silenced.Why)
|
||||||
|
}
|
||||||
|
if len(c.Tried) > 0 {
|
||||||
|
fmt.Println("\n tried:")
|
||||||
|
for _, t := range c.Tried {
|
||||||
|
fmt.Printf(" %s %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), t.What, t.Outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println("\n evidence, newest first:")
|
||||||
|
for _, e := range c.Evidence {
|
||||||
|
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolverWords(r string) string {
|
||||||
|
switch r {
|
||||||
|
case conditions.ResolverSelf:
|
||||||
|
return "itself: it clears when observation says it is resolved"
|
||||||
|
case conditions.ResolverOperator:
|
||||||
|
return "the operator: nothing in the mesh will repair it"
|
||||||
|
case conditions.ResolverAgent:
|
||||||
|
return "an agent"
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
||||||
|
// who and why before it is done, its cause the condition's kind unless one is given.
|
||||||
|
func silenceCondition(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
||||||
|
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
||||||
|
acts := addHandActFlags(set)
|
||||||
|
rest, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 1 {
|
||||||
|
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
||||||
|
}
|
||||||
|
key := rest[0]
|
||||||
|
if err := acts.require("conditions silence"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
d, err := parseFor(*forFlag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d > conditions.MaxSilence {
|
||||||
|
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
||||||
|
}
|
||||||
|
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||||
|
c, found, err := k.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*acts.cause) == "" {
|
||||||
|
*acts.cause = c.Kind
|
||||||
|
}
|
||||||
|
*acts.condition = key
|
||||||
|
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
||||||
|
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
||||||
|
"`status` still says it; it clears when observation says it is resolved\n",
|
||||||
|
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseFor reads a duration, days included.
|
||||||
|
func parseFor(s string) (time.Duration, error) {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
||||||
|
}
|
||||||
|
if days, ok := strings.CutSuffix(s, "d"); ok {
|
||||||
|
n, err := strconv.Atoi(days)
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return 0, fmt.Errorf("%q is not a number of days", s)
|
||||||
|
}
|
||||||
|
return time.Duration(n) * 24 * time.Hour, nil
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(s)
|
||||||
|
if err != nil || d <= 0 {
|
||||||
|
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func conditionHistory(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||||
|
days := set.Int("days", 7, "how many days back, at most 90")
|
||||||
|
key := set.String("key", "", "only this condition")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New("conditions history [--days N] [--key K] [--json]")
|
||||||
|
}
|
||||||
|
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
||||||
|
var events []conditions.Event
|
||||||
|
read := func(h conditions.History) error {
|
||||||
|
var err error
|
||||||
|
events, err = h.Since(ctx, since)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
events, err = conditionsFrom.HistorySince(ctx, since)
|
||||||
|
} else {
|
||||||
|
err = onTheBus(func(conn *nats.Conn) error {
|
||||||
|
_, history, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return read(history)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var out []conditions.Event
|
||||||
|
for _, e := range events {
|
||||||
|
if *key == "" || e.Key == *key {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
if out == nil {
|
||||||
|
out = []conditions.Event{}
|
||||||
|
}
|
||||||
|
return printJSON(map[string]any{"history": out, "days": *days})
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, e := range out {
|
||||||
|
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
||||||
|
switch e.Change {
|
||||||
|
case conditions.ChangeRaised, conditions.ChangeReopened:
|
||||||
|
line += " — " + e.Summary
|
||||||
|
case conditions.ChangeSeverity:
|
||||||
|
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
||||||
|
case conditions.ChangeResolver:
|
||||||
|
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
||||||
|
default:
|
||||||
|
if e.Why != "" {
|
||||||
|
line += " — " + e.Why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
||||||
|
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
||||||
|
// is not "none open".
|
||||||
|
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
||||||
|
if unread != "" {
|
||||||
|
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(list) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
urgent := 0
|
||||||
|
for _, c := range list {
|
||||||
|
if c.Severity == conditions.Urgent {
|
||||||
|
urgent++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
||||||
|
for _, line := range conditionLines(list, now) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
||||||
|
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
|
||||||
|
// by what is open.
|
||||||
|
|
||||||
|
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
args map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{map[string]any{}, "conditions --json"},
|
||||||
|
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
|
||||||
|
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
|
||||||
|
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
|
||||||
|
"conditions history --json --days 3 --key machine.ace.silent"},
|
||||||
|
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
|
||||||
|
"conditions silence machine.ace.silent --for 2h --why on the train"},
|
||||||
|
{map[string]any{"run": "true"}, "doctor run --json"},
|
||||||
|
{map[string]any{}, "doctor --json"},
|
||||||
|
} {
|
||||||
|
verb := "conditions"
|
||||||
|
if strings.HasPrefix(c.want, "doctor") {
|
||||||
|
verb = "doctor"
|
||||||
|
}
|
||||||
|
argv, err := argvFor(verb, c.args)
|
||||||
|
if err != nil || strings.Join(argv, " ") != c.want {
|
||||||
|
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
}{
|
||||||
|
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
|
||||||
|
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
|
||||||
|
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
|
||||||
|
} {
|
||||||
|
if argv, err := argvFor(c.verb, c.args); err == nil {
|
||||||
|
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
|
||||||
|
t.Error("a silence is not a hand act")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A silence through the verb is recorded and bounded**; the condition stays open.
|
||||||
|
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
|
||||||
|
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
|
||||||
|
t.Fatal("silenced without saying why")
|
||||||
|
}
|
||||||
|
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
|
||||||
|
t.Fatal("silenced for more than a week")
|
||||||
|
}
|
||||||
|
said := printed(t, func() error {
|
||||||
|
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
|
||||||
|
})
|
||||||
|
if !strings.Contains(said, "is silenced until") {
|
||||||
|
t.Fatalf("%s", said)
|
||||||
|
}
|
||||||
|
c, found, _ := k.Get(ctx, "machine.ace.silent")
|
||||||
|
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
|
||||||
|
t.Fatalf("%+v", c)
|
||||||
|
}
|
||||||
|
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
|
||||||
|
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
|
||||||
|
t.Fatalf("%s", listed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||||
|
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
_, store := withConditionsInMemory(t)
|
||||||
|
store.Fail = errors.New("the bus is away")
|
||||||
|
asked, err := theThreeQuestions(t.Context(), open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asked.well() || asked.conditionsUnread == "" {
|
||||||
|
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||||
|
}
|
||||||
|
said := printed(t, func() error { return printStatus(asked) })
|
||||||
|
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||||
|
t.Fatalf("%s", said)
|
||||||
|
}
|
||||||
|
store.Fail = nil
|
||||||
|
asked, _ = theThreeQuestions(t.Context(), open)
|
||||||
|
said = printed(t, func() error { return printStatus(asked) })
|
||||||
|
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||||
|
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
|
||||||
|
type recordingDelivery struct {
|
||||||
|
did []string
|
||||||
|
grantErr error
|
||||||
|
declareErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
|
||||||
|
for _, s := range sending {
|
||||||
|
r.did = append(r.did, "grant "+s.node)
|
||||||
|
}
|
||||||
|
return r.grantErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
|
||||||
|
if r.declareErr != nil {
|
||||||
|
return "", r.declareErr
|
||||||
|
}
|
||||||
|
r.did = append(r.did, "declare "+s.node)
|
||||||
|
return "digest-" + s.node, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ready(names ...string) []readyNode {
|
||||||
|
var out []readyNode
|
||||||
|
for _, n := range names {
|
||||||
|
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
|
||||||
|
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
|
||||||
|
// carries the controller's own right to issue them, and goes first.
|
||||||
|
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
|
||||||
|
d := &recordingDelivery{}
|
||||||
|
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
|
||||||
|
if !reflect.DeepEqual(d.did, want) {
|
||||||
|
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
|
||||||
|
}
|
||||||
|
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
|
||||||
|
t.Fatalf("the digests sent were not answered: %v", digests)
|
||||||
|
}
|
||||||
|
|
||||||
|
held := &recordingDelivery{}
|
||||||
|
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
|
||||||
|
if !reflect.DeepEqual(held.did, want) {
|
||||||
|
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
|
||||||
|
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
|
||||||
|
// grant that would let the controller issue memberships is never held behind them.
|
||||||
|
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
|
||||||
|
// A failure naming no machine (the buckets): everything but the bus's machine.
|
||||||
|
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
|
||||||
|
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
|
||||||
|
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
|
||||||
|
!strings.Contains(err.Error(), "anchor, laptop not sent") {
|
||||||
|
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
|
||||||
|
}
|
||||||
|
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
|
||||||
|
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A membership that failed for one machine: that machine alone.
|
||||||
|
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
|
||||||
|
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
|
||||||
|
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
|
||||||
|
t.Fatalf("one machine's refused membership was not said: %v", err)
|
||||||
|
}
|
||||||
|
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
|
||||||
|
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The announced upgrade that hit it is asked again.
|
||||||
|
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
|
||||||
|
t.Fatal("an announcement whose send stopped at its grants is not asked again")
|
||||||
|
}
|
||||||
|
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
|
||||||
|
t.Fatal("any failure is asked again, not only a grant's")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing to send is nothing granted either.
|
||||||
|
none := &recordingDelivery{grantErr: errors.New("never asked")}
|
||||||
|
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
|
||||||
|
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whether the bus's machine goes first is read from the user list alone, by its digest.
|
||||||
|
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||||
|
list := "users: [a, b]"
|
||||||
|
if userListBehind(list, digestOf([]byte(list))) {
|
||||||
|
t.Fatal("the list it was sent reads as behind")
|
||||||
|
}
|
||||||
|
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
|
||||||
|
t.Fatal("a changed or never-sent list reads as current")
|
||||||
|
}
|
||||||
|
if userListBehind("", "") {
|
||||||
|
t.Fatal("a machine sent no list reads as behind")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||||
|
// checked against. Among the machines it is moved to the front; not among them it is added only
|
||||||
|
// when it is behind.
|
||||||
|
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
names []string
|
||||||
|
holder string
|
||||||
|
behind bool
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
|
||||||
|
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
|
||||||
|
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
|
||||||
|
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
|
||||||
|
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
|
||||||
|
} {
|
||||||
|
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
|
||||||
|
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,536 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
|
||||||
|
//
|
||||||
|
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
|
||||||
|
// design — every machine's declaration composes and validates, every resolver answers, every seat's
|
||||||
|
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
|
||||||
|
// condition it raises when the invariant does not hold. The serving controller runs the registry every
|
||||||
|
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
|
||||||
|
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
|
||||||
|
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
|
||||||
|
// machine: a controller that stops checking is itself said, through a channel that does not pass
|
||||||
|
// through it.
|
||||||
|
//
|
||||||
|
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
|
||||||
|
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
|
||||||
|
|
||||||
|
// The self-check's clocks.
|
||||||
|
var (
|
||||||
|
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
|
||||||
|
// starts the first run waits, so what the controller hears at its start has arrived.
|
||||||
|
doctorEvery = 5 * time.Minute
|
||||||
|
doctorFirstAfter = time.Minute
|
||||||
|
// probeWithin is each probe's bound.
|
||||||
|
probeWithin = 30 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// The verdicts a probe can have.
|
||||||
|
const (
|
||||||
|
verdictPass = "pass"
|
||||||
|
verdictFail = "fail"
|
||||||
|
verdictFailedToRun = "failed-to-run"
|
||||||
|
verdictDeferred = "deferred"
|
||||||
|
)
|
||||||
|
|
||||||
|
// probe is one live invariant.
|
||||||
|
type probe struct {
|
||||||
|
ID string
|
||||||
|
Asserts string
|
||||||
|
From string
|
||||||
|
// Kind is the condition kind raised when the invariant does not hold.
|
||||||
|
Kind string
|
||||||
|
Phase int
|
||||||
|
// Deferred says why it is not run yet; empty for one that is.
|
||||||
|
Deferred string
|
||||||
|
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
|
||||||
|
// controller's grant names every one (a test over this registry): a probe whose question the bus
|
||||||
|
// refuses checks nothing (D8, 2026-10-06).
|
||||||
|
Asks []broker.SeatVerb
|
||||||
|
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||||
|
// probe added to a design is a row added here.
|
||||||
|
var probeRegistry = []probe{
|
||||||
|
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||||
|
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
|
||||||
|
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||||
|
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||||
|
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||||
|
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
|
||||||
|
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
|
||||||
|
Kind: "archives-unheld", Phase: 1, run: probeArchives},
|
||||||
|
{ID: "D5", Asserts: "exactly one lease holder; no message from a stale epoch in the last interval",
|
||||||
|
From: "issue 204", Kind: "lease-split", Phase: 2,
|
||||||
|
Deferred: "the lease and epoch are built in Phase 2 (to-be 45 §6): nothing holds one yet"},
|
||||||
|
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||||
|
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Phase: 1, run: probeConsumers},
|
||||||
|
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
|
||||||
|
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
|
||||||
|
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||||
|
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
|
||||||
|
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
|
||||||
|
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
|
||||||
|
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
|
||||||
|
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
|
||||||
|
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
|
||||||
|
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||||
|
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeVerdict is one probe's outcome in a run.
|
||||||
|
type probeVerdict struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Verdict string `json:"verdict"`
|
||||||
|
Found []string `json:"found,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Took string `json:"took,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorCounts are a run's verdicts, counted.
|
||||||
|
type doctorCounts struct {
|
||||||
|
Passed int `json:"passed"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
FailedToRun int `json:"failed-to-run"`
|
||||||
|
Deferred int `json:"deferred"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorRun is one run of the registry, and the body of its heartbeat.
|
||||||
|
type doctorRun struct {
|
||||||
|
Run string `json:"run"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Took string `json:"took"`
|
||||||
|
IntervalSeconds int `json:"interval-seconds"`
|
||||||
|
Counts doctorCounts `json:"counts"`
|
||||||
|
Probes []probeVerdict `json:"probes"`
|
||||||
|
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
|
||||||
|
Controller string `json:"controller"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
// Unsaid is how many condition transitions this controller could not say, since it started.
|
||||||
|
Unsaid int `json:"unsaid,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctor is the registry and what its probes need.
|
||||||
|
type doctor struct {
|
||||||
|
open *stores
|
||||||
|
js *broker.JetStream
|
||||||
|
keeper *conditions.Keeper
|
||||||
|
teller conditions.Teller
|
||||||
|
watchdogs *watchdogs
|
||||||
|
host string
|
||||||
|
|
||||||
|
running sync.Mutex
|
||||||
|
mu sync.Mutex
|
||||||
|
last *doctorRun
|
||||||
|
ended time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastRunEnded is when the last run ended; zero before the first.
|
||||||
|
func (d *doctor) lastRunEnded() time.Time {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
return d.ended
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastRun is the last run's verdict; nil before the first.
|
||||||
|
func (d *doctor) lastRun() *doctorRun {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
return d.last
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep runs the registry on its schedule until ctx ends.
|
||||||
|
func (d *doctor) keep(ctx context.Context) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(doctorFirstAfter):
|
||||||
|
}
|
||||||
|
tick := time.NewTicker(doctorEvery)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
// Only the controller acting checks the mesh on a schedule: one standing by would say a
|
||||||
|
// heartbeat for a self-check that is not the mesh's.
|
||||||
|
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
|
||||||
|
d.runOnce(ctx, "the schedule")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var doctorRuns struct {
|
||||||
|
sync.Mutex
|
||||||
|
n uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
|
||||||
|
// at a time: a person's `doctor run` during a scheduled one waits for it.
|
||||||
|
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||||
|
d.running.Lock()
|
||||||
|
defer d.running.Unlock()
|
||||||
|
doctorRuns.Lock()
|
||||||
|
doctorRuns.n++
|
||||||
|
n := doctorRuns.n
|
||||||
|
doctorRuns.Unlock()
|
||||||
|
started := time.Now()
|
||||||
|
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||||
|
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
|
||||||
|
|
||||||
|
type result struct {
|
||||||
|
obs []conditions.Observation
|
||||||
|
err error
|
||||||
|
took time.Duration
|
||||||
|
}
|
||||||
|
results := make([]result, len(probeRegistry))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, p := range probeRegistry {
|
||||||
|
if p.run == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, p probe) {
|
||||||
|
defer wg.Done()
|
||||||
|
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
|
||||||
|
defer cancel()
|
||||||
|
began := time.Now()
|
||||||
|
done := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
obs, err := p.run(probing, d)
|
||||||
|
done <- result{obs: obs, err: err}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case r := <-done:
|
||||||
|
r.took = time.Since(began)
|
||||||
|
results[i] = r
|
||||||
|
case <-probing.Done():
|
||||||
|
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
|
||||||
|
}
|
||||||
|
}(i, p)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
var blind []conditions.Observation
|
||||||
|
for i, p := range probeRegistry {
|
||||||
|
v := probeVerdict{ID: p.ID}
|
||||||
|
r := results[i]
|
||||||
|
switch {
|
||||||
|
case p.run == nil:
|
||||||
|
v.Verdict = verdictDeferred
|
||||||
|
run.Counts.Deferred++
|
||||||
|
case r.err != nil:
|
||||||
|
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
|
||||||
|
run.Counts.FailedToRun++
|
||||||
|
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
|
||||||
|
Token: "failed", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||||
|
Said: firstLine(r.err.Error())})
|
||||||
|
default:
|
||||||
|
if err := d.keeper.Reconcile(ctx, p.ID, kindedAs(r.obs, p.Kind)); err != nil {
|
||||||
|
v.Error = "what it found could not be kept: " + err.Error()
|
||||||
|
}
|
||||||
|
if len(r.obs) == 0 {
|
||||||
|
v.Verdict = verdictPass
|
||||||
|
run.Counts.Passed++
|
||||||
|
} else {
|
||||||
|
v.Verdict = verdictFail
|
||||||
|
run.Counts.Failed++
|
||||||
|
for _, o := range r.obs {
|
||||||
|
v.Found = append(v.Found, o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if p.run != nil {
|
||||||
|
v.Took = r.took.Round(time.Millisecond).String()
|
||||||
|
}
|
||||||
|
run.Probes = append(run.Probes, v)
|
||||||
|
}
|
||||||
|
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
|
||||||
|
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
|
||||||
|
}
|
||||||
|
ended := time.Now()
|
||||||
|
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
|
||||||
|
d.mu.Lock()
|
||||||
|
d.last, d.ended = &run, ended
|
||||||
|
d.mu.Unlock()
|
||||||
|
d.sayHeartbeat(ctx, run)
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceDoctor is what raises a probe's own failure to run.
|
||||||
|
const sourceDoctor = "doctor"
|
||||||
|
|
||||||
|
// kindedAs gives each observation of a probe the probe's kind where it named none.
|
||||||
|
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
|
||||||
|
out := make([]conditions.Observation, 0, len(obs))
|
||||||
|
for _, o := range obs {
|
||||||
|
if o.Kind == "" {
|
||||||
|
o.Kind = kind
|
||||||
|
}
|
||||||
|
out = append(out, o)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
|
||||||
|
// says it outward: the watcher hears nothing.
|
||||||
|
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
|
||||||
|
if d.teller == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(run)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
|
||||||
|
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
|
||||||
|
"will say the self-check is silent: %v\n", run.Run, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorFrom is the serving controller's self-check; nil in any other process.
|
||||||
|
var doctorFrom *doctor
|
||||||
|
|
||||||
|
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
|
||||||
|
func doctorCommand(ctx context.Context, args []string) error {
|
||||||
|
sub := ""
|
||||||
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
sub, args = args[0], args[1:]
|
||||||
|
}
|
||||||
|
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New("doctor [run|probes|signals] [--json]")
|
||||||
|
}
|
||||||
|
answer, err := doctorAnswer(ctx, sub)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return printJSON(answer)
|
||||||
|
}
|
||||||
|
fmt.Print(doctorText(answer))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorAnswer is what the verb answers, as data.
|
||||||
|
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||||
|
switch sub {
|
||||||
|
case "":
|
||||||
|
if doctorFrom != nil {
|
||||||
|
if run := doctorFrom.lastRun(); run != nil {
|
||||||
|
return verdictAnswer(*run, time.Now()), nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||||
|
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||||
|
}
|
||||||
|
run, err := lastHeartbeat(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return verdictAnswer(run, time.Now()), nil
|
||||||
|
case "run":
|
||||||
|
d := doctorFrom
|
||||||
|
if d == nil {
|
||||||
|
local, closeIt, err := localDoctor(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer closeIt()
|
||||||
|
d = local
|
||||||
|
}
|
||||||
|
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||||
|
case "probes":
|
||||||
|
return probesAnswer(), nil
|
||||||
|
case "signals":
|
||||||
|
if doctorFrom == nil || doctorFrom.watchdogs == nil {
|
||||||
|
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
|
||||||
|
"hears them: ask it through the mesh-controller seat's doctor verb")
|
||||||
|
}
|
||||||
|
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
|
||||||
|
}
|
||||||
|
|
||||||
|
// verdictAnswer is a run as the verb answers it, with its age.
|
||||||
|
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||||
|
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||||
|
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// probesAnswer is the registry.
|
||||||
|
func probesAnswer() map[string]any {
|
||||||
|
var out []map[string]any
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
|
||||||
|
if p.Deferred != "" {
|
||||||
|
row["deferred"] = p.Deferred
|
||||||
|
}
|
||||||
|
out = append(out, row)
|
||||||
|
}
|
||||||
|
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// signalsAnswer is every row of the signals table with the age of its newest signal.
|
||||||
|
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
|
||||||
|
var rows []map[string]any
|
||||||
|
for _, r := range signalsTable {
|
||||||
|
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
|
||||||
|
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
|
||||||
|
switch {
|
||||||
|
case r.Deferred != "":
|
||||||
|
row["deferred"] = r.Deferred
|
||||||
|
case f == nil:
|
||||||
|
row["newest"] = "not yet looked at"
|
||||||
|
default:
|
||||||
|
if err := r.needs(f); err != nil {
|
||||||
|
row["blind"] = err.Error()
|
||||||
|
} else if newest := r.newest(f); newest.IsZero() {
|
||||||
|
row["newest"] = "none heard"
|
||||||
|
} else {
|
||||||
|
row["newest"] = newest.UTC().Format(time.RFC3339)
|
||||||
|
row["age"] = f.now.Sub(newest).Round(time.Second).String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows = append(rows, row)
|
||||||
|
}
|
||||||
|
out := map[string]any{"signals": rows, "every": watchEvery.String()}
|
||||||
|
if !ticked.IsZero() {
|
||||||
|
out["looked"] = ticked.UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorText is an answer as a person reads it.
|
||||||
|
func doctorText(answer any) string {
|
||||||
|
body, _ := json.Marshal(answer)
|
||||||
|
var b strings.Builder
|
||||||
|
var verdict struct {
|
||||||
|
Run doctorRun `json:"run"`
|
||||||
|
Age string `json:"age"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
|
||||||
|
r := verdict.Run
|
||||||
|
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
|
||||||
|
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
|
||||||
|
for _, p := range r.Probes {
|
||||||
|
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
|
||||||
|
for _, f := range p.Found {
|
||||||
|
fmt.Fprintf(&b, " %s\n", f)
|
||||||
|
}
|
||||||
|
if p.Error != "" {
|
||||||
|
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
pretty, _ := json.MarshalIndent(answer, "", " ")
|
||||||
|
return string(pretty) + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
|
||||||
|
// the serving controller answers `doctor` from.
|
||||||
|
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
|
||||||
|
var run doctorRun
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
js, err := conn.JetStream(nats.Context(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
|
||||||
|
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||||
|
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
return json.Unmarshal(msg.Data, &run)
|
||||||
|
})
|
||||||
|
return run, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
|
||||||
|
// its own connection, and its own keeper, closed after.
|
||||||
|
func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
open.Close()
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
k, err := keeperOn(ctx, js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
js.Close()
|
||||||
|
open.Close()
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
jsCtx := js.Context()
|
||||||
|
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
|
||||||
|
host: controlHost(ctx, open.inventory)}
|
||||||
|
return d, func() {
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
k.Close(flushing)
|
||||||
|
js.Close()
|
||||||
|
open.Close()
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
|
||||||
|
func sortedFound(obs []conditions.Observation) []conditions.Observation {
|
||||||
|
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
|
||||||
|
return obs
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
|
||||||
|
type probeAsksKey struct{}
|
||||||
|
|
||||||
|
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
|
||||||
|
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
|
||||||
|
p, ok := ctx.Value(probeAsksKey{}).(probe)
|
||||||
|
if !ok {
|
||||||
|
return "a caller outside the self-check", false
|
||||||
|
}
|
||||||
|
for _, v := range p.Asks {
|
||||||
|
if v.Seat == seat && v.Verb == verb {
|
||||||
|
return p.ID, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return p.ID, false
|
||||||
|
}
|
||||||
@@ -0,0 +1,425 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"golang.org/x/net/dns/dnsmessage"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||||
|
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
|
||||||
|
|
||||||
|
// withProbes runs the test with a registry of its own.
|
||||||
|
func withProbes(t *testing.T, probes ...probe) {
|
||||||
|
t.Helper()
|
||||||
|
before := probeRegistry
|
||||||
|
probeRegistry = probes
|
||||||
|
t.Cleanup(func() { probeRegistry = before })
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||||
|
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||||
|
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||||
|
var broken atomic.Bool
|
||||||
|
broken.Store(true)
|
||||||
|
withProbes(t,
|
||||||
|
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||||
|
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
return []conditions.Observation{failing}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
return nil, errors.New("the store is away")
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||||
|
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
<-ctx.Done()
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||||
|
)
|
||||||
|
before := probeWithin
|
||||||
|
probeWithin = 200 * time.Millisecond
|
||||||
|
t.Cleanup(func() { probeWithin = before })
|
||||||
|
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
told := &conditions.Told{}
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||||
|
run := d.runOnce(t.Context(), "a test")
|
||||||
|
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||||
|
t.Fatalf("counted %+v", run.Counts)
|
||||||
|
}
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range open {
|
||||||
|
keys = append(keys, c.Key+"="+c.Kind)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||||
|
"probe.P4.failed=probe-failed"} {
|
||||||
|
if !slices.Contains(keys, want) {
|
||||||
|
t.Errorf("%s is not open: %v", want, keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||||
|
if len(told.Names) != 1 || told.Names[0] != conditions.HeartbeatEvent {
|
||||||
|
t.Fatalf("said %v", told.Names)
|
||||||
|
}
|
||||||
|
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||||
|
t.Fatal("the run is not the last verdict")
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(run)
|
||||||
|
var shape map[string]any
|
||||||
|
_ = json.Unmarshal(body, &shape)
|
||||||
|
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||||
|
if _, ok := shape[field]; !ok {
|
||||||
|
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mended: the next run clears every one of them.
|
||||||
|
broken.Store(false)
|
||||||
|
d.runOnce(t.Context(), "a test")
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("a passing run left open %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The registry says what each probe asserts**, and a probe not built says why and when.
|
||||||
|
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
if seen[p.ID] {
|
||||||
|
t.Errorf("%s twice", p.ID)
|
||||||
|
}
|
||||||
|
seen[p.ID] = true
|
||||||
|
if p.Asserts == "" || p.From == "" || p.Kind == "" {
|
||||||
|
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
|
||||||
|
}
|
||||||
|
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
|
||||||
|
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
|
||||||
|
if !seen[id] {
|
||||||
|
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
|
||||||
|
// stopped is S10 (signals_test.go).
|
||||||
|
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
|
||||||
|
w := &watchdogs{started: time.Now().Add(-time.Hour)}
|
||||||
|
d := &doctor{watchdogs: w, host: "anchor"}
|
||||||
|
got, err := probeWatchdogs(t.Context(), d)
|
||||||
|
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("%+v %v", got, err)
|
||||||
|
}
|
||||||
|
w.ticked = time.Now()
|
||||||
|
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
|
||||||
|
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
|
||||||
|
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
one, two := rivals()
|
||||||
|
register(t, open, one)
|
||||||
|
register(t, open, two)
|
||||||
|
for _, m := range []string{"rival-one", "rival-two"} {
|
||||||
|
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
|
||||||
|
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||||
|
answerAs := func(rcode dnsmessage.RCode) string {
|
||||||
|
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = conn.Close() })
|
||||||
|
go func() {
|
||||||
|
buf := make([]byte, 1500)
|
||||||
|
for {
|
||||||
|
n, from, err := conn.ReadFrom(buf)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var q dnsmessage.Message
|
||||||
|
if q.Unpack(buf[:n]) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
|
||||||
|
if q.Questions[0].Type == dnsmessage.TypeA {
|
||||||
|
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
|
||||||
|
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
|
||||||
|
} else {
|
||||||
|
reply.RCode = rcode
|
||||||
|
}
|
||||||
|
packed, _ := reply.Pack()
|
||||||
|
_, _ = conn.WriteTo(packed, from)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||||
|
return port
|
||||||
|
}
|
||||||
|
before := resolverPort
|
||||||
|
t.Cleanup(func() { resolverPort = before })
|
||||||
|
|
||||||
|
resolverPort = answerAs(dnsmessage.RCodeSuccess)
|
||||||
|
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
|
||||||
|
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
|
||||||
|
t.Fatalf("%v %v %v", v4, rcode, err)
|
||||||
|
}
|
||||||
|
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
|
||||||
|
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
|
||||||
|
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
|
||||||
|
}
|
||||||
|
resolverPort = answerAs(dnsmessage.RCodeNameError)
|
||||||
|
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
|
||||||
|
t.Fatalf("NXDOMAIN read as %v", rcode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||||
|
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||||
|
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
open := aMesh(t)
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||||
|
_ = js.Context().DeleteStream(s)
|
||||||
|
}
|
||||||
|
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d := &doctor{open: open, js: js}
|
||||||
|
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||||
|
got, err := p(t.Context(), d)
|
||||||
|
if err != nil || len(got) != 0 {
|
||||||
|
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := js.Context().StreamInfo("EVENTS")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := info.Config
|
||||||
|
cfg.MaxMsgsPerSubject = 3
|
||||||
|
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
consumers, err := probeConsumers(t.Context(), d)
|
||||||
|
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||||
|
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||||
|
}
|
||||||
|
streams, err := probeStreams(t.Context(), d)
|
||||||
|
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||||
|
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||||
|
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
heard := make(chan *nats.Msg, 16)
|
||||||
|
for _, subject := range broker.BusAdvisories {
|
||||||
|
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
api, _ := jetstream.New(conn)
|
||||||
|
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||||
|
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||||
|
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||||
|
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||||
|
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||||
|
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kinds := map[string]string{}
|
||||||
|
deadline := time.After(5 * time.Second)
|
||||||
|
for len(kinds) < 2 {
|
||||||
|
select {
|
||||||
|
case m := <-heard:
|
||||||
|
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||||
|
kinds[a.Kind] = a.Said
|
||||||
|
}
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatalf("the bus said only %v", kinds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||||
|
t.Fatalf("%v", kinds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
|
||||||
|
// machine read as behind right after a push sent it the current build — it says the digest-named
|
||||||
|
// directory it runs from, and the mesh holds a commit).
|
||||||
|
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
|
||||||
|
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
|
||||||
|
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
|
||||||
|
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
|
||||||
|
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
|
||||||
|
}}
|
||||||
|
delivered := deliveredVersions(m)
|
||||||
|
if !slices.Equal(delivered, []string{"31045596c83a"}) {
|
||||||
|
t.Fatalf("%v", delivered)
|
||||||
|
}
|
||||||
|
commit := "1545b00a9f0c"
|
||||||
|
for _, c := range []struct {
|
||||||
|
reported string
|
||||||
|
behind bool
|
||||||
|
}{
|
||||||
|
{"31045596c83a", false}, // the live case: current, and was called behind
|
||||||
|
{"0123456789ab", true}, // another delivery
|
||||||
|
{"1545b00a", false}, // placed by hand, stamped with the commit
|
||||||
|
{"", false}, // not said
|
||||||
|
} {
|
||||||
|
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
|
||||||
|
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if engineBehind("31045596c83a", nil, commit) {
|
||||||
|
t.Error("behind a mesh that holds no delivered build")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
|
||||||
|
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
|
||||||
|
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
|
||||||
|
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
asked := 0
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
for _, v := range p.Asks {
|
||||||
|
asked++
|
||||||
|
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
|
||||||
|
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
|
||||||
|
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
|
||||||
|
}
|
||||||
|
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
|
||||||
|
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if asked == 0 {
|
||||||
|
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
|
||||||
|
}
|
||||||
|
// And a probe asking what it did not declare is refused before anything is sent.
|
||||||
|
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
|
||||||
|
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "does not declare") {
|
||||||
|
t.Fatalf("an undeclared question was asked: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// subjectMatches is the bus's matching of a permission pattern against a subject.
|
||||||
|
func subjectMatches(pattern, subject string) bool {
|
||||||
|
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
|
||||||
|
for i, tok := range p {
|
||||||
|
if tok == ">" {
|
||||||
|
return len(s) > i
|
||||||
|
}
|
||||||
|
if i >= len(s) || (tok != "*" && tok != s[i]) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(p) == len(s)
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
|
||||||
|
// store at all, so anything that tried to record or register would fail rather than pass quietly.
|
||||||
|
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
|
||||||
|
err := builds{}.Built(t.Context(), link.BuildResult{
|
||||||
|
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
|
||||||
|
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The mark survives the wire both ways: asked as a dry run, answered as one.
|
||||||
|
func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
|
||||||
|
raw, _ := json.Marshal(link.BuildRequest{ID: "build-1", Repository: "r", DryRun: true})
|
||||||
|
var asked link.BuildRequest
|
||||||
|
if err := json.Unmarshal(raw, &asked); err != nil || !asked.DryRun {
|
||||||
|
t.Fatalf("the request lost its dry-run mark: %s", raw)
|
||||||
|
}
|
||||||
|
raw, _ = json.Marshal(link.BuildResult{ID: "build-1", DryRun: true})
|
||||||
|
var answered link.BuildResult
|
||||||
|
if err := json.Unmarshal(raw, &answered); err != nil || !answered.DryRun {
|
||||||
|
t.Fatalf("the outcome lost its dry-run mark: %s", raw)
|
||||||
|
}
|
||||||
|
raw, _ = json.Marshal(link.BuildResult{ID: "build-2"})
|
||||||
|
if string(raw) != `{"id":"build-2","repository":"","on":""}` {
|
||||||
|
t.Fatalf("an ordinary outcome carries a dry-run mark: %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||||
|
//
|
||||||
|
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||||
|
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||||
|
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||||
|
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||||
|
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||||
|
// build's outcome — and summarised here per machine, repository or module.
|
||||||
|
|
||||||
|
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||||
|
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||||
|
if asked.IsZero() || result.ID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
subject := result.Module
|
||||||
|
if subject == "" {
|
||||||
|
subject = result.Repository
|
||||||
|
}
|
||||||
|
detail := "built"
|
||||||
|
if result.Failed != "" {
|
||||||
|
detail = "failed: " + firstLine(result.Failed)
|
||||||
|
}
|
||||||
|
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||||
|
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// durationSummary is one subject's measurements of one kind.
|
||||||
|
type durationSummary struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
Median string `json:"median"`
|
||||||
|
P90 string `json:"p90"`
|
||||||
|
Max string `json:"max"`
|
||||||
|
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||||
|
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||||
|
Suggests string `json:"suggests,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func summarise(ds []inventory.Duration) []durationSummary {
|
||||||
|
type key struct{ kind, subject string }
|
||||||
|
by := map[key][]time.Duration{}
|
||||||
|
for _, d := range ds {
|
||||||
|
k := key{d.Kind, d.Subject}
|
||||||
|
by[k] = append(by[k], d.Took)
|
||||||
|
}
|
||||||
|
var out []durationSummary
|
||||||
|
for k, took := range by {
|
||||||
|
slices.Sort(took)
|
||||||
|
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||||
|
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||||
|
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||||
|
switch k.kind {
|
||||||
|
case inventory.DurationApply:
|
||||||
|
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||||
|
case inventory.DurationHeartbeatGap:
|
||||||
|
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||||
|
}
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||||
|
if ki != kj {
|
||||||
|
return ki < kj
|
||||||
|
}
|
||||||
|
return out[i].Subject < out[j].Subject
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func round(d time.Duration) string {
|
||||||
|
switch {
|
||||||
|
case d < time.Second:
|
||||||
|
return d.Round(time.Millisecond).String()
|
||||||
|
case d < time.Minute:
|
||||||
|
return d.Round(100 * time.Millisecond).String()
|
||||||
|
default:
|
||||||
|
return d.Round(time.Second).String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||||
|
func durationsCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||||
|
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||||
|
days := set.Int("days", 14, "how many days back")
|
||||||
|
asJSON := set.Bool("json", false, "the summary as data")
|
||||||
|
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||||
|
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *all {
|
||||||
|
body, err := json.MarshalIndent(ds, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
summary := summarise(ds)
|
||||||
|
if *asJSON {
|
||||||
|
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(summary) == 0 {
|
||||||
|
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||||
|
"plan-tier and build durations as it hears them\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||||
|
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||||
|
for _, s := range summary {
|
||||||
|
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||||
|
if s.Suggests != "" {
|
||||||
|
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||||
|
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||||
|
for {
|
||||||
|
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||||
|
} else if n > 0 {
|
||||||
|
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(24 * time.Hour):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||||
|
//
|
||||||
|
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||||
|
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||||
|
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||||
|
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||||
|
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||||
|
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||||
|
// (Phase 1).
|
||||||
|
|
||||||
|
// handActFlags are the flags every repairing verb takes.
|
||||||
|
type handActFlags struct {
|
||||||
|
why, cause, condition *string
|
||||||
|
}
|
||||||
|
|
||||||
|
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||||
|
return handActFlags{
|
||||||
|
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||||
|
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||||
|
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// given is whether a reason was given.
|
||||||
|
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||||
|
|
||||||
|
// require refuses an act without a reason, before anything is done.
|
||||||
|
func (f handActFlags) require(verb string) error {
|
||||||
|
if f.given() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||||
|
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||||
|
// command dials its own.
|
||||||
|
var handActConn *nats.Conn
|
||||||
|
|
||||||
|
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||||
|
func onTheBus(f func(*nats.Conn) error) error {
|
||||||
|
if handActConn != nil {
|
||||||
|
return f(handActConn)
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
return f(js.Conn())
|
||||||
|
}
|
||||||
|
|
||||||
|
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||||
|
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||||
|
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||||
|
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||||
|
if !f.given() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||||
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
act = written
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActCommand is `hand-act record` and `hand-acts`.
|
||||||
|
func handActCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) > 0 && args[0] == "record" {
|
||||||
|
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||||
|
f := addHandActFlags(set)
|
||||||
|
positionals, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||||
|
if what == "" {
|
||||||
|
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||||
|
}
|
||||||
|
if err := f.require("hand-act record"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*f.cause) == "" {
|
||||||
|
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||||
|
"act for the same reason will use — it is how a repair done twice is found")
|
||||||
|
}
|
||||||
|
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||||
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||||
|
written.Why, written.Cause)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||||
|
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] == "list" {
|
||||||
|
args = args[1:]
|
||||||
|
}
|
||||||
|
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||||
|
days := set.Int("days", 14, "how many days back")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
now := time.Now()
|
||||||
|
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
repeated := link.RepeatedCauses(acts, now)
|
||||||
|
if *asJSON {
|
||||||
|
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(acts) == 0 {
|
||||||
|
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(acts) - 1; i >= 0; i-- {
|
||||||
|
a := acts[i]
|
||||||
|
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||||
|
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||||
|
if a.Condition != "" {
|
||||||
|
fmt.Printf(", condition %s", a.Condition)
|
||||||
|
}
|
||||||
|
fmt.Println(")")
|
||||||
|
}
|
||||||
|
if len(repeated) > 0 {
|
||||||
|
causes := make([]string, 0, len(repeated))
|
||||||
|
for c, n := range repeated {
|
||||||
|
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||||
|
}
|
||||||
|
sort.Strings(causes)
|
||||||
|
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||||
|
strings.Join(causes, ", "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||||
|
// the log could not be read, which status says rather than reading as none.
|
||||||
|
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||||
|
n := -1
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||||
|
n = len(acts)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return -1, err.Error()
|
||||||
|
}
|
||||||
|
return n, ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||||
|
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||||
|
// the verb.
|
||||||
|
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
}{
|
||||||
|
{"push", map[string]any{"node": "anchor"}},
|
||||||
|
{"push", map[string]any{}},
|
||||||
|
{"plans", map[string]any{"close": "plan-1"}},
|
||||||
|
{"plans", map[string]any{"stop": "plan-1"}},
|
||||||
|
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||||
|
{"command", map[string]any{"command": "push anchor"}},
|
||||||
|
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||||
|
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||||
|
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||||
|
} {
|
||||||
|
argv, err := argvFor(c.verb, c.args)
|
||||||
|
if c.verb == "plans" && err == nil {
|
||||||
|
// The seat composes the command line; the command refuses it, before opening anything.
|
||||||
|
err = plansCommand(context.Background(), argv[1:])
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||||
|
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||||
|
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Errorf("consumer-reset without why: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Errorf("hand-act record without why: %v", err)
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--cause") {
|
||||||
|
t.Errorf("hand-act record without a cause: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||||
|
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||||
|
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||||
|
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||||
|
"plans close plan-1 --why the report will not come"},
|
||||||
|
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||||
|
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||||
|
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||||
|
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||||
|
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||||
|
} {
|
||||||
|
argv, err := argvFor(c.verb, c.args)
|
||||||
|
if err != nil || strings.Join(argv, " ") != c.want {
|
||||||
|
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||||
|
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||||
|
var ds []inventory.Duration
|
||||||
|
for i := 1; i <= 10; i++ {
|
||||||
|
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||||
|
Took: time.Duration(i) * time.Second})
|
||||||
|
}
|
||||||
|
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||||
|
got := summarise(ds)
|
||||||
|
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||||
|
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||||
|
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,241 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
|
||||||
|
// issue 259, ADR 0221).
|
||||||
|
//
|
||||||
|
// A named push ends by sending every other machine whose declaration differs from what it was last
|
||||||
|
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
|
||||||
|
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
|
||||||
|
// out makes every machine running it differ from the merge on, and so did a module whose plan was
|
||||||
|
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
|
||||||
|
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
|
||||||
|
// everywhere at once.
|
||||||
|
//
|
||||||
|
// What tells the two apart is which build of each module the machine was last sent, kept with every
|
||||||
|
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
|
||||||
|
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
|
||||||
|
|
||||||
|
// heldBack is why a push that did not name a machine must not send it, empty when it may.
|
||||||
|
//
|
||||||
|
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
|
||||||
|
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
|
||||||
|
// machine was last sent — including a module the machine was never sent at all. A move is held when
|
||||||
|
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
|
||||||
|
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
|
||||||
|
// is not known, so a held upgrade cannot be told from anything else.
|
||||||
|
func heldBack(node string, modules []string, sent map[string]string, known bool,
|
||||||
|
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
|
||||||
|
if !known {
|
||||||
|
return []string{"which builds it was last sent is not known — it was last sent before the " +
|
||||||
|
"mesh kept them, or sent a declaration by hand"}
|
||||||
|
}
|
||||||
|
var why []string
|
||||||
|
for _, m := range modules {
|
||||||
|
now := current[m]
|
||||||
|
was, carried := sent[m]
|
||||||
|
if carried && was == now.Commit {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
|
||||||
|
if !now.RollOut {
|
||||||
|
why = append(why, move+", which its upgrade policy records rather than rolls out")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if id := planStillToSend(plans, m, node); id != "" {
|
||||||
|
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(why)
|
||||||
|
return why
|
||||||
|
}
|
||||||
|
|
||||||
|
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
|
||||||
|
// one holding the module that has neither finished sending it nor sent it here first, and has not
|
||||||
|
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
|
||||||
|
// per machine.
|
||||||
|
func planStillToSend(plans []inventory.Plan, module, node string) string {
|
||||||
|
for _, p := range plans {
|
||||||
|
s, holds := p.Modules[module]
|
||||||
|
if !p.Open() || !holds {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if s == nil {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
|
if s.SentAt != nil || s.State == "failed" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
first := false
|
||||||
|
for _, n := range s.First {
|
||||||
|
if n == node {
|
||||||
|
first = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !first {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromBuild(was string, carried bool) string {
|
||||||
|
if !carried {
|
||||||
|
return "(never sent it) "
|
||||||
|
}
|
||||||
|
return "from " + buildName(was) + " "
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildName(commit string) string {
|
||||||
|
if commit == "" {
|
||||||
|
return "a build with no source"
|
||||||
|
}
|
||||||
|
return shortCommit(commit)
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldMachines reads, for each machine named, why a push that did not name it must not send it
|
||||||
|
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
|
||||||
|
// to the send, which says why.
|
||||||
|
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
|
||||||
|
out := map[string][]string{}
|
||||||
|
if len(names) == 0 {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
inv := open.inventory
|
||||||
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, node := range names {
|
||||||
|
plan, _, err := planFor(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
modules := make([]string, 0, len(plan.Modules))
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
modules = append(modules, m.Module)
|
||||||
|
}
|
||||||
|
sent, known, err := inv.SentBuilds(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
|
||||||
|
out[node] = why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
|
||||||
|
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
|
||||||
|
func sayHeld(w io.Writer, node string, why []string) {
|
||||||
|
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
|
||||||
|
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
|
||||||
|
"grant from this push among it — waits with it. `push %s` sends it\n",
|
||||||
|
node, strings.Join(why, "; "), node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
|
||||||
|
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
|
||||||
|
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
|
||||||
|
//
|
||||||
|
// `handled` is every machine already sent or already said; it is not considered again. Answers the
|
||||||
|
// machines that could not be composed, as refusals.
|
||||||
|
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
||||||
|
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
||||||
|
w io.Writer) ([]string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
var refusals []string
|
||||||
|
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
||||||
|
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
|
||||||
|
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
|
||||||
|
// cascade legitimately still converging, so it is said rather than passed over in silence.
|
||||||
|
rounds := 0
|
||||||
|
for {
|
||||||
|
would, err := wouldSend(ctx, open, nodes)
|
||||||
|
if err != nil {
|
||||||
|
return refusals, err
|
||||||
|
}
|
||||||
|
behind, err := inv.Waiting(ctx, would)
|
||||||
|
if err != nil {
|
||||||
|
return refusals, err
|
||||||
|
}
|
||||||
|
var also []string
|
||||||
|
for _, m := range behind {
|
||||||
|
if !handled[m.Node] {
|
||||||
|
also = append(also, m.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(also) == 0 {
|
||||||
|
return refusals, nil
|
||||||
|
}
|
||||||
|
if rounds++; rounds > len(nodes) {
|
||||||
|
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
|
||||||
|
"should not happen; run `push --behind` to finish\n",
|
||||||
|
rounds-1, strings.Join(also, ", "))
|
||||||
|
return refusals, nil
|
||||||
|
}
|
||||||
|
sort.Strings(also)
|
||||||
|
held, err := heldMachines(ctx, open, also)
|
||||||
|
if err != nil {
|
||||||
|
return refusals, err
|
||||||
|
}
|
||||||
|
var sending []string
|
||||||
|
for _, name := range also {
|
||||||
|
// Every candidate this round is marked handled — the sent ones so they are not
|
||||||
|
// re-listed, the held ones because they stay held, and the refused ones so a machine
|
||||||
|
// that cannot be composed does not make the loop spin on it for ever.
|
||||||
|
handled[name] = true
|
||||||
|
if why, isHeld := held[name]; isHeld {
|
||||||
|
sayHeld(w, name, why)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sending = append(sending, name)
|
||||||
|
}
|
||||||
|
if len(sending) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
|
||||||
|
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
|
||||||
|
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
|
||||||
|
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
||||||
|
// Held for this round only, and after the last round's were given back, so two pushes
|
||||||
|
// cascading into each other's machines never each wait on the other.
|
||||||
|
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
||||||
|
refusals = append(refusals, refused...)
|
||||||
|
if err != nil {
|
||||||
|
return refusals, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
|
||||||
|
// is left out of it said, and its declaration allocated.
|
||||||
|
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
|
||||||
|
return func(held context.Context, node string) (sendable, error) {
|
||||||
|
plan, settings, err := planFor(held, open, node)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
reportUnhostable(node, plan)
|
||||||
|
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||||
|
if err == nil {
|
||||||
|
reportLeftOut(node, declared)
|
||||||
|
}
|
||||||
|
return declared, err
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,335 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
|
||||||
|
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
|
||||||
|
// else that moved is still a consequence the push sends (ADR 0083).
|
||||||
|
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
|
||||||
|
current := map[string]inventory.CurrentBuild{
|
||||||
|
"resolver": {Commit: "c2c2c2c2c2"},
|
||||||
|
"agent": {Commit: "a2", RollOut: true},
|
||||||
|
"network": {},
|
||||||
|
}
|
||||||
|
modules := []string{"network", "resolver", "agent"}
|
||||||
|
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
|
||||||
|
|
||||||
|
// A module whose policy records moved: held, naming it, both builds and why.
|
||||||
|
why := heldBack("laptop", modules, sent, true, current, nil)
|
||||||
|
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
|
||||||
|
!strings.Contains(why[0], "upgrade policy records") {
|
||||||
|
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
|
||||||
|
sent["resolver"] = "c2c2c2c2c2"
|
||||||
|
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||||
|
t.Fatalf("a machine whose builds are all current was held: %v", why)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
|
||||||
|
sent["agent"] = "a1"
|
||||||
|
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||||
|
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The last send's builds are not known: held whole.
|
||||||
|
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
|
||||||
|
!strings.Contains(why[0], "not known") {
|
||||||
|
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module the machine was never sent, under a recording policy: held, and said so.
|
||||||
|
delete(sent, "resolver")
|
||||||
|
sent["agent"] = "a2"
|
||||||
|
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
|
||||||
|
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
|
||||||
|
t.Fatalf("a module never sent under a recording policy: %v", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
|
||||||
|
// naming some other machine must not send them for it.
|
||||||
|
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
|
||||||
|
at := time.Now()
|
||||||
|
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
|
||||||
|
sent := map[string]string{"agent": "a1"}
|
||||||
|
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||||
|
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
|
||||||
|
|
||||||
|
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
|
||||||
|
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
|
||||||
|
t.Fatalf("a machine the plan has not reached was not held: %v", why)
|
||||||
|
}
|
||||||
|
// The first machine itself was sent by the plan: not held by it.
|
||||||
|
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
|
||||||
|
t.Fatalf("the plan's first machine was held: %v", why)
|
||||||
|
}
|
||||||
|
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
|
||||||
|
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
|
||||||
|
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
|
||||||
|
Modules: map[string]*inventory.PlanModule{"agent": s}}}
|
||||||
|
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
|
||||||
|
t.Errorf("%s: not held: %v", what, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
|
||||||
|
for what, plans := range map[string][]inventory.Plan{
|
||||||
|
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||||
|
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
|
||||||
|
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||||
|
"agent": {State: "failed"}}}},
|
||||||
|
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
|
||||||
|
"agent": {State: "built"}}}},
|
||||||
|
} {
|
||||||
|
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
|
||||||
|
t.Errorf("%s: held: %v", what, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A send records the build of each module it carried; a module left out of it keeps the build it
|
||||||
|
// was last sent, since the machine keeps that one.
|
||||||
|
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
|
||||||
|
current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
|
||||||
|
got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
|
||||||
|
current, map[string]string{"a": "a1", "b": "b1"})
|
||||||
|
if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("carried %v, wanted %v", got, want)
|
||||||
|
}
|
||||||
|
// Not known before: the left-out module is not recorded at all, so it reads as never sent.
|
||||||
|
got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
|
||||||
|
if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("carried %v, wanted %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
|
||||||
|
// reads the machine as current — and writes down which machines it declared.
|
||||||
|
type recordedDelivery struct {
|
||||||
|
inv *inventory.Inventory
|
||||||
|
declared []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||||
|
|
||||||
|
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||||
|
r.declared = append(r.declared, s.node)
|
||||||
|
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
|
||||||
|
}
|
||||||
|
|
||||||
|
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||||
|
// says which build it is.
|
||||||
|
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
|
||||||
|
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
|
||||||
|
}}
|
||||||
|
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
|
||||||
|
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A third machine on the private network: once it is sent, every other machine's peers change with
|
||||||
|
// it, which is a consequence a push must still send — no build moved.
|
||||||
|
func aThirdMachine(t *testing.T, open *stores) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
record, err := open.inventory.AddNode(ctx, "spare")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||||
|
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
|
||||||
|
{"name": "systemd", "present": true}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var profile map[string]any
|
||||||
|
if err := json.Unmarshal(reported, &profile); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
|
||||||
|
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
|
||||||
|
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
asked := time.Now().Add(-time.Hour)
|
||||||
|
aResolver(t, open, "c1c1c1c1c1", asked)
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gens, err := generators(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
compose := composeForPush(open, gens)
|
||||||
|
d := &recordedDelivery{inv: inv}
|
||||||
|
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||||
|
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
|
||||||
|
}
|
||||||
|
digestOfLaptop := func() string {
|
||||||
|
sent, err := inv.Outstanding(ctx, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return sent
|
||||||
|
}
|
||||||
|
before := digestOfLaptop()
|
||||||
|
|
||||||
|
// The change merges; the policy is the default, record. `push anchor` sends the anchor...
|
||||||
|
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||||
|
d.declared = nil
|
||||||
|
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// ...and its cascade leaves the laptop, saying so.
|
||||||
|
var said bytes.Buffer
|
||||||
|
d.declared = nil
|
||||||
|
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
|
||||||
|
if err != nil || len(refused) != 0 {
|
||||||
|
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||||
|
}
|
||||||
|
if len(d.declared) != 0 {
|
||||||
|
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
|
||||||
|
}
|
||||||
|
if digestOfLaptop() != before {
|
||||||
|
t.Fatal("the laptop's last send moved: it was sent the held build")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
|
||||||
|
"upgrade policy records", "`push laptop` sends it"} {
|
||||||
|
if !strings.Contains(said.String(), want) {
|
||||||
|
t.Errorf("the push did not say %q:\n%s", want, said.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
|
||||||
|
// is held, and that what else it is owed waits with it.
|
||||||
|
aThirdMachine(t, open)
|
||||||
|
d.declared = nil
|
||||||
|
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.declared = nil
|
||||||
|
said.Reset()
|
||||||
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||||
|
compose, d, "", &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||||
|
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
|
||||||
|
t.Fatalf("the push did not say both:\n%s", said.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A policy that rolls out: the laptop is a consequence like any other, and sent.
|
||||||
|
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said.Reset()
|
||||||
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||||
|
compose, d, "", &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||||
|
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
|
||||||
|
}
|
||||||
|
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
|
||||||
|
t.Fatalf("the new send did not record the new build: %v", builds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
|
||||||
|
// for another reason is sent by a push that names someone else.
|
||||||
|
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
|
||||||
|
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gens, err := generators(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
compose := composeForPush(open, gens)
|
||||||
|
d := &recordedDelivery{inv: inv}
|
||||||
|
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// `push spare`, the machine just placed: the others' peers change with it.
|
||||||
|
aThirdMachine(t, open)
|
||||||
|
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.declared = nil
|
||||||
|
var said bytes.Buffer
|
||||||
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||||
|
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
|
||||||
|
}
|
||||||
|
if strings.Contains(said.String(), "was not sent") {
|
||||||
|
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
|
||||||
|
record, err := inv.NodeByName(ctx, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.declared = nil
|
||||||
|
said.Reset()
|
||||||
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||||
|
// question.
|
||||||
|
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
|
||||||
|
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,16 +18,16 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
|||||||
plain := func(context.Context, string) (sendable, error) {
|
plain := func(context.Context, string) (sendable, error) {
|
||||||
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
||||||
}
|
}
|
||||||
failing := func(readyNode, []byte) error { return errors.New("the broker went away") }
|
failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
|
||||||
fine := func(readyNode, []byte) error { return nil }
|
fine := &recordingDelivery{}
|
||||||
|
|
||||||
for name, round := range map[string]func() error{
|
for name, round := range map[string]func() error{
|
||||||
"a body that cannot be marshalled": func() error {
|
"a body that cannot be marshalled": func() error {
|
||||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
|
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
|
||||||
return err
|
return err
|
||||||
},
|
},
|
||||||
"a send that fails": func() error {
|
"a send that fails": func() error {
|
||||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
|
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
|
||||||
return err
|
return err
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
)
|
)
|
||||||
|
|
||||||
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
||||||
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
|
|||||||
}
|
}
|
||||||
return said, nil
|
return said, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
|
||||||
|
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
|
||||||
|
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
|
||||||
|
//
|
||||||
|
// **The holders on record, and only the sole claimant when there are none** — the same answer the
|
||||||
|
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
|
||||||
|
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
|
||||||
|
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
|
||||||
|
// listed here. A holder off the private network is left out: a resolver named at an address nothing
|
||||||
|
// answers is a lookup that waits out its timeout on every name.
|
||||||
|
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
|
||||||
|
var replicated []catalogue.Seat
|
||||||
|
for _, s := range catalogue.Seats() {
|
||||||
|
if s.Replicated && s.Scope == catalogue.ScopeMesh {
|
||||||
|
replicated = append(replicated, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(replicated) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
recorded, err := inv.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
places, err := onTheNetwork(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
address := map[string]string{}
|
||||||
|
for _, p := range places {
|
||||||
|
address[p.Name] = p.Address
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]map[string]string{}
|
||||||
|
for _, seat := range replicated {
|
||||||
|
var nodes []string
|
||||||
|
for _, h := range recorded {
|
||||||
|
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
|
||||||
|
nodes = append(nodes, h.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
var derived []string
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||||
|
derived = append(derived, e.On...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(derived) == 1 {
|
||||||
|
nodes = derived
|
||||||
|
}
|
||||||
|
}
|
||||||
|
at := map[string]string{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if address[n] != "" {
|
||||||
|
at[overlay.InternalName(n)] = address[n]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out[seat.Name] = at
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||||
|
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
||||||
|
// one consumer's identity.
|
||||||
|
|
||||||
|
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
||||||
|
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
write := func(name, body string) string {
|
||||||
|
p := filepath.Join(dir, name+".json")
|
||||||
|
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
objects := write("objects", `{"module":"objects","version":"1",
|
||||||
|
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||||
|
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
||||||
|
resolver := write("resolver", `{"module":"resolver","version":"1",
|
||||||
|
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
||||||
|
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
||||||
|
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
||||||
|
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
out.Reset()
|
||||||
|
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
||||||
|
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
||||||
|
strings.Contains(out.String(), "networkmanager wants") {
|
||||||
|
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
||||||
|
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||||
|
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||||
|
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||||
|
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||||
|
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||||
|
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||||
|
Requires: []string{"wildcard-resolution"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||||
|
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||||
|
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||||
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||||
|
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||||
|
consumer, _, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
over := consumer.Overflowing()
|
||||||
|
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||||
|
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||||
|
// networkmanager, and no push to the provider could go through.
|
||||||
|
plan, settings, err := planFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||||
|
}
|
||||||
|
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||||
|
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||||
|
}
|
||||||
|
grants, _, err := grantsFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keyless bool
|
||||||
|
for _, g := range grants {
|
||||||
|
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||||
|
}
|
||||||
|
if !keyless {
|
||||||
|
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||||
|
}
|
||||||
|
var granted []string
|
||||||
|
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||||
|
granted = append(granted, c.From)
|
||||||
|
}
|
||||||
|
if strings.Join(granted, ",") != "files" {
|
||||||
|
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||||
|
}
|
||||||
|
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||||
|
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||||
|
!strings.Contains(said, "left out of anchor's grants") {
|
||||||
|
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And `status` names it, and does not call the mesh well while it stands.
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||||
|
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||||
|
}
|
||||||
|
if asked.well() {
|
||||||
|
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||||
|
}
|
||||||
|
shown := printed(t, func() error { return printStatus(asked) })
|
||||||
|
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||||
|
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||||
|
t.Fatalf("status does not say it:\n%s", shown)
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||||
|
doc.Overflowing[0].Bound.Max != 20 {
|
||||||
|
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||||
digest, err := recordSent(ctx, inv, "anchor", body)
|
digest, err := recordSent(ctx, inv, "anchor", body, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||||
// through the detached context, so look the node up again on a live one.
|
// through the detached context, so look the node up again on a live one.
|
||||||
|
|||||||
@@ -73,6 +73,23 @@ func run() error {
|
|||||||
return askCommand(ctx, args[1:])
|
return askCommand(ctx, args[1:])
|
||||||
case "builds":
|
case "builds":
|
||||||
return buildsCommand(ctx, args[1:])
|
return buildsCommand(ctx, args[1:])
|
||||||
|
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||||
|
case "queue":
|
||||||
|
return queueCommand(ctx, args[1:])
|
||||||
|
case "cancel":
|
||||||
|
return cancelCommand(ctx, args[1:])
|
||||||
|
case "clear":
|
||||||
|
return clearCommand(ctx, args[1:])
|
||||||
|
case "rebuild":
|
||||||
|
return rebuildCommand(ctx, args[1:])
|
||||||
|
case "replay":
|
||||||
|
return replayCommand(ctx, args[1:])
|
||||||
|
case "kill":
|
||||||
|
return killCommand(ctx, args[1:])
|
||||||
|
case "pause", "resume":
|
||||||
|
return pauseCommand(ctx, args[0], args[1:])
|
||||||
|
case "collection":
|
||||||
|
return collectionCommand(ctx, args[1:])
|
||||||
case "plans":
|
case "plans":
|
||||||
return plansCommand(ctx, args[1:])
|
return plansCommand(ctx, args[1:])
|
||||||
case "pin":
|
case "pin":
|
||||||
@@ -128,6 +145,19 @@ func run() error {
|
|||||||
return seatCommand(ctx, args[1:])
|
return seatCommand(ctx, args[1:])
|
||||||
case "status":
|
case "status":
|
||||||
return statusCommand(ctx, args[1:])
|
return statusCommand(ctx, args[1:])
|
||||||
|
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||||
|
case "hand-act":
|
||||||
|
return handActCommand(ctx, args[1:])
|
||||||
|
case "hand-acts":
|
||||||
|
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||||
|
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||||
|
case "durations":
|
||||||
|
return durationsCommand(ctx, args[1:])
|
||||||
|
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||||
|
case "conditions":
|
||||||
|
return conditionsCommand(ctx, args[1:])
|
||||||
|
case "doctor":
|
||||||
|
return doctorCommand(ctx, args[1:])
|
||||||
case "version":
|
case "version":
|
||||||
fmt.Println(version)
|
fmt.Println(version)
|
||||||
return nil
|
return nil
|
||||||
@@ -177,10 +207,11 @@ func usage() {
|
|||||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||||
|
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
||||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||||
assign <node> <module> put a module on a node
|
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||||
unassign <node> <module> take it off
|
unassign <node> <module>... take them off
|
||||||
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
||||||
what it declares; --yes <digest> cuts it over as previewed
|
what it declares; --yes <digest> cuts it over as previewed
|
||||||
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
||||||
@@ -200,18 +231,41 @@ func usage() {
|
|||||||
build --behind build every module the mesh holds older than its source
|
build --behind build every module the mesh holds older than its source
|
||||||
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
||||||
builds [<module>] what has been built lately, and what came of it
|
builds [<module>] what has been built lately, and what came of it
|
||||||
|
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
|
||||||
|
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
|
||||||
|
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
|
||||||
|
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
|
||||||
|
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
|
||||||
|
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||||
|
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||||
|
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||||
|
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||||
|
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||||
|
record an act done by hand outside the mesh (to-be 45 §7)
|
||||||
|
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||||
|
conditions [--scope S] [--severity S] [--machine M] [--json]
|
||||||
|
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
||||||
|
conditions show <key> one condition whole, with its evidence
|
||||||
|
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
||||||
|
conditions history [--days N] [--key K] every raising, change and clearing lately
|
||||||
|
doctor [run|probes|signals] [--json]
|
||||||
|
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||||
|
durations [--kind K] [--days N] [--json]
|
||||||
|
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||||
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||||
delivered as the builder module's broker secret (module add it first)
|
delivered as the builder module's broker secret (module add it first)
|
||||||
licence add|list|use|key model access, under the name a person calls it
|
licence add|list|use|key model access, under the name a person calls it
|
||||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||||
licence refresh <name> mint a new access token and seal it to every holder
|
licence refresh <name> mint a new access token and seal it to every holder
|
||||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||||
pin <node> <provision> <from-node> <module>
|
pin <node> <provision> <from-node> <module>
|
||||||
which provider this one gets a provision from: the module, and its node
|
which provider this one gets a provision from: the module, and its node
|
||||||
unpin <node> <provision> put that question back
|
unpin <node> <provision> put that question back
|
||||||
plan <node> [--files|--json] what that node would run, and why
|
plan <node> [--files|--json] what that node would run, and why
|
||||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||||
|
that need it; --why records it in the hand-act log
|
||||||
version what this binary is
|
version what this binary is
|
||||||
|
|
||||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||||
@@ -253,15 +307,25 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
|||||||
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
||||||
// became of a build nobody was watching.
|
// became of a build nobody was watching.
|
||||||
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||||
|
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
|
||||||
|
// unmerged branch was heard here like any build, registered, and its definition reached a machine
|
||||||
|
// before anyone had reviewed it.
|
||||||
|
if result.DryRun {
|
||||||
|
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||||
asked, _ := link.BuildAskedAt(result.ID)
|
asked, _ := link.BuildAskedAt(result.ID)
|
||||||
|
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||||
|
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||||
|
recordBuildDuration(ctx, b.inv, result, asked)
|
||||||
switch {
|
switch {
|
||||||
case err != nil && result.Failed != "":
|
case err != nil && result.Failed != "":
|
||||||
fmt.Printf("%s: %v\n", result.ID, err)
|
fmt.Printf("%s: %v\n", result.ID, err)
|
||||||
if result.Module != "" {
|
if result.Module != "" {
|
||||||
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked)
|
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
|
||||||
} else {
|
} else {
|
||||||
planFailedBuild(ctx, b.open, result)
|
planFailedBuild(ctx, b.open, result)
|
||||||
}
|
}
|
||||||
@@ -270,18 +334,20 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
// Not a failure: the module is already at what a later request built. A plan that asked
|
// Not a failure: the module is already at what a later request built. A plan that asked
|
||||||
// before that later request is answered by it; one that asked after it ignores this.
|
// before that later request is answered by it; one that asked after it ignores this.
|
||||||
fmt.Printf("%s: %v\n", result.ID, err)
|
fmt.Printf("%s: %v\n", result.ID, err)
|
||||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked)
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||||
return nil
|
return nil
|
||||||
case err != nil:
|
case err != nil:
|
||||||
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
|
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
|
||||||
if manifest.Module != "" {
|
if manifest.Module != "" {
|
||||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked)
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked)
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||||
|
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||||
|
statusFrom.nudge()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
"crypto/ecdh"
|
"crypto/ecdh"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
@@ -37,6 +40,9 @@ func aMesh(t *testing.T) *stores {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(open.Close)
|
t.Cleanup(open.Close)
|
||||||
|
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
|
||||||
|
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
|
||||||
|
withConditionsInMemory(t)
|
||||||
for _, m := range provided {
|
for _, m := range provided {
|
||||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -106,3 +112,17 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
|||||||
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||||
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
|
||||||
|
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
|
||||||
|
t.Helper()
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
before := conditionsFrom
|
||||||
|
conditionsFrom = k
|
||||||
|
t.Cleanup(func() {
|
||||||
|
conditionsFrom = before
|
||||||
|
k.Close(context.Background())
|
||||||
|
})
|
||||||
|
return k, store
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,189 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
|
||||||
|
//
|
||||||
|
// The forge's poll announces every merge on the events stream, and the controller acts on what its
|
||||||
|
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
|
||||||
|
// server moved the consumer past it — a fault of consumers with several filters in the server the
|
||||||
|
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
|
||||||
|
// built from that repository stayed behind and were built by hand.
|
||||||
|
//
|
||||||
|
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
|
||||||
|
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
|
||||||
|
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
|
||||||
|
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
|
||||||
|
// would still move something was never acted on — it is said, and acted on now.
|
||||||
|
const (
|
||||||
|
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
|
||||||
|
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
|
||||||
|
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
|
||||||
|
mergeGrace = 10 * time.Minute
|
||||||
|
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
|
||||||
|
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
|
||||||
|
mergeLookBack = 24 * time.Hour
|
||||||
|
// mergeCatchUpEvery is how often the stream is read back.
|
||||||
|
mergeCatchUpEvery = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// merges is what reads back the forge's announcements; the link server, or a test's list.
|
||||||
|
type merges interface {
|
||||||
|
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||||
|
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||||
|
f := following{open}
|
||||||
|
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||||
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
read, err := open.inventory.ReadRepositories(ctx)
|
||||||
|
return entries, read, err
|
||||||
|
}
|
||||||
|
failing := ""
|
||||||
|
tick := time.NewTicker(mergeCatchUpEvery)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
watchedMerges.begin()
|
||||||
|
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||||
|
fmt.Printf(format+"\n", args...)
|
||||||
|
})
|
||||||
|
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||||
|
// says that instead, and leaves what the last one found standing.
|
||||||
|
watchedMerges.end(time.Now(), err)
|
||||||
|
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||||
|
why := ""
|
||||||
|
if err != nil {
|
||||||
|
why = err.Error()
|
||||||
|
}
|
||||||
|
if why != failing {
|
||||||
|
if why != "" {
|
||||||
|
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
|
||||||
|
} else {
|
||||||
|
fmt.Println("merges the bus may not have handed over are looked for again")
|
||||||
|
}
|
||||||
|
failing = why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
|
||||||
|
// move something is said and acted on, oldest first.
|
||||||
|
//
|
||||||
|
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
|
||||||
|
// because acting on an earlier missed merge of the same repository may have moved what a later one
|
||||||
|
// would have.
|
||||||
|
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||||
|
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
|
||||||
|
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
|
||||||
|
|
||||||
|
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
entries, read, err := catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, a := range all {
|
||||||
|
if now.Sub(a.At) < mergeGrace {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if entries, read, err = catalogued(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
moves := wouldMove(a.SourceMoved, entries, read)
|
||||||
|
if len(moves) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, e := range moves {
|
||||||
|
names = append(names, e.Manifest.Module)
|
||||||
|
}
|
||||||
|
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
||||||
|
At: a.At, Modules: names})
|
||||||
|
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||||
|
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||||
|
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||||
|
isAre(len(names)))
|
||||||
|
if err := act(ctx, a.SourceMoved); err != nil {
|
||||||
|
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
|
||||||
|
a.Owner, a.Repo, a.Commit, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if entries, read, err = catalogued(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
||||||
|
type missedMerge struct {
|
||||||
|
Owner, Repo, Base, Commit string
|
||||||
|
// At is when the bus took the announcement.
|
||||||
|
At time.Time
|
||||||
|
Modules []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
||||||
|
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
||||||
|
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
||||||
|
// the fault. The next pass, finding it acted on, clears it.
|
||||||
|
type mergeWatch struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
passed time.Time
|
||||||
|
err error
|
||||||
|
finding []missedMerge
|
||||||
|
missed []missedMerge
|
||||||
|
}
|
||||||
|
|
||||||
|
var watchedMerges = &mergeWatch{}
|
||||||
|
|
||||||
|
func (w *mergeWatch) begin() {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.finding = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *mergeWatch) found(m missedMerge) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.finding = append(w.finding, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *mergeWatch) end(at time.Time, err error) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.passed, w.err = at, err
|
||||||
|
if err == nil {
|
||||||
|
w.missed = w.finding
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
||||||
|
// could not read.
|
||||||
|
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
||||||
|
}
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// announcedList is the events stream's merges as a test gives them.
|
||||||
|
type announcedList []link.AnnouncedMerge
|
||||||
|
|
||||||
|
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||||
|
var out []link.AnnouncedMerge
|
||||||
|
for _, m := range a {
|
||||||
|
if !m.At.Before(since) {
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
|
||||||
|
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
|
||||||
|
type aCatalogue struct {
|
||||||
|
entries []inventory.Entry
|
||||||
|
acted []string
|
||||||
|
fail error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||||
|
return append([]inventory.Entry(nil), c.entries...), nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
|
||||||
|
return func(_ context.Context, m link.SourceMoved) error {
|
||||||
|
if c.fail != nil {
|
||||||
|
return c.fail
|
||||||
|
}
|
||||||
|
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
|
||||||
|
for _, moved := range wouldMove(m, c.entries, nil) {
|
||||||
|
for i := range c.entries {
|
||||||
|
if c.entries[i].Manifest.Module == moved.Manifest.Module {
|
||||||
|
c.entries[i].Source.Head = m.Commit
|
||||||
|
c.entries[i].Source.Seen = now()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func at(s string) time.Time {
|
||||||
|
t, err := time.Parse(time.RFC3339, s)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
|
||||||
|
return link.AnnouncedMerge{
|
||||||
|
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||||
|
MergedAt: mergedAt, Paths: paths,
|
||||||
|
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
|
||||||
|
At: at(onTheBus),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func built(module, repo, path, commit, seen string) inventory.Entry {
|
||||||
|
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
|
||||||
|
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
|
||||||
|
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
|
||||||
|
// events stream; the bus never handed it to the controller, which acted on the merges around it and
|
||||||
|
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
|
||||||
|
// move something — so it is said and acted on, once, and only after the controller's own consumer
|
||||||
|
// has had its time with it.
|
||||||
|
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
|
||||||
|
cat := &aCatalogue{entries: []inventory.Entry{
|
||||||
|
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||||
|
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||||
|
// Acted on when it was announced: looked at after it was merged.
|
||||||
|
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
|
||||||
|
}}
|
||||||
|
stream := announcedList{
|
||||||
|
// Nothing the mesh holds is built from the records repository.
|
||||||
|
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
|
||||||
|
// Acted on: its module was looked at since.
|
||||||
|
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
|
||||||
|
// Never handed over.
|
||||||
|
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
|
||||||
|
"node-tools/internal/console/console.go"),
|
||||||
|
}
|
||||||
|
var said []string
|
||||||
|
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||||
|
clock := at("2026-10-05T22:50:00Z")
|
||||||
|
now := func() time.Time { return clock }
|
||||||
|
pass := func() {
|
||||||
|
t.Helper()
|
||||||
|
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pass()
|
||||||
|
if len(cat.acted) != 0 {
|
||||||
|
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
|
||||||
|
}
|
||||||
|
|
||||||
|
clock = at("2026-10-05T22:58:00Z")
|
||||||
|
pass()
|
||||||
|
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
|
||||||
|
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
|
||||||
|
}
|
||||||
|
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
|
||||||
|
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
|
||||||
|
t.Fatalf("the missed merge was not said as one: %q", said)
|
||||||
|
}
|
||||||
|
|
||||||
|
clock = at("2026-10-05T23:03:00Z")
|
||||||
|
pass()
|
||||||
|
if len(cat.acted) != 1 || len(said) != 1 {
|
||||||
|
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
|
||||||
|
// that could not be acted on is said and tried again on the next pass.
|
||||||
|
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
|
||||||
|
cat := &aCatalogue{
|
||||||
|
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
|
||||||
|
fail: errors.New("the store is restarting"),
|
||||||
|
}
|
||||||
|
stream := announcedList{
|
||||||
|
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
|
||||||
|
"modules/plex/module.json", "modules/plex/x.ts"),
|
||||||
|
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
|
||||||
|
}
|
||||||
|
var said []string
|
||||||
|
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||||
|
clock := at("2026-10-05T22:00:00Z")
|
||||||
|
now := func() time.Time { return clock }
|
||||||
|
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
|
||||||
|
t.Fatalf("a failed catch-up was not said: %q", said)
|
||||||
|
}
|
||||||
|
cat.fail = nil
|
||||||
|
clock = at("2026-10-05T22:05:00Z")
|
||||||
|
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
|
||||||
|
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge older than the look-back is left to the operator: a controller that did not run this
|
||||||
|
// missed it, and acting on it days later would be a surprise rebuild.
|
||||||
|
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
|
||||||
|
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
|
||||||
|
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
|
||||||
|
clock := at("2026-10-05T22:00:00Z")
|
||||||
|
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
|
||||||
|
func(string, ...any) {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cat.acted) != 0 {
|
||||||
|
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -59,8 +59,19 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||||
if args[0] == "check" {
|
if args[0] == "check" {
|
||||||
|
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
||||||
|
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
||||||
|
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
||||||
|
"judge each module's identity on a machine name this many characters long")
|
||||||
|
given, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *longest < 1 {
|
||||||
|
return errors.New("--longest-machine-name is a length, at least 1")
|
||||||
|
}
|
||||||
var paths []string
|
var paths []string
|
||||||
for _, a := range args[1:] {
|
for _, a := range given {
|
||||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||||
under, err := manifestsUnder(a)
|
under, err := manifestsUnder(a)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -71,7 +82,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
paths = append(paths, a)
|
paths = append(paths, a)
|
||||||
}
|
}
|
||||||
return moduleCheck(paths, os.Stdout)
|
return moduleCheckFor(paths, *longest, os.Stdout)
|
||||||
}
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -334,8 +345,10 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func assignCommand(ctx context.Context, verb string, args []string) error {
|
func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||||
if len(args) != 2 {
|
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
|
||||||
return fmt.Errorf("%s <node> <module>", verb)
|
// service manager and the package manager — can only go on, or come off, together.
|
||||||
|
if len(args) < 2 {
|
||||||
|
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
|
||||||
}
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -349,7 +362,7 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
|
|||||||
if verb == "unassign" {
|
if verb == "unassign" {
|
||||||
act = unassign
|
act = unassign
|
||||||
}
|
}
|
||||||
said, err := act(ctx, open, args[0], args[1])
|
said, err := act(ctx, open, args[0], args[1:]...)
|
||||||
if said != "" {
|
if said != "" {
|
||||||
fmt.Println(said)
|
fmt.Println(said)
|
||||||
}
|
}
|
||||||
@@ -659,7 +672,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
|||||||
// durable subscription nobody reads.
|
// durable subscription nobody reads.
|
||||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||||
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||||
}); needed {
|
}); needed {
|
||||||
if busAddress == "" {
|
if busAddress == "" {
|
||||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||||
|
|||||||
@@ -275,25 +275,9 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// No registry trust is composed here any more: the container runtime's module states it, told
|
||||||
|
// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
|
||||||
g, err := overlay.From(nodes, cidr, "")
|
g, err := overlay.From(nodes, cidr, "")
|
||||||
if g != nil {
|
|
||||||
// The artifact store, as this network reaches it. Found rather than configured: the
|
|
||||||
// provider is whichever module offers it, on whichever machine holds that module — and if
|
|
||||||
// nothing does yet (genesis raises the registry before the catalogue knows it), there is
|
|
||||||
// no trust to write and nothing is written (novox/hq ADR 0082).
|
|
||||||
//
|
|
||||||
// Refused rather than composed without it when the question could not be answered: a
|
|
||||||
// declaration missing the trust because a lookup failed is a machine that cannot pull,
|
|
||||||
// delivered by a push that reported success — and nothing recomposes it until the next
|
|
||||||
// push (the shape of novox/hq issues 042/048, reappearing as a race).
|
|
||||||
at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
|
|
||||||
if storeErr != nil {
|
|
||||||
return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
|
|
||||||
}
|
|
||||||
if found {
|
|
||||||
g.TrustRegistry(overlay.InternalName(at) + ":" + port)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil && len(refused) > 0 {
|
if err != nil && len(refused) > 0 {
|
||||||
// The network is missing something, and some machines could not be resolved at all. Those
|
// The network is missing something, and some machines could not be resolved at all. Those
|
||||||
// are almost always the same fact: a node that does not resolve contributes nothing, so
|
// are almost always the same fact: a node that does not resolve contributes nothing, so
|
||||||
|
|||||||
@@ -254,11 +254,10 @@ func theResolver(t *testing.T) catalogue.Manifest {
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// The resolver is handed every machine on the private network as a wildcard, the same set and the
|
// The resolver is handed every machine on the private network as a wildcard, and is handed it again
|
||||||
// same source as the hosts file, and is handed it again when a machine leaves — through the
|
// when a machine leaves — through the module's own manifest asking for the fact, with no module of the
|
||||||
// module's own manifest asking for the fact, with no module of the mesh's own in between (hal
|
// mesh's own in between (hal dnsmasq-app conversion, novox/hq 08-connectivity). It is the mesh's one
|
||||||
// dnsmasq-app conversion, novox/hq 08-connectivity). The runtime on that machine is pointed at the
|
// resolver (ADR 0194), and the container runtime is given no resolver of its own (ADR 0196).
|
||||||
// machine's own address, where the resolver answers for its containers.
|
|
||||||
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
|
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -293,11 +292,12 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
|||||||
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
|
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The container runtime is given no resolver of its own (novox/hq ADR 0196): it copies its
|
||||||
|
// machine's, which name the mesh's resolver first. A `dns` key would be a second account of where a
|
||||||
|
// container asks, read only when the runtime starts.
|
||||||
for _, r := range composed(t, open, "anchor").Resources {
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
if r["id"] == "dnsmasq.runtime-dns" {
|
if r["id"] == "dnsmasq.runtime-dns" {
|
||||||
if !strings.Contains(r["content"].(string), `"10.77.0.1"`) || r["into"] != "json" {
|
t.Errorf("the resolver still writes the runtime's own dns: %v", r)
|
||||||
t.Errorf("the runtime is not pointed at this machine's own address, written into its file: %v", r)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -386,8 +387,12 @@ func brokerCommand(ctx context.Context, args []string) error {
|
|||||||
if len(args) > 0 && args[0] == "accounts" {
|
if len(args) > 0 && args[0] == "accounts" {
|
||||||
return busAccounts(ctx, args[1:])
|
return busAccounts(ctx, args[1:])
|
||||||
}
|
}
|
||||||
|
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||||
|
return consumerReset(ctx, args[1:])
|
||||||
|
}
|
||||||
if len(args) == 0 || args[0] != "show" {
|
if len(args) == 0 || args[0] != "show" {
|
||||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file>")
|
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||||
|
"broker consumer-reset <stream> <consumer>")
|
||||||
}
|
}
|
||||||
known, err := broker.FromEnvironment()
|
known, err := broker.FromEnvironment()
|
||||||
if errors.Is(err, broker.ErrNotConfigured) {
|
if errors.Is(err, broker.ErrNotConfigured) {
|
||||||
@@ -407,6 +412,45 @@ func brokerCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||||
|
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||||
|
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||||
|
func consumerReset(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||||
|
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||||
|
why := addHandActFlags(set)
|
||||||
|
args, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(args) != 2 {
|
||||||
|
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||||
|
}
|
||||||
|
if err := why.require("broker consumer-reset"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
why.record(ctx, "broker consumer-reset", args)
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
before, after, err := js.ResetConsumer(args[0], args[1])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("consumer %s on %s re-made to deliver from now\n", args[1], args[0])
|
||||||
|
fmt.Printf(" before: delivers %s, delivered to %d, acknowledged to %d, %d pending, %d unacknowledged\n",
|
||||||
|
before.DeliverPolicy, before.Delivered, before.AckFloor, before.Pending, before.AckPending)
|
||||||
|
fmt.Printf(" after: delivers %s, %d pending; what was pending is dropped. A holder bound to it may need its "+
|
||||||
|
"process restarted to bind again\n", after.DeliverPolicy, after.Pending)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
||||||
//
|
//
|
||||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
||||||
@@ -473,6 +517,26 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
|||||||
fmt.Printf(" public domain %s\n", domain)
|
fmt.Printf(" public domain %s\n", domain)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
|
||||||
|
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
|
||||||
|
// is something not working now and they are a description. Unreadable is said, not passed over.
|
||||||
|
open, err := openConditions(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
|
||||||
|
}
|
||||||
|
var here []conditions.Condition
|
||||||
|
for _, c := range open {
|
||||||
|
if concerns(c, name) {
|
||||||
|
here = append(here, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(here) > 0 {
|
||||||
|
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
|
||||||
|
for _, line := range conditionLines(here, time.Now()) {
|
||||||
|
fmt.Printf(" %s\n", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
held, err := inv.Profile(ctx, name)
|
held, err := inv.Profile(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -147,6 +147,14 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
|||||||
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
|
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
|
||||||
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
|
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
|
||||||
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
|
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
|
||||||
|
// novox/hq issue 252: a module the mesh has never registered is still a module, when the merge
|
||||||
|
// shows it is one — and a directory that may be shared code is still shared.
|
||||||
|
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
|
||||||
|
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
|
||||||
|
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
|
||||||
|
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), "gitea,keycloak"},
|
||||||
|
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), "gitea,keycloak"},
|
||||||
|
{"the root's files still", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||||
} {
|
} {
|
||||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||||
|
|||||||
+292
-32
@@ -8,8 +8,10 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
@@ -127,6 +129,7 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
|||||||
// a mesh-wide gatherer may pass over — see notResolvable.
|
// a mesh-wide gatherer may pass over — see notResolvable.
|
||||||
return catalogue.Resolution{}, nil, notResolvable{err}
|
return catalogue.Resolution{}, nil, notResolvable{err}
|
||||||
}
|
}
|
||||||
|
logUnheld(nodeName, resolved.Unheld)
|
||||||
|
|
||||||
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
|
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
|
||||||
// password a provider is told to create is the one its consumer was given — and sealed to
|
// password a provider is told to create is the one its consumer was given — and sealed to
|
||||||
@@ -394,9 +397,49 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return sendable{}, err
|
return sendable{}, err
|
||||||
}
|
}
|
||||||
return sendable{Resources: composed.Resources, Adoption: adoption,
|
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||||
Received: composed.Received, Mesh: with.Mesh,
|
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
|
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld}
|
||||||
|
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||||
|
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||||
|
if choosing == Allocating {
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
if !known {
|
||||||
|
before = nil
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(plan.Modules))
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
names = append(names, m.Module)
|
||||||
|
}
|
||||||
|
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
|
||||||
|
// issue 259): the module's current build for each module in it, and for a module left out of it
|
||||||
|
// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
|
||||||
|
// that is not known. Never nil, so a send through here always records what it knows.
|
||||||
|
func carriedBuilds(modules []string, leftOut map[string]string, current map[string]inventory.CurrentBuild,
|
||||||
|
before map[string]string) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
for _, m := range modules {
|
||||||
|
if _, left := leftOut[m]; left {
|
||||||
|
if was, kept := before[m]; kept {
|
||||||
|
out[m] = was
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[m] = current[m].Commit
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
|
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
|
||||||
@@ -421,6 +464,11 @@ func reportLeftOut(node string, declared sendable) {
|
|||||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||||
node, m, declared.leftOutWhy[m])
|
node, m, declared.leftOutWhy[m])
|
||||||
}
|
}
|
||||||
|
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||||
|
// 0225): the machine is sent everything else, and the consumer is named.
|
||||||
|
for _, o := range declared.withheld {
|
||||||
|
fmt.Printf("%s: %s\n", node, o)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||||
@@ -428,7 +476,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
grants, err := grantsFor(ctx, open, node)
|
grants, withheld, err := grantsFor(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
@@ -657,6 +705,20 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
machines[name] = at
|
machines[name] = at
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// And every zone a module in the mesh answers itself (novox/hq ADR 0199), for the mesh's resolver
|
||||||
|
// to forward.
|
||||||
|
zones, err := zonesInTheMesh(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
|
||||||
|
// lists every holder of the mesh's resolver.
|
||||||
|
replicas, err := replicatedHolders(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
|
|
||||||
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
||||||
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
||||||
// before it had an address on the private network. A machine joins through the tunnel now, and
|
// before it had an address on the private network. A machine joins through the tunnel now, and
|
||||||
@@ -722,18 +784,90 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
|
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
|
||||||
|
// 0222): the artifact store as this network reaches it, which the container runtime is told to
|
||||||
|
// trust (ADR 0082). The same address composed into every reference the mesh built.
|
||||||
|
var reach map[string]string
|
||||||
|
if artifactStore != "" {
|
||||||
|
reach = map[string]string{"mesh-artifact-store": artifactStore}
|
||||||
|
}
|
||||||
return catalogue.Rendering{
|
return catalogue.Rendering{
|
||||||
BusMembership: memberships[node],
|
BusMembership: memberships[node],
|
||||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||||
Machines: machines,
|
Machines: machines, Zones: zones, Holders: replicas,
|
||||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||||
BusUsers: busUsers,
|
BusUsers: busUsers, Withheld: withheld,
|
||||||
}, record, nil
|
}, record, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||||
|
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||||
|
// answering listen is published on there.
|
||||||
|
//
|
||||||
|
// Read across every machine's resolution, as the roster once read routed names: a node whose set does
|
||||||
|
// not compose declares nothing and is passed over, so one broken machine does not cost the rest their
|
||||||
|
// zones; a store that cannot be read is raised, naming the machine, because returning the zones
|
||||||
|
// without it would withdraw them from the resolver as if the operator had (novox/hq 04-ISSUES/152).
|
||||||
|
// What the mesh refuses about the zones together — one declared twice, one shadowing the mesh's
|
||||||
|
// suffix or a node's public domain — is refused here, by name.
|
||||||
|
func zonesInTheMesh(ctx context.Context, open *stores) ([]catalogue.ZoneAt, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
places, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
address := map[string]string{}
|
||||||
|
for _, p := range places {
|
||||||
|
if strings.TrimSpace(p.Address) != "" {
|
||||||
|
address[p.Name] = p.Address
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var zones []catalogue.ZoneAt
|
||||||
|
var public []string
|
||||||
|
for _, n := range nodes {
|
||||||
|
plan, _, err := planFor(ctx, open, n.Name)
|
||||||
|
switch {
|
||||||
|
case unresolvable(err):
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("the zones %s answers cannot be read: %w", n.Name, err)
|
||||||
|
}
|
||||||
|
if plan.PublicDomain != "" {
|
||||||
|
public = append(public, plan.PublicDomain)
|
||||||
|
}
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
if m.Zone == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at := address[n.Name]
|
||||||
|
if at == "" {
|
||||||
|
// Not on the private network yet: nothing could reach its answerer.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
published, layers, err := portsGivenOn(ctx, inv, n.Name, m)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the zone %s declares on %s cannot be read: %w", m.Module, n.Name, err)
|
||||||
|
}
|
||||||
|
z, err := catalogue.ZoneOn(m, layers, published, n.Name, at)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
zones = append(zones, *z)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if problems := catalogue.ZonesProblems(zones, overlay.Suffix(), public); len(problems) > 0 {
|
||||||
|
return nil, fmt.Errorf("the mesh's zones cannot be forwarded:\n - %s", strings.Join(problems, "\n - "))
|
||||||
|
}
|
||||||
|
return zones, nil
|
||||||
|
}
|
||||||
|
|
||||||
// certificateFor is what the mesh certifies about one machine's internal name.
|
// certificateFor is what the mesh certifies about one machine's internal name.
|
||||||
//
|
//
|
||||||
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
|
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
|
||||||
@@ -801,28 +935,34 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
|
|||||||
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
||||||
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
||||||
// the mesh hands over something it cannot itself use.
|
// the mesh hands over something it cannot itself use.
|
||||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
//
|
||||||
|
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
|
||||||
|
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
|
||||||
|
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
|
||||||
|
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
|
||||||
|
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, []catalogue.Overflow, error) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
issued, err := inv.SecretsFrom(ctx, node)
|
issued, err := inv.SecretsFrom(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
||||||
// the mesh names machines.
|
// the mesh names machines.
|
||||||
shelf, err := inv.Catalogue(ctx)
|
shelf, err := inv.Catalogue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
||||||
// from a record beside it. A provider told to create a password and not what to create it for
|
// from a record beside it. A provider told to create a password and not what to create it for
|
||||||
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
||||||
out := make([]catalogue.Grant, 0, len(issued))
|
out := make([]catalogue.Grant, 0, len(issued))
|
||||||
|
var withheld []catalogue.Overflow
|
||||||
for _, s := range issued {
|
for _, s := range issued {
|
||||||
plan, settings, err := planFor(ctx, open, s.Consumer)
|
plan, settings, err := planFor(ctx, open, s.Consumer)
|
||||||
switch {
|
switch {
|
||||||
@@ -835,11 +975,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
||||||
// nothing — and withholding a grant on that reading takes a consumer's access away
|
// nothing — and withholding a grant on that reading takes a consumer's access away
|
||||||
// (novox/hq 04-ISSUES/152).
|
// (novox/hq 04-ISSUES/152).
|
||||||
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
return nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||||
}
|
}
|
||||||
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
// A port in there is the consumer's software port until this. The consumer is on another
|
// A port in there is the consumer's software port until this. The consumer is on another
|
||||||
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
||||||
@@ -847,7 +987,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
||||||
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
||||||
from := s.ConsumerModule
|
from := s.ConsumerModule
|
||||||
@@ -858,9 +998,10 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
from = ""
|
from = ""
|
||||||
}
|
}
|
||||||
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
||||||
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
|
||||||
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
// this provision, as the consumer's resolution states it from the provider's offer (ADR
|
||||||
// remedy a short slug rather than a login a provider silently shortened.
|
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
|
||||||
|
// provider silently shortened — and rather than this whole machine refused for it.
|
||||||
slug := ""
|
slug := ""
|
||||||
for _, mm := range plan.Modules {
|
for _, mm := range plan.Modules {
|
||||||
if mm.Module == s.ConsumerModule {
|
if mm.Module == s.ConsumerModule {
|
||||||
@@ -869,15 +1010,32 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if from != "" {
|
if from != "" {
|
||||||
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
bound := boundOfGrant(plan, s, node)
|
||||||
return nil, err
|
source := catalogue.IdentitySource(slug, s.ConsumerModule)
|
||||||
|
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
|
||||||
|
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
|
||||||
|
Consumer: s.Consumer, Module: s.ConsumerModule,
|
||||||
|
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out = append(out, catalogue.Grant{
|
out = append(out, catalogue.Grant{
|
||||||
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
||||||
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, withheld, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
|
||||||
|
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
|
||||||
|
// than to none: what the provider keeps of it is not known here.
|
||||||
|
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
|
||||||
|
for _, n := range consumer.Needs {
|
||||||
|
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
|
||||||
|
return n.Identity
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return catalogue.DefaultIdentityBound
|
||||||
}
|
}
|
||||||
|
|
||||||
// listensLines is what a person is told about what this module would open, and why — the same
|
// listensLines is what a person is told about what this module would open, and why — the same
|
||||||
@@ -953,6 +1111,11 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
left := plan.LeftOut(settings, record.Adopted)
|
left := plan.LeftOut(settings, record.Adopted)
|
||||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||||
}
|
}
|
||||||
|
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||||
|
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||||
|
for _, o := range plan.Overflowing() {
|
||||||
|
fmt.Printf("%s: %s\n", args[0], o)
|
||||||
|
}
|
||||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||||
// stored before its definition moved from under it.
|
// stored before its definition moved from under it.
|
||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
@@ -1266,6 +1429,20 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
|||||||
//
|
//
|
||||||
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
|
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
|
||||||
// resource of that module, so the question is whether it is here.
|
// resource of that module, so the question is whether it is here.
|
||||||
|
list, missing, err := busUserList(ctx, inv, onThisNode)
|
||||||
|
if len(missing) > 0 {
|
||||||
|
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||||
|
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||||
|
len(missing), strings.Join(missing, ", "))
|
||||||
|
}
|
||||||
|
return list, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// busUserList is composeBusUsers without saying anything: the list, and the users left out of it
|
||||||
|
// for want of a credential. Asked on every send to decide whether the machine holding the bus must
|
||||||
|
// go first (novox/hq issue 249), where saying the same missing users each time would bury them.
|
||||||
|
func busUserList(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
onThisNode []catalogue.Manifest) (string, []string, error) {
|
||||||
holdsTheBus := false
|
holdsTheBus := false
|
||||||
for _, m := range onThisNode {
|
for _, m := range onThisNode {
|
||||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||||
@@ -1273,37 +1450,33 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !holdsTheBus {
|
if !holdsTheBus {
|
||||||
return "", nil
|
return "", nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
records, err := inv.BusRecords(ctx)
|
records, err := inv.BusRecords(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", nil, err
|
||||||
}
|
}
|
||||||
users, err := broker.Users(records)
|
users, err := broker.Users(records)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", nil, err
|
||||||
}
|
}
|
||||||
kept, err := inv.BusUsers(ctx)
|
kept, err := inv.BusUsers(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", nil, err
|
||||||
}
|
}
|
||||||
hashes := make(map[string]string, len(kept))
|
hashes := make(map[string]string, len(kept))
|
||||||
for name, u := range kept {
|
for name, u := range kept {
|
||||||
hashes[name] = u.PasswordHash
|
hashes[name] = u.PasswordHash
|
||||||
}
|
}
|
||||||
filled, missing := broker.WithPasswords(users, hashes)
|
filled, missing := broker.WithPasswords(users, hashes)
|
||||||
if len(missing) > 0 {
|
|
||||||
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
|
||||||
"for: %s. Each is a user that cannot connect until one is issued\n",
|
|
||||||
len(missing), strings.Join(missing, ", "))
|
|
||||||
}
|
|
||||||
if len(filled) == 0 {
|
if len(filled) == 0 {
|
||||||
return "", fmt.Errorf(
|
return "", missing, fmt.Errorf(
|
||||||
"this machine runs the bus and not one user has a credential, so the composed list " +
|
"this machine runs the bus and not one user has a credential, so the composed list " +
|
||||||
"would refuse every connection in the mesh")
|
"would refuse every connection in the mesh")
|
||||||
}
|
}
|
||||||
return broker.ComposeAccounts(filled)
|
list, err := broker.ComposeAccounts(filled)
|
||||||
|
return list, missing, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
||||||
@@ -1325,3 +1498,90 @@ func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.C
|
|||||||
}
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
|
||||||
|
// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
|
||||||
|
//
|
||||||
|
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
|
||||||
|
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
|
||||||
|
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
|
||||||
|
// burying the line about the node it acted on. A command says what concerns its own act instead
|
||||||
|
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
|
||||||
|
var (
|
||||||
|
unheldLogged = map[string]string{}
|
||||||
|
unheldLoggedMu sync.Mutex
|
||||||
|
logUnheldChanges bool
|
||||||
|
)
|
||||||
|
|
||||||
|
// logUnheld logs a node's unmet seat dependencies when they differ from what was last logged for
|
||||||
|
// it, including when they become none — in the serving controller, and nowhere else.
|
||||||
|
func logUnheld(node string, unheld []catalogue.Unheld) {
|
||||||
|
if !logUnheldChanges {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
lines := make([]string, 0, len(unheld))
|
||||||
|
for _, u := range unheld {
|
||||||
|
lines = append(lines, u.String())
|
||||||
|
}
|
||||||
|
now := strings.Join(lines, "\n")
|
||||||
|
unheldLoggedMu.Lock()
|
||||||
|
before, seen := unheldLogged[node]
|
||||||
|
unheldLogged[node] = now
|
||||||
|
unheldLoggedMu.Unlock()
|
||||||
|
if (seen && before == now) || (!seen && now == "") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if now == "" {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
|
||||||
|
node, len(lines), strings.Join(lines, "\n "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over
|
||||||
|
// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not —
|
||||||
|
// which includes every one of a module just assigned — and each now met that was not. Nothing about
|
||||||
|
// any other node, and nothing that was already true before the act.
|
||||||
|
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
|
||||||
|
judge := func(names []string) map[string]catalogue.Unheld {
|
||||||
|
var set []catalogue.Manifest
|
||||||
|
for _, n := range names {
|
||||||
|
if m, known := shelf[n]; known {
|
||||||
|
set = append(set, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out := map[string]catalogue.Unheld{}
|
||||||
|
for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
|
||||||
|
out[u.Module+" "+u.Seat] = u
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
was, now := judge(before), judge(after)
|
||||||
|
var lines []string
|
||||||
|
for _, k := range sortedNames(now) {
|
||||||
|
if _, already := was[k]; !already {
|
||||||
|
lines = append(lines, "but "+now[k].String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, k := range sortedNames(was) {
|
||||||
|
if _, still := now[k]; still {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
u := was[k]
|
||||||
|
if !slices.Contains(after, u.Module) {
|
||||||
|
continue // went with its module, which says nothing about the seat
|
||||||
|
}
|
||||||
|
lines = append(lines, fmt.Sprintf("and %s on %s now has %s held", u.Module, node, u.Seat))
|
||||||
|
}
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedNames[V any](m map[string]V) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,401 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A plan follows what is done to the build queue (novox/hq ADR 0219).
|
||||||
|
//
|
||||||
|
// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
|
||||||
|
//
|
||||||
|
// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
|
||||||
|
// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
|
||||||
|
// sends a reader looking for a fault that is a decision. It says what it waits on instead.
|
||||||
|
// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
|
||||||
|
// failed plan's failed modules and the plan goes on from that tier as if they had built the
|
||||||
|
// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
|
||||||
|
// beside it — beside it, the plan would either ask it again or stay failed on an outcome the
|
||||||
|
// rebuild has already replaced.
|
||||||
|
|
||||||
|
// pauseView is whether the build seat takes work: the holders that said they are paused, and
|
||||||
|
// whether that is every holder.
|
||||||
|
type pauseView struct {
|
||||||
|
Nodes []string
|
||||||
|
All bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
|
||||||
|
// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
|
||||||
|
func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
|
||||||
|
if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
var asked *time.Time
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
|
||||||
|
if asked == nil || s.AskedAt.Before(*asked) {
|
||||||
|
asked = s.AskedAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if asked == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
waited := now.Sub(*asked)
|
||||||
|
said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
|
||||||
|
strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
|
||||||
|
// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
|
||||||
|
// longer than a day is named.
|
||||||
|
if waited > pausedTooLong {
|
||||||
|
said += " — PAUSED OVER A DAY: `resume` takes builds again"
|
||||||
|
}
|
||||||
|
return said, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
|
||||||
|
const pausedTooLong = 24 * time.Hour
|
||||||
|
|
||||||
|
// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
|
||||||
|
// where the seat being paused changes what a plan says.
|
||||||
|
func awaitsABuild(plans []inventory.Plan) bool {
|
||||||
|
for _, p := range plans {
|
||||||
|
if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
|
||||||
|
// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
|
||||||
|
// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
|
||||||
|
// reads as late, which is what it said before this existed.
|
||||||
|
func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
|
||||||
|
if !awaitsABuild(plans) {
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
seat := buildSeatAmong(entries)
|
||||||
|
holders := holdersAmong(entries, seat)
|
||||||
|
if len(holders) == 0 {
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
said, err := link.PausedSaid(js, seat, holders)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
return pauseOf(holders, said)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pauseOf is the view from what each holder said.
|
||||||
|
func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
|
||||||
|
var v pauseView
|
||||||
|
for _, n := range holders {
|
||||||
|
if said[n].Paused {
|
||||||
|
v.Nodes = append(v.Nodes, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(v.Nodes)
|
||||||
|
v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedIn is the modules of a plan's current tier that failed to build, sorted.
|
||||||
|
func failedIn(p inventory.Plan) []string {
|
||||||
|
if p.Tier >= len(p.Tiers) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
|
||||||
|
// that holds what this one held now (issue 254, ADR 0218).
|
||||||
|
func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||||
|
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return q, true
|
||||||
|
}
|
||||||
|
return inventory.Plan{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||||
|
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||||
|
switch {
|
||||||
|
case p.State == inventory.PlanDone:
|
||||||
|
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||||
|
case p.State == inventory.PlanSuperseded:
|
||||||
|
return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
|
||||||
|
case p.Open():
|
||||||
|
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||||
|
}
|
||||||
|
if len(failedIn(p)) > 0 {
|
||||||
|
if q, found := newerOpenPlan(p, plans); found {
|
||||||
|
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||||
|
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
stopped := stoppedRollouts(p)
|
||||||
|
if len(stopped) == 0 {
|
||||||
|
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
|
||||||
|
p.Tier, p.ID, p.Note)
|
||||||
|
}
|
||||||
|
// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
|
||||||
|
// sent — a newer build, and sending this one again would put the older build back on its machines.
|
||||||
|
for _, m := range stopped {
|
||||||
|
if q, found := newerPlanFor(m, p, plans); found {
|
||||||
|
return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
|
||||||
|
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
|
||||||
|
// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
|
||||||
|
func stoppedRollouts(p inventory.Plan) []string {
|
||||||
|
if p.Tier >= len(p.Tiers) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
|
||||||
|
len(s.First) > 0 && s.Why != "" {
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
|
||||||
|
func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, tier := range q.Tiers {
|
||||||
|
for _, m := range tier {
|
||||||
|
if m == module {
|
||||||
|
return q, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inventory.Plan{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
|
||||||
|
// variable so a test of a retried rollout needs no machine.
|
||||||
|
var sendRollout = sendToEach
|
||||||
|
|
||||||
|
// resumed sets a failed plan building again once nothing in its tier is failed.
|
||||||
|
func resumed(p *inventory.Plan, why string) {
|
||||||
|
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||||
|
p.State = inventory.PlanBuilding
|
||||||
|
p.Note = why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||||
|
// that tier: what follows is the plan going on as if they had built the first time.
|
||||||
|
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
p, err := inv.PlanByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
recent, err := inv.RecentPlans(ctx, 50)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
plans = append(plans, recent...)
|
||||||
|
if err := retryRefusal(p, plans); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if len(failedIn(p)) == 0 {
|
||||||
|
return retryRollouts(ctx, open, &p)
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
byName := map[string]inventory.Entry{}
|
||||||
|
for _, e := range entries {
|
||||||
|
byName[e.Manifest.Module] = e
|
||||||
|
}
|
||||||
|
failed := failedIn(p)
|
||||||
|
var asked []string
|
||||||
|
for _, m := range failed {
|
||||||
|
askModule(ctx, &p, m, byName)
|
||||||
|
if s := p.Modules[m]; s.State == "asked" {
|
||||||
|
asked = append(asked, m+" as "+s.Build)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if p.State != inventory.PlanBuilding {
|
||||||
|
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
|
||||||
|
p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
|
||||||
|
// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
|
||||||
|
func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
openPlans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
recent, err := inv.RecentPlans(ctx, 20)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
p, found := planHolding(module, openPlans, recent)
|
||||||
|
if !found {
|
||||||
|
return false, "", nil
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
byName := map[string]inventory.Entry{}
|
||||||
|
for _, e := range entries {
|
||||||
|
byName[e.Manifest.Module] = e
|
||||||
|
}
|
||||||
|
was := p.State
|
||||||
|
askModule(ctx, &p, module, byName)
|
||||||
|
s := p.Modules[module]
|
||||||
|
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
if s.State != "asked" {
|
||||||
|
return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
|
||||||
|
}
|
||||||
|
said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
|
||||||
|
s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
|
||||||
|
switch {
|
||||||
|
case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
|
||||||
|
said += "; the plan had failed and builds again from this tier"
|
||||||
|
case was == inventory.PlanFailed:
|
||||||
|
said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
|
||||||
|
}
|
||||||
|
return true, said, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
|
||||||
|
// yet built, else the newest failed plan whose current tier does, and that no open plan of its
|
||||||
|
// repository supersedes.
|
||||||
|
func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
|
||||||
|
holds := func(p inventory.Plan) bool {
|
||||||
|
if p.Tier >= len(p.Tiers) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if m == module {
|
||||||
|
s := p.Modules[m]
|
||||||
|
return s == nil || s.State != "built"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, p := range openPlans {
|
||||||
|
if holds(p) {
|
||||||
|
return p, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sorted := append([]inventory.Plan(nil), recent...)
|
||||||
|
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
|
||||||
|
for _, p := range sorted {
|
||||||
|
if p.State != inventory.PlanFailed || !holds(p) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, superseded := newerOpenPlan(p, openPlans); superseded {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return p, true
|
||||||
|
}
|
||||||
|
return inventory.Plan{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
|
||||||
|
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
|
||||||
|
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
|
||||||
|
func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
|
||||||
|
var said []string
|
||||||
|
for _, m := range stoppedRollouts(*p) {
|
||||||
|
s := p.Modules[m]
|
||||||
|
sent, err := sendRollout(ctx, open, s.First)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
|
||||||
|
m, strings.Join(s.First, ", "), p.ID, err)
|
||||||
|
}
|
||||||
|
now := time.Now().UTC()
|
||||||
|
s.First, s.FirstAt, s.Why = sent, &now, ""
|
||||||
|
said = append(said, m+" to "+strings.Join(sent, ", "))
|
||||||
|
}
|
||||||
|
p.State = inventory.PlanRolling
|
||||||
|
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||||
|
if err := open.inventory.SavePlan(ctx, *p); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
|
||||||
|
"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,820 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/micro"
|
||||||
|
"github.com/novox/mesh-host/validate"
|
||||||
|
"golang.org/x/net/dns/dnsmessage"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The probes of the self-check's registry (doctor.go), one function each. A probe answers what is
|
||||||
|
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
||||||
|
// look" (ADR 0227 rule 4).
|
||||||
|
|
||||||
|
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
||||||
|
// (mesh-host's `validate`, the package the host runs) takes it.
|
||||||
|
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
open := d.open
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
// The private network every declaration is composed with. Not computable is not "could not
|
||||||
|
// look": it is the finding — no machine's declaration composes without it — and the machines that
|
||||||
|
// do not resolve, which are usually why, are named below.
|
||||||
|
gens, gensErr := generators(ctx, open)
|
||||||
|
if gensErr != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "private-network",
|
||||||
|
Token: "uncomposable", Severity: conditions.Urgent,
|
||||||
|
Summary: "the private network cannot be computed, so no machine's declaration composes",
|
||||||
|
Said: firstLine(gensErr.Error())})
|
||||||
|
}
|
||||||
|
for _, n := range nodes {
|
||||||
|
plan, settings, err := planFor(ctx, open, n.Name)
|
||||||
|
if err != nil && !unresolvable(err) {
|
||||||
|
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
if err == nil && gensErr == nil {
|
||||||
|
var declared sendable
|
||||||
|
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
||||||
|
var body []byte
|
||||||
|
if body, err = declared.Body(); err == nil {
|
||||||
|
problems = validate.Declaration(body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "uncomposable",
|
||||||
|
Machine: n.Name, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("nothing can be sent to %s: its declaration does not compose — %s", n.Name,
|
||||||
|
oneLine(err.Error())),
|
||||||
|
Said: oneLine(err.Error())})
|
||||||
|
case len(problems) > 0:
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "refused",
|
||||||
|
Machine: n.Name, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
||||||
|
n.Name, len(problems), problems[0]),
|
||||||
|
Said: strings.Join(problems, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
||||||
|
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
||||||
|
// takes as final (issue 262).
|
||||||
|
func probeResolvers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
holders, err := replicatedHolders(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resolvers := holders["mesh-dns-resolver"]
|
||||||
|
if len(resolvers) == 0 {
|
||||||
|
return nil, nil // the mesh holds no resolver of its own: nothing is asked of one
|
||||||
|
}
|
||||||
|
places, err := onTheNetwork(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
suffix := overlay.Suffix()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for holder, at := range resolvers {
|
||||||
|
var wrong []string
|
||||||
|
for _, p := range places {
|
||||||
|
if p.Address == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := p.Name + "." + suffix
|
||||||
|
v4, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeA)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "A", name, err))
|
||||||
|
continue
|
||||||
|
case rcode != dnsmessage.RCodeSuccess:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (A)", holder, rcode, name))
|
||||||
|
case !slices.Contains(v4, p.Address):
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (A), not %s", holder, v4, name, p.Address))
|
||||||
|
}
|
||||||
|
v6, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeAAAA)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "AAAA", name, err))
|
||||||
|
case rcode != dnsmessage.RCodeSuccess:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (AAAA), not NODATA: a musl machine "+
|
||||||
|
"takes that as no such name", holder, rcode, name))
|
||||||
|
case len(v6) > 0:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (AAAA); the mesh has no IPv6 addresses", holder, v6, name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(wrong) > 0 {
|
||||||
|
node := strings.TrimSuffix(holder, "."+suffix)
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver." + node,
|
||||||
|
Token: "wrong", Machine: node, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s",
|
||||||
|
node, wrong[0]),
|
||||||
|
Said: strings.Join(wrong, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolverPort is where a resolver answers; a variable so a test can stand one up.
|
||||||
|
var resolverPort = "53"
|
||||||
|
|
||||||
|
// askResolver asks one resolver one question over UDP, and answers the addresses and the code.
|
||||||
|
func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]string, dnsmessage.RCode, error) {
|
||||||
|
q, err := dnsmessage.NewName(strings.TrimSuffix(name, ".") + ".")
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
msg := dnsmessage.Message{Header: dnsmessage.Header{ID: uint16(time.Now().UnixNano()), RecursionDesired: true},
|
||||||
|
Questions: []dnsmessage.Question{{Name: q, Type: kind, Class: dnsmessage.ClassINET}}}
|
||||||
|
packed, err := msg.Pack()
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
dialer := net.Dialer{Timeout: 3 * time.Second}
|
||||||
|
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(at, resolverPort))
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
|
||||||
|
if _, err := conn.Write(packed); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
buf := make([]byte, 1500)
|
||||||
|
n, err := conn.Read(buf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("no answer from %s: %w", at, err)
|
||||||
|
}
|
||||||
|
var answer dnsmessage.Message
|
||||||
|
if err := answer.Unpack(buf[:n]); err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("an answer from %s that cannot be read: %w", at, err)
|
||||||
|
}
|
||||||
|
var addresses []string
|
||||||
|
for _, a := range answer.Answers {
|
||||||
|
switch r := a.Body.(type) {
|
||||||
|
case *dnsmessage.AResource:
|
||||||
|
addresses = append(addresses, net.IP(r.A[:]).String())
|
||||||
|
case *dnsmessage.AAAAResource:
|
||||||
|
addresses = append(addresses, net.IP(r.AAAA[:]).String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return addresses, answer.RCode, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
||||||
|
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
||||||
|
// S1's, and is not asked about here.
|
||||||
|
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
entries, err := d.open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
recorded, err := d.open.inventory.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
expected := map[string]map[string]bool{} // seat → machine
|
||||||
|
add := func(seat, node string) {
|
||||||
|
if expected[seat] == nil {
|
||||||
|
expected[seat] = map[string]bool{}
|
||||||
|
}
|
||||||
|
expected[seat][node] = true
|
||||||
|
}
|
||||||
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||||
|
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
||||||
|
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
||||||
|
}
|
||||||
|
onRecord := false
|
||||||
|
for _, h := range recorded {
|
||||||
|
if h.Claim == s.Name && heard[h.Node] {
|
||||||
|
add(s.Name, h.Node)
|
||||||
|
onRecord = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if c.Name != s.Name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, node := range e.On {
|
||||||
|
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
||||||
|
add(s.Name, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(expected) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
answering, err := discoverHolders(ctx, d.js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
for seat, nodes := range expected {
|
||||||
|
for node := range nodes {
|
||||||
|
if answering[seat][node] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: seat + "." + node,
|
||||||
|
Token: "silent", Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s's holder on %s does not answer the bus: its verbs reach nothing there", seat, node),
|
||||||
|
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
||||||
|
func heardMachines(d *doctor) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
if d.watchdogs == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
f := d.watchdogs.lastFacts()
|
||||||
|
if f == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||||
|
out[m.name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveryPatience is how long the discovery's answers are waited for after the last arrived, and at
|
||||||
|
// the most (ADR 0197: a large runtime's answer arrives last).
|
||||||
|
const (
|
||||||
|
discoveryQuiet = 1500 * time.Millisecond
|
||||||
|
discoveryPatience = 8 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// discoverHolders asks the bus's discovery who serves what, and answers seat → machine for every
|
||||||
|
// endpoint a seat's verb is served on.
|
||||||
|
func discoverHolders(ctx context.Context, conn *nats.Conn) (map[string]map[string]bool, error) {
|
||||||
|
inbox := conn.NewRespInbox()
|
||||||
|
sub, err := conn.SubscribeSync(inbox)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||||
|
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||||
|
}
|
||||||
|
out := map[string]map[string]bool{}
|
||||||
|
deadline := time.Now().Add(discoveryPatience)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||||
|
msg, err := sub.NextMsgWithContext(wait)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var info micro.Info
|
||||||
|
if json.Unmarshal(msg.Data, &info) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range info.Endpoints {
|
||||||
|
seat, node := e.Metadata["seat"], e.Metadata["node"]
|
||||||
|
if seat == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if node == "" {
|
||||||
|
node = info.ID
|
||||||
|
}
|
||||||
|
if out[seat] == nil {
|
||||||
|
out[seat] = map[string]bool{}
|
||||||
|
}
|
||||||
|
out[seat][node] = true
|
||||||
|
}
|
||||||
|
// A holder that announces itself as its seat, by name and machine.
|
||||||
|
if out[info.Name] == nil {
|
||||||
|
out[info.Name] = map[string]bool{}
|
||||||
|
}
|
||||||
|
out[info.Name][info.ID] = true
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeArchives is D4: every archive the mesh keeps is held by its manifest in the artifact store.
|
||||||
|
func probeArchives(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
kept, err := inv.KeptArchives(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(kept) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
store, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if store == "" {
|
||||||
|
return nil, errors.New("the mesh keeps archives and has no artifact store on its network to ask")
|
||||||
|
}
|
||||||
|
var report collectionReport
|
||||||
|
if unasked, stopped := askHeld(ctx, artifacts.Store{Address: store}, kept, &report); stopped != "" {
|
||||||
|
return nil, fmt.Errorf("%d kept archive(s) could not be asked about: %s", unasked, stopped)
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
if n := len(report.Unheld); n > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-unheld",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%d of %d kept archive(s) are not held by a manifest: the store's collector would "+
|
||||||
|
"delete them", n, len(kept)),
|
||||||
|
Said: "first: " + report.Unheld[0]})
|
||||||
|
}
|
||||||
|
if n := len(report.Missing); n > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-missing",
|
||||||
|
Kind: "archives-missing", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%d kept archive(s) are not in the artifact store at all", n),
|
||||||
|
Said: "first: " + report.Missing[0]})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerFarBehind is how far a durable consumer may be from its stream's head (D6).
|
||||||
|
const consumerFarBehind = 1000
|
||||||
|
|
||||||
|
// probeConsumers is D6: every durable consumer the mesh expects exists, as the controller defines it,
|
||||||
|
// and is near its stream's head.
|
||||||
|
func probeConsumers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
_, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js := d.js.Context()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range consumers {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
info, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||||
|
who := c.Stream + "." + c.Name
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "missing",
|
||||||
|
Kind: "consumer-lost", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(c)),
|
||||||
|
Said: "no consumer " + c.Name + " on " + c.Stream})
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||||
|
}
|
||||||
|
if differs := consumerDiffers(c, info.Config); differs != "" {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "redefined",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is not as the controller defines it: %s", consumerWords(c), differs),
|
||||||
|
Said: differs})
|
||||||
|
}
|
||||||
|
if c.Stream != "NODES" && info.NumPending > consumerFarBehind {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(c), info.NumPending),
|
||||||
|
Said: fmt.Sprintf("%d pending, %d handed out and not settled", info.NumPending, info.NumAckPending)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerWords is a consumer as the mesh says it, with its name.
|
||||||
|
func consumerWords(c broker.Consumer) string {
|
||||||
|
return fmt.Sprintf("%s (%s on %s)", link.ConsumerInWords(c.Stream, c.Name), c.Name, c.Stream)
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerDiffers is what about a consumer on the bus is not as defined; empty when nothing is. The
|
||||||
|
// delivery subject is not compared: one kept as it was while a holder is bound is the assertion's
|
||||||
|
// stated choice (novox/hq issue 156).
|
||||||
|
func consumerDiffers(want broker.Consumer, have nats.ConsumerConfig) string {
|
||||||
|
var differs []string
|
||||||
|
haveFilters := append([]string(nil), have.FilterSubjects...)
|
||||||
|
if have.FilterSubject != "" {
|
||||||
|
haveFilters = append(haveFilters, have.FilterSubject)
|
||||||
|
}
|
||||||
|
wantFilters := append([]string(nil), want.Filters...)
|
||||||
|
sort.Strings(haveFilters)
|
||||||
|
sort.Strings(wantFilters)
|
||||||
|
if !slices.Equal(haveFilters, wantFilters) {
|
||||||
|
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
||||||
|
}
|
||||||
|
if have.AckPolicy != nats.AckExplicitPolicy {
|
||||||
|
differs = append(differs, "acknowledges "+have.AckPolicy.String()+", defined explicit")
|
||||||
|
}
|
||||||
|
if wantMax := want.MaxDeliver; wantMax != 0 && have.MaxDeliver != wantMax {
|
||||||
|
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, wantMax))
|
||||||
|
}
|
||||||
|
if wantWait := time.Duration(want.AckWaitSeconds) * time.Second; wantWait != 0 && have.AckWait != wantWait {
|
||||||
|
differs = append(differs, fmt.Sprintf("waits %s for an acknowledgement, defined %s", have.AckWait, wantWait))
|
||||||
|
}
|
||||||
|
if want.MaxAckPending != 0 && have.MaxAckPending != want.MaxAckPending {
|
||||||
|
differs = append(differs, fmt.Sprintf("hands out %d at once, defined %d", have.MaxAckPending, want.MaxAckPending))
|
||||||
|
}
|
||||||
|
if wantPush, havePush := want.Push || want.Queue != "", have.DeliverSubject != ""; wantPush != havePush {
|
||||||
|
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
||||||
|
}
|
||||||
|
return strings.Join(differs, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeStreams is D7: every stream the controller defines exists as defined, and its own buckets.
|
||||||
|
func probeStreams(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
streams, _, err := expectedBusObjects(ctx, d.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js := d.js.Context()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, s := range streams {
|
||||||
|
info, err := js.StreamInfo(s.Name, nats.Context(ctx))
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrStreamNotFound):
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-missing",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||||
|
Said: "no stream " + s.Name})
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
if differs := streamDiffers(s, info.Config); differs != "" {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-redefined",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the stream %s is not as the controller defines it: %s", s.Name, differs),
|
||||||
|
Said: differs})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bucket := range broker.ControllerBuckets() {
|
||||||
|
if _, err := js.StreamInfo("KV_"+bucket, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: bucket, Token: "bucket-missing",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the controller's bucket %s is not on the bus: what it keeps there is not kept", bucket),
|
||||||
|
Said: "no bucket " + bucket})
|
||||||
|
} else if err != nil {
|
||||||
|
return nil, fmt.Errorf("the bucket %s cannot be read: %w", bucket, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamDiffers is what about a stream on the bus is not as defined; empty when nothing is.
|
||||||
|
func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||||
|
var differs []string
|
||||||
|
haveSubjects := append([]string(nil), have.Subjects...)
|
||||||
|
wantSubjects := append([]string(nil), want.Subjects...)
|
||||||
|
sort.Strings(haveSubjects)
|
||||||
|
sort.Strings(wantSubjects)
|
||||||
|
if !slices.Equal(haveSubjects, wantSubjects) {
|
||||||
|
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
||||||
|
}
|
||||||
|
retention, perSubject := nats.LimitsPolicy, int64(want.MaxMsgsPerSubject)
|
||||||
|
switch want.Retention {
|
||||||
|
case broker.RetentionWorkQueue:
|
||||||
|
retention = nats.WorkQueuePolicy
|
||||||
|
case broker.RetentionLastPerSubject:
|
||||||
|
perSubject = 1
|
||||||
|
}
|
||||||
|
if have.Retention != retention {
|
||||||
|
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
||||||
|
}
|
||||||
|
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
||||||
|
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
||||||
|
}
|
||||||
|
return strings.Join(differs, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ipLike finds the addresses in a ban list, whatever shape its holder answers in.
|
||||||
|
var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
||||||
|
|
||||||
|
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are
|
||||||
|
// every machine's private address and the address its endpoint names; the ban lists are asked of each
|
||||||
|
// machine's holder of `node-intrusion-prevention` that is heard from.
|
||||||
|
func probeBans(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
places, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
owned := map[string]string{} // address → whose
|
||||||
|
for _, p := range places {
|
||||||
|
if p.Address != "" {
|
||||||
|
owned[p.Address] = p.Name + "'s private address"
|
||||||
|
}
|
||||||
|
if host, _, err := net.SplitHostPort(p.Endpoint); err == nil && host != "" {
|
||||||
|
if ip := net.ParseIP(host); ip != nil {
|
||||||
|
owned[ip.String()] = p.Name + "'s endpoint"
|
||||||
|
} else if addrs, err := net.DefaultResolver.LookupHost(ctx, host); err == nil {
|
||||||
|
for _, a := range addrs {
|
||||||
|
owned[a] = p.Name + "'s endpoint (" + host + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
var holders []string
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if c.Name == "node-intrusion-prevention" {
|
||||||
|
for _, node := range e.On {
|
||||||
|
if heard[node] && !slices.Contains(holders, node) {
|
||||||
|
holders = append(holders, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(holders)
|
||||||
|
var out []conditions.Observation
|
||||||
|
// Every machine asked at once: one after another, four holders that each wait their bound
|
||||||
|
// outlast the probe's thirty seconds, and the probe says nothing about any of them.
|
||||||
|
answers := make([]json.RawMessage, len(holders))
|
||||||
|
errs := make([]error, len(holders))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, node := range holders {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, node string) {
|
||||||
|
defer wg.Done()
|
||||||
|
answers[i], errs[i] = askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
|
||||||
|
}(i, node)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
var unasked []string
|
||||||
|
for i, node := range holders {
|
||||||
|
answer, err := answers[i], errs[i]
|
||||||
|
if err != nil {
|
||||||
|
unasked = append(unasked, err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var banned []string
|
||||||
|
for _, ip := range ipLike.FindAllString(string(answer), -1) {
|
||||||
|
if whose, ours := owned[ip]; ours && !slices.Contains(banned, ip+" ("+whose+")") {
|
||||||
|
banned = append(banned, ip+" ("+whose+")")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(banned) > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "bans-the-mesh",
|
||||||
|
Machine: node, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es): %s — the mesh is locked out "+
|
||||||
|
"of itself there (ADR 0186)", node, len(banned), strings.Join(banned, ", ")),
|
||||||
|
Said: strings.Join(banned, ", ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unasked) > 0 {
|
||||||
|
return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; "))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeStatus is D9: `status` answers in full within ten seconds, from a summary composed lately.
|
||||||
|
func probeStatus(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
started := time.Now()
|
||||||
|
stale := ""
|
||||||
|
if statusFrom != nil {
|
||||||
|
answer, err := statusFrom.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
stale = err.Error()
|
||||||
|
} else if m, ok := answer.(map[string]any); ok {
|
||||||
|
if failed, _ := m["lastAttemptFailed"].(string); failed != "" {
|
||||||
|
stale = "its last composition failed: " + failed
|
||||||
|
}
|
||||||
|
if composed, err := time.Parse(time.RFC3339, fmt.Sprint(m["composed"])); err == nil &&
|
||||||
|
time.Since(composed) > 3*statusEvery+statusComposeWithin {
|
||||||
|
stale = fmt.Sprintf("it answers a summary composed %s ago", time.Since(composed).Round(time.Second))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if _, err := theThreeQuestions(ctx, d.open); err != nil {
|
||||||
|
stale = err.Error()
|
||||||
|
}
|
||||||
|
took := time.Since(started)
|
||||||
|
var out []conditions.Observation
|
||||||
|
if took > 10*time.Second || stale != "" {
|
||||||
|
why := stale
|
||||||
|
if why == "" {
|
||||||
|
why = fmt.Sprintf("it took %s", took.Round(time.Millisecond))
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller", Token: "status-slow",
|
||||||
|
Machine: d.host, Severity: conditions.Warning,
|
||||||
|
Summary: "status does not answer in full within ten seconds: " + why, Said: why})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeCoreBuilds is D10: every machine runs the node-engine and the node tools the mesh holds, or a
|
||||||
|
// plan is rolling one of them out. The node-engine says its build in each report; the node tools' is
|
||||||
|
// the build the machine was last sent, once it reported applying that send.
|
||||||
|
func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
hostVersions := deliveredVersions(shelf[hostModule])
|
||||||
|
rolling := map[string]bool{}
|
||||||
|
for _, p := range plans {
|
||||||
|
for m := range p.Modules {
|
||||||
|
rolling[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
applied := map[string]bool{}
|
||||||
|
for _, r := range reports {
|
||||||
|
applied[r.Node] = r.Current
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, n := range nodes {
|
||||||
|
if !heard[n.Name] {
|
||||||
|
continue // a machine not heard from is S1's
|
||||||
|
}
|
||||||
|
var behind []string
|
||||||
|
// **A node-engine says its version as the directory it was delivered into** — its archive's
|
||||||
|
// digest, twelve characters (catalogue.versionOf, ADR 0141) — not the commit it was built
|
||||||
|
// from. Compared as a commit, every machine read as behind right after a push sent it the
|
||||||
|
// current one (2026-10-06). An engine placed by hand reports its link-time stamp instead,
|
||||||
|
// which a commit can match.
|
||||||
|
if !rolling[hostModule] && engineBehind(n.HostVersion, hostVersions, current[hostModule].Commit) {
|
||||||
|
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)",
|
||||||
|
n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit)))
|
||||||
|
}
|
||||||
|
assigned, err := inv.Assigned(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tools := current[broker.RuntimeModule].Commit
|
||||||
|
if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] {
|
||||||
|
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !known:
|
||||||
|
case !sameCommit(sent[broker.RuntimeModule], tools):
|
||||||
|
behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s",
|
||||||
|
short(orNotKnown(sent[broker.RuntimeModule])), short(tools)))
|
||||||
|
case !applied[n.Name]:
|
||||||
|
behind = append(behind, "the node tools it was last sent, not yet reported applied")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(behind) > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind",
|
||||||
|
Machine: n.Name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s runs %s, and no plan is rolling them out", n.Name, strings.Join(behind, "; ")),
|
||||||
|
Said: strings.Join(behind, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
||||||
|
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
||||||
|
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
||||||
|
func deliveredVersions(m catalogue.Manifest) []string {
|
||||||
|
var out []string
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
path, _ := r["path"].(string)
|
||||||
|
before, version, found := strings.Cut(path, "/versions/")
|
||||||
|
if !found || before == "" || version == "" || strings.Contains(version, "/") || strings.Contains(version, "$") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !slices.Contains(out, version) {
|
||||||
|
out = append(out, version)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineBehind says a node-engine's reported version is not the build the mesh holds: neither the
|
||||||
|
// directory that build is delivered as, nor (for an engine placed by hand) its commit. A machine that
|
||||||
|
// has not said, or a mesh that holds no delivered build, is not behind anything.
|
||||||
|
func engineBehind(reported string, delivered []string, commit string) bool {
|
||||||
|
if reported == "" || len(delivered) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return !slices.Contains(delivered, reported) && !sameCommit(reported, commit)
|
||||||
|
}
|
||||||
|
|
||||||
|
// hostModule is the node-engine's module.
|
||||||
|
const hostModule = "mesh-host"
|
||||||
|
|
||||||
|
// sameCommit says two commits are one, either written short.
|
||||||
|
func sameCommit(a, b string) bool {
|
||||||
|
if a == "" || b == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNotKnown(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "(not known)"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeWatchdogs is DW: the watchdogs ran within three of their intervals. The doctor and the
|
||||||
|
// watchdogs watch each other: S10 is the other half.
|
||||||
|
func probeWatchdogs(_ context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
if d.watchdogs == nil {
|
||||||
|
return nil, nil // a self-check run outside the serving controller has none to watch
|
||||||
|
}
|
||||||
|
ticked := d.watchdogs.lastTick()
|
||||||
|
since := ticked
|
||||||
|
if since.IsZero() {
|
||||||
|
since = d.watchdogs.started
|
||||||
|
}
|
||||||
|
if time.Since(since) <= 3*watchEvery {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "watchdogs", Token: "silent",
|
||||||
|
Machine: d.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the watchdogs of the signals table have not run since %s: no late signal is being said",
|
||||||
|
since.UTC().Format("2006-01-02 15:04 MST")),
|
||||||
|
Said: fmt.Sprintf("no tick for %s", time.Since(since).Round(time.Second))}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// askSeatTool asks one machine's holder of a node seat a verb and answers its result; the holder's
|
||||||
|
// own refusal is an error.
|
||||||
|
func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string) (json.RawMessage, error) {
|
||||||
|
if who, declared := declaredBy(ctx, seat, verb); !declared {
|
||||||
|
return nil, fmt.Errorf("%s asks %s.%s, which it does not declare in the probe registry — and so the "+
|
||||||
|
"controller is not granted it", who, seat, verb)
|
||||||
|
}
|
||||||
|
answer, err := link.AskSeatTool(ctx, conn, seat, verb, node, map[string]any{}, 10*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if answer.Error != "" {
|
||||||
|
return nil, fmt.Errorf("%s's %s.%s refused: %s", node, seat, verb, answer.Error)
|
||||||
|
}
|
||||||
|
return answer.Result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// oneLine is a message of several lines said on one, its runs of space made one.
|
||||||
|
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||||
+494
-188
@@ -8,6 +8,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -62,6 +64,9 @@ func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.En
|
|||||||
}
|
}
|
||||||
|
|
||||||
func serve(ctx context.Context) error {
|
func serve(ctx context.Context) error {
|
||||||
|
// The one process whose log is read over time, so the one that says each change to a node's
|
||||||
|
// unmet seat dependencies once (novox/hq ADR 0207).
|
||||||
|
logUnheldChanges = true
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -101,7 +106,10 @@ func serve(ctx context.Context) error {
|
|||||||
|
|
||||||
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
||||||
OnNATS: true}
|
OnNATS: true}
|
||||||
server, err := connectLink(ctx, inv, work, work)
|
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
|
||||||
|
// something new is one thing that moves it, so the listener nudges it.
|
||||||
|
statusFrom = newStatusSummary(composeStatus(open))
|
||||||
|
server, err := connectLink(ctx, inv, work, nudgingListener{work, statusFrom})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -117,17 +125,27 @@ func serve(ctx context.Context) error {
|
|||||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||||
go planTicker(ctx, open)
|
go planTicker(ctx, open)
|
||||||
|
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||||
|
go forgettingOldDurations(ctx, inv)
|
||||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||||
// above; this is the other half — the control plane is the only one of the three that knows
|
// above; this is the other half — the control plane is the only one of the three that knows
|
||||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||||
if err := server.Follows(following{open}); err != nil {
|
if err := server.Follows(following{open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// And the merges the bus announced and never handed over, read back on a timer and acted on late
|
||||||
|
// rather than never (novox/hq issue 266).
|
||||||
|
go catchingUpOnMerges(ctx, open, server)
|
||||||
// And a catalogue that has just started, asking for what it missed. The same type answers
|
// And a catalogue that has just started, asking for what it missed. The same type answers
|
||||||
// both: what a build meant and what the builds were are two questions about one record.
|
// both: what a build meant and what the builds were are two questions about one record.
|
||||||
if err := server.Answers(following{open}); err != nil {
|
if err := server.Answers(following{open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||||
|
// 0224): a provider's journal must not be the only place that says so.
|
||||||
|
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
|
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
|
||||||
// from the store's row, so what the seat declares is what is answered.
|
// from the store's row, so what the seat declares is what is answered.
|
||||||
@@ -146,6 +164,28 @@ func serve(ctx context.Context) error {
|
|||||||
if !isNATS {
|
if !isNATS {
|
||||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||||
}
|
}
|
||||||
|
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||||
|
handActConn = bus.Conn
|
||||||
|
// Composed now and kept current, before the verb that answers from it is served.
|
||||||
|
go statusFrom.keep(ctx)
|
||||||
|
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
||||||
|
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
||||||
|
// And every call is kept on the bus, so a restart of this process keeps what came of each
|
||||||
|
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
|
||||||
|
// answering no calls at all would be worse than answering them without the record.
|
||||||
|
said := log.New(os.Stdout, "", log.LstdFlags)
|
||||||
|
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
|
||||||
|
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
|
||||||
|
} else if err := link.Calls.Durably(ctx, keeper, controllerProcess(), said); err != nil {
|
||||||
|
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
|
||||||
|
}
|
||||||
|
// What is wrong, kept and said (novox/hq to-be 45 §2): the condition store, the watchdogs of the
|
||||||
|
// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
|
||||||
|
// is said and the controller serves on: status then says the conditions cannot be read, and is
|
||||||
|
// not well — louder than not serving at all, and the push that repairs the bus still runs.
|
||||||
|
if stopWatching := watchTheMesh(ctx, open, server, bus); stopWatching != nil {
|
||||||
|
defer stopWatching()
|
||||||
|
}
|
||||||
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -213,8 +253,9 @@ func declare(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
|
// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
|
||||||
// is still what the machine was last told, and status must not read it as current for the one
|
// is still what the machine was last told, and status must not read it as current for the one
|
||||||
// the mesh would compose.
|
// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
|
||||||
if _, err := recordSent(ctx, inv, node, raw); err != nil {
|
// the mesh did not compose it, so a push that does not name this machine treats it as held.
|
||||||
|
if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||||
@@ -245,6 +286,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
||||||
wait := set.Duration("wait", 0,
|
wait := set.Duration("wait", 0,
|
||||||
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
||||||
|
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
|
||||||
|
// recorded when given at a shell — see handacts.go for why a shell is not refused.
|
||||||
|
why := addHandActFlags(set)
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -259,6 +303,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
||||||
"other asks which machines need one")
|
"other asks which machines need one")
|
||||||
}
|
}
|
||||||
|
recorded := append([]string(nil), args...)
|
||||||
|
if *behind {
|
||||||
|
recorded = append(recorded, "--behind")
|
||||||
|
}
|
||||||
|
why.record(ctx, "push", recorded)
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -312,7 +361,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
if len(needsOne) == 0 {
|
if len(needsOne) == 0 {
|
||||||
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
|
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
|
||||||
// must never look the same.
|
// must never look the same.
|
||||||
fmt.Println("every machine is doing what it was told")
|
fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -353,6 +402,46 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
asked = append(asked, n.Name)
|
asked = append(asked, n.Name)
|
||||||
}
|
}
|
||||||
|
// **A push that named no machine says so, first.** Through the console a machine the caller
|
||||||
|
// meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as
|
||||||
|
// `push --behind`, and every machine behind was pushed by someone who thought they had pushed one.
|
||||||
|
// Its whole-mesh reach is the first line of the answer, with the machines it is about to send.
|
||||||
|
if len(args) == 0 {
|
||||||
|
which := "every machine"
|
||||||
|
if *behind {
|
||||||
|
which = "every machine that is behind"
|
||||||
|
}
|
||||||
|
fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n",
|
||||||
|
which, len(asked), strings.Join(asked, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's
|
||||||
|
// user list, and a module's new grants are refused by the bus until that list says them. Among
|
||||||
|
// the machines asked it goes first; not among them and behind, it is added — a named push whose
|
||||||
|
// module gained a state would otherwise send the code and leave the right to use it for the
|
||||||
|
// cascade below, after.
|
||||||
|
holder, holderBehind, err := brokerBehind(ctx, open, asked)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// **Not when its own modules are held back** (novox/hq issue 259, ADR 0221): added rather than
|
||||||
|
// named, it is sent its whole declaration, and a build its policy records or a plan has not sent
|
||||||
|
// it yet would go with the user list. Named and left, with what that costs.
|
||||||
|
saidHeld := map[string]bool{}
|
||||||
|
if holderBehind && len(args) == 1 {
|
||||||
|
held, err := heldMachines(ctx, open, []string{holder})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if why, isHeld := held[holder]; isHeld {
|
||||||
|
sayHeld(os.Stdout, holder, why)
|
||||||
|
fmt.Printf("%s holds the bus, and the user list it would carry has changed: until it is "+
|
||||||
|
"sent, the bus may refuse what this push's machines were newly granted\n", holder)
|
||||||
|
holderBehind = false
|
||||||
|
saidHeld[holder] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
asked = brokerFirst(asked, holder, holderBehind)
|
||||||
|
|
||||||
// Held from composing to sending, so a converge on one of them cannot send between the two
|
// Held from composing to sending, so a converge on one of them cannot send between the two
|
||||||
// and be overtaken by what was composed before it (novox/hq ADR 0100).
|
// and be overtaken by what was composed before it (novox/hq ADR 0100).
|
||||||
@@ -360,6 +449,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// Each machine's unmet seat dependencies (novox/hq ADR 0207), said after the sends: in full
|
||||||
|
// for a machine named, as a count for each of many — the full list is `status`'s.
|
||||||
|
unheld := map[string][]catalogue.Unheld{}
|
||||||
sending, refusals := composeEach(asked, allotting(held, inv), func(node string) (sendable, error) {
|
sending, refusals := composeEach(asked, allotting(held, inv), func(node string) (sendable, error) {
|
||||||
plan, settings, err := planFor(held, open, node)
|
plan, settings, err := planFor(held, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -369,6 +461,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
// healthy modules beside it are still resolved and sent. Reported so it is not silently
|
// healthy modules beside it are still resolved and sent. Reported so it is not silently
|
||||||
// dropped — the remedy is to move it, and until then the rest of the node converges.
|
// dropped — the remedy is to move it, and until then the rest of the node converges.
|
||||||
reportUnhostable(node, plan)
|
reportUnhostable(node, plan)
|
||||||
|
unheld[node] = plan.Unheld
|
||||||
// The private network is in here with everything else. It used to be composed separately
|
// The private network is in here with everything else. It used to be composed separately
|
||||||
// and prepended, which meant every machine with an address was on it and no machine could
|
// and prepended, which meant every machine with an address was on it and no machine could
|
||||||
// be kept off. It is a module now, so it arrives the way a module does.
|
// be kept off. It is a module now, so it arrives the way a module does.
|
||||||
@@ -379,34 +472,21 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
return declared, err
|
return declared, err
|
||||||
})
|
})
|
||||||
|
|
||||||
sentDigest := map[string]string{}
|
|
||||||
defer release()
|
defer release()
|
||||||
for _, s := range sending {
|
// Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push
|
||||||
// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
|
// is the one most operators run, and on 2026-10-01 it was the one path that issued none.
|
||||||
// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
|
bus := overTheBus{open: open, server: server, signer: ident}
|
||||||
body, err := s.declared.Body()
|
sentDigest, err := deliver(ctx, bus, holder, sending)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// After it is away, not before. A digest recorded for something that failed to send would
|
|
||||||
// make the machine look current for a declaration it never received.
|
|
||||||
digest, err := recordSent(ctx, inv, s.node, body)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
sentDigest[s.node] = digest
|
|
||||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
|
||||||
}
|
|
||||||
release()
|
|
||||||
fmt.Printf("\n%d node(s) told\n", len(sending))
|
|
||||||
// And each machine's memberships, as every other send does (ADR 0160): a push is the one most
|
|
||||||
// operators run, and on 2026-10-01 it was the one path that issued none.
|
|
||||||
if err := issueMemberships(ctx, open, server, sending); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
release()
|
||||||
|
told := make([]string, 0, len(sending))
|
||||||
|
for _, r := range sending {
|
||||||
|
told = append(told, r.node)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", "))
|
||||||
|
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
|
||||||
|
|
||||||
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
||||||
// issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's
|
// issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's
|
||||||
@@ -416,86 +496,21 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
//
|
//
|
||||||
// Compared against what each machine was last SENT, not against a before/after of this push:
|
// Compared against what each machine was last SENT, not against a before/after of this push:
|
||||||
// the mint usually happened at `assign` or `module issue`, before this command ran, so the
|
// the mint usually happened at `assign` or `module issue`, before this command ran, so the
|
||||||
// only durable signal is "what it should be" versus "what it last received". A machine behind
|
// only durable signal is "what it should be" versus "what it last received". Bounded: a
|
||||||
// for an unrelated reason is caught here too, which is not a cost — a named push that knew a
|
|
||||||
// machine was behind and left it so would be the very silence this removes. Bounded: a
|
|
||||||
// flushed send may itself mint, so this converges over a few rounds.
|
// flushed send may itself mint, so this converges over a few rounds.
|
||||||
|
//
|
||||||
|
// **Except a machine a policy or a plan holds back** (novox/hq issue 259, ADR 0221): one whose
|
||||||
|
// modules would move to a build their upgrade policy records rather than rolls out, or that an
|
||||||
|
// open plan has not sent it yet. It is named, with why, and left for a push that names it.
|
||||||
if len(args) == 1 {
|
if len(args) == 1 {
|
||||||
flushed := map[string]bool{args[0]: true}
|
handled := map[string]bool{args[0]: true}
|
||||||
// Bounded by the node count: a node is marked flushed the round it is handled and is
|
for n := range saidHeld {
|
||||||
// never handled twice, so the loop cannot run more than len(nodes) rounds. The bound is
|
handled[n] = true
|
||||||
// a guard against a logic error, not a real limit — if it were ever hit, that is a bug
|
}
|
||||||
// rather than a cascade legitimately still converging, so it is said rather than passed
|
refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, os.Stdout)
|
||||||
// over in silence, unlike the earlier fixed cap that could stop a real cascade short.
|
refusals = append(refusals, refused...)
|
||||||
rounds := 0
|
if err != nil {
|
||||||
for {
|
return err
|
||||||
would, err := wouldSend(ctx, open, nodes)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
behind, err := inv.Waiting(ctx, would)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var also []string
|
|
||||||
for _, m := range behind {
|
|
||||||
if !flushed[m.Node] {
|
|
||||||
also = append(also, m.Node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(also) == 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if rounds++; rounds > len(nodes) {
|
|
||||||
fmt.Printf("\nstopped cascading after %d rounds with %s still behind — this "+
|
|
||||||
"should not happen; run `push --behind` to finish\n",
|
|
||||||
rounds-1, strings.Join(also, ", "))
|
|
||||||
break
|
|
||||||
}
|
|
||||||
sort.Strings(also)
|
|
||||||
fmt.Printf("\nthis push left %s behind — a provision granted from there, or a "+
|
|
||||||
"declaration since changed; sending it too\n", strings.Join(also, ", "))
|
|
||||||
// Tolerantly, exactly as the named send above: a machine that cannot be composed is
|
|
||||||
// collected as a refusal and reported at the end, and the others are still sent
|
|
||||||
// (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is
|
|
||||||
// all-or-nothing — so one swept machine's compose error failed the operator's named
|
|
||||||
// push and skipped its --wait, the very intolerance the main path exists to avoid.
|
|
||||||
// Held for this round only, and after the last round's were given back, so two pushes
|
|
||||||
// cascading into each other's machines never each wait on the other.
|
|
||||||
refused, err := sendRound(ctx, open, also,
|
|
||||||
func(held context.Context, node string) (sendable, error) {
|
|
||||||
plan, settings, err := planFor(held, open, node)
|
|
||||||
if err != nil {
|
|
||||||
return sendable{}, err
|
|
||||||
}
|
|
||||||
reportUnhostable(node, plan)
|
|
||||||
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
|
||||||
if err == nil {
|
|
||||||
reportLeftOut(node, declared)
|
|
||||||
}
|
|
||||||
return declared, err
|
|
||||||
},
|
|
||||||
func(s readyNode, body []byte) error {
|
|
||||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body,
|
|
||||||
15*time.Second); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := recordSent(ctx, inv, s.node, body); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
refusals = append(refusals, refused...)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Every candidate this round is marked handled — the sent ones so they are not
|
|
||||||
// re-listed, and the refused ones so a machine that cannot be composed does not make
|
|
||||||
// the loop spin on it for ever. Its refusal is already in the report.
|
|
||||||
for _, name := range also {
|
|
||||||
flushed[name] = true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -612,10 +627,10 @@ func composeEach(names []string, allot func(node string) (int64, error),
|
|||||||
// sendRound holds the named nodes, composes each and sends each that composed, and gives the hold
|
// sendRound holds the named nodes, composes each and sends each that composed, and gives the hold
|
||||||
// back on every way out — a body that cannot be marshalled and a send that fails included
|
// back on every way out — a body that cannot be marshalled and a send that fails included
|
||||||
// (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are
|
// (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are
|
||||||
// still sent.
|
// still sent. Their memberships go before their declarations, as every send's do (issue 249).
|
||||||
func sendRound(ctx context.Context, open *stores, names []string,
|
func sendRound(ctx context.Context, open *stores, names []string,
|
||||||
compose func(held context.Context, node string) (sendable, error),
|
compose func(held context.Context, node string) (sendable, error),
|
||||||
send func(s readyNode, body []byte) error) ([]string, error) {
|
d delivery, holder string) ([]string, error) {
|
||||||
held, release, err := holdNodes(ctx, open, names)
|
held, release, err := holdNodes(ctx, open, names)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -624,18 +639,246 @@ func sendRound(ctx context.Context, open *stores, names []string,
|
|||||||
sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
|
sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
|
||||||
return compose(held, node)
|
return compose(held, node)
|
||||||
})
|
})
|
||||||
for _, s := range sending {
|
if _, err := deliver(held, d, holder, sending); err != nil {
|
||||||
body, err := s.declared.Body()
|
return refused, err
|
||||||
if err != nil {
|
|
||||||
return refused, err
|
|
||||||
}
|
|
||||||
if err := send(s, body); err != nil {
|
|
||||||
return refused, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return refused, nil
|
return refused, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// delivery is the two acts of sending machines what they should be, apart, so the order between
|
||||||
|
// them is one function's and can be read and tested there (novox/hq issue 249).
|
||||||
|
type delivery interface {
|
||||||
|
// grant issues what the machines' modules may do — each module's state raised and its
|
||||||
|
// membership issued — for every machine about to be sent.
|
||||||
|
grant(ctx context.Context, sending []readyNode) error
|
||||||
|
// declare sends one machine its declaration and records it sent, answering the digest.
|
||||||
|
declare(ctx context.Context, s readyNode, body []byte) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// errGrants marks a send that stopped because what the machines' modules may do could not be issued
|
||||||
|
// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an
|
||||||
|
// announcement that hits it is held and asked again.
|
||||||
|
var errGrants = errors.New("what the machines' modules may do on the bus could not be issued, and code " +
|
||||||
|
"sent before its grants is refused there")
|
||||||
|
|
||||||
|
// grantsRefused is a grant that failed for some machines and not others: their memberships could not
|
||||||
|
// be issued, by machine, and only those machines are held back.
|
||||||
|
type grantsRefused struct{ nodes map[string]error }
|
||||||
|
|
||||||
|
func (g *grantsRefused) Error() string {
|
||||||
|
names := make([]string, 0, len(g.nodes))
|
||||||
|
for n := range g.nodes {
|
||||||
|
names = append(names, n)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return fmt.Sprintf("the memberships of %s could not be issued; the first: %v",
|
||||||
|
strings.Join(names, ", "), g.nodes[names[0]])
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliver sends the machines their declarations: **the machine holding the bus, then the grants,
|
||||||
|
// then the rest** (novox/hq issue 249).
|
||||||
|
//
|
||||||
|
// A merge gave a module a new state; its bundle reached every machine within a minute, and the
|
||||||
|
// machines' permissions on the bus did not include the state until somebody pushed by hand: the code
|
||||||
|
// arrived before the right to use it. A module that read its new state on start failed its start; the
|
||||||
|
// one that was there retried for two minutes. The memberships were issued after the declarations —
|
||||||
|
// "because the runtime it is for arrives with it" — and a membership is retained last-per-subject on
|
||||||
|
// the bus (internal/link/bus.go), so issued first it waits for the runtime that arrives after it. A
|
||||||
|
// runtime still on the old code merely holds a grant it does not use yet.
|
||||||
|
//
|
||||||
|
// **The holder's declaration before the grants, though.** The bus's user list travels in it, and the
|
||||||
|
// controller's own right to publish memberships and raise buckets is in that list (the precedent of
|
||||||
|
// issue 183): grants first, and a grant the controller is not yet allowed to make would hold the very
|
||||||
|
// declaration that allows it — a lock only a hand on the broker could open. A runtime already running
|
||||||
|
// on that machine follows a membership issued after its declaration, as it always has.
|
||||||
|
//
|
||||||
|
// **A grant that cannot be issued holds back what it concerns, and says so as an error.** It used to
|
||||||
|
// be said and passed over — "the machines keep what they derive until the next push" — which reported
|
||||||
|
// a rollout done that had delivered code its machines could not run. A membership that failed holds
|
||||||
|
// back its own machine; a failure that names no machine (the buckets) holds back every machine but the
|
||||||
|
// holder, already sent. The error carries errGrants, so the caller's rollout is not marked sent and is
|
||||||
|
// tried again.
|
||||||
|
//
|
||||||
|
// The grants are issued, not waited on: a membership is a retained message the runtime reads when it
|
||||||
|
// comes, and the bus answers its publication; nothing here waits for a runtime to have read one.
|
||||||
|
func deliver(ctx context.Context, d delivery, holder string, sending []readyNode) (map[string]string, error) {
|
||||||
|
digests := map[string]string{}
|
||||||
|
if len(sending) == 0 {
|
||||||
|
return digests, nil
|
||||||
|
}
|
||||||
|
send := func(s readyNode) error {
|
||||||
|
// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
|
||||||
|
// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
|
||||||
|
body, err := s.declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
digest, err := d.declare(ctx, s, body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
digests[s.node] = digest
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var rest []readyNode
|
||||||
|
for _, s := range sending {
|
||||||
|
if holder != "" && s.node == holder {
|
||||||
|
if err := send(s); err != nil {
|
||||||
|
return digests, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rest = append(rest, s)
|
||||||
|
}
|
||||||
|
held := map[string]error{}
|
||||||
|
if err := d.grant(ctx, sending); err != nil {
|
||||||
|
var some *grantsRefused
|
||||||
|
if !errors.As(err, &some) {
|
||||||
|
var names []string
|
||||||
|
for _, s := range rest {
|
||||||
|
names = append(names, s.node)
|
||||||
|
}
|
||||||
|
if len(names) == 0 {
|
||||||
|
return digests, fmt.Errorf("%w: %w", errGrants, err)
|
||||||
|
}
|
||||||
|
return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(names, ", "), err)
|
||||||
|
}
|
||||||
|
held = some.nodes
|
||||||
|
}
|
||||||
|
var notSent []string
|
||||||
|
for _, s := range rest {
|
||||||
|
if _, refused := held[s.node]; refused {
|
||||||
|
notSent = append(notSent, s.node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := send(s); err != nil {
|
||||||
|
return digests, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(notSent) > 0 {
|
||||||
|
return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(notSent, ", "),
|
||||||
|
&grantsRefused{nodes: held})
|
||||||
|
}
|
||||||
|
if len(held) > 0 {
|
||||||
|
// Only the holder's own memberships failed, and it was sent before them.
|
||||||
|
return digests, fmt.Errorf("%w: %w", errGrants, &grantsRefused{nodes: held})
|
||||||
|
}
|
||||||
|
return digests, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// overTheBus is delivery as the mesh does it: memberships on the bus, declarations signed.
|
||||||
|
type overTheBus struct {
|
||||||
|
open *stores
|
||||||
|
server *link.Server
|
||||||
|
signer link.Signer
|
||||||
|
// indent is put before each "sent" line, for the callers whose output is nested.
|
||||||
|
indent string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||||
|
return issueMemberships(ctx, b.open, b.server, sending)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||||
|
if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||||
|
// the machine look current for a declaration it never received.
|
||||||
|
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if s.declared.BusUsers != "" {
|
||||||
|
// And the user list it carried, so the next send reads whether it must go first from the
|
||||||
|
// list alone (novox/hq issue 249). On the same outliving context as the send's record.
|
||||||
|
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
|
err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers)))
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources))
|
||||||
|
return digest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine
|
||||||
|
// holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a
|
||||||
|
// module's new permissions are refused by the bus until that list says them. Among the machines it
|
||||||
|
// is moved to the front; not among them, it is added only when it is behind.
|
||||||
|
func brokerFirst(names []string, holder string, behind bool) []string {
|
||||||
|
if holder == "" {
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
present := false
|
||||||
|
rest := make([]string, 0, len(names))
|
||||||
|
for _, n := range names {
|
||||||
|
if n == holder {
|
||||||
|
present = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rest = append(rest, n)
|
||||||
|
}
|
||||||
|
if !present && !behind {
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
return append([]string{holder}, rest...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// brokerBehind is the machine holding the bus — the one whose declaration carries the user list —
|
||||||
|
// and, when it is not among the machines named, whether the user list it would be sent now differs
|
||||||
|
// from the one it was last sent (novox/hq issue 249).
|
||||||
|
//
|
||||||
|
// **The user list alone, not the whole declaration.** Read from the whole declaration, any change
|
||||||
|
// pending on that machine — an upgrade its policy records rather than rolls out — went with every
|
||||||
|
// send anywhere, and a module running there always put it in its first wave. A digest of the list
|
||||||
|
// last sent is kept for this (ADR 0043: the list is composed on each push, never kept itself).
|
||||||
|
func brokerBehind(ctx context.Context, open *stores, names []string) (string, bool, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
holders, err := seatHolders(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
h, held := holders[theBrokerSeat]
|
||||||
|
if !held || h.Node == "" {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if m, known := shelf[h.Module]; !known || m.BusUsers == "" {
|
||||||
|
// A holder that is sent no user list carries no grant: nothing to send first.
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
for _, n := range names {
|
||||||
|
if n == h.Node {
|
||||||
|
return h.Node, false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
plan, _, err := planFor(ctx, open, h.Node)
|
||||||
|
if err != nil {
|
||||||
|
// It cannot be worked out: sending it would refuse the whole send, and `plan` says why.
|
||||||
|
return h.Node, false, nil
|
||||||
|
}
|
||||||
|
list, _, err := busUserList(ctx, inv, plan.Modules)
|
||||||
|
if err != nil {
|
||||||
|
return h.Node, false, nil
|
||||||
|
}
|
||||||
|
sent, err := inv.SentBusUsers(ctx, h.Node)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
return h.Node, userListBehind(list, sent), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
|
||||||
|
// empty list composed is never behind: there is nothing for it to carry.
|
||||||
|
func userListBehind(now, sentDigest string) bool {
|
||||||
|
return now != "" && digestOf([]byte(now)) != sentDigest
|
||||||
|
}
|
||||||
|
|
||||||
// couldNotBeResolved is what a push ends with when some machines could not be worked out.
|
// couldNotBeResolved is what a push ends with when some machines could not be worked out.
|
||||||
//
|
//
|
||||||
// **After the rest have been sent, never instead of sending them.** It is still an error, because
|
// **After the rest have been sent, never instead of sending them.** It is still an error, because
|
||||||
@@ -658,23 +901,36 @@ func couldNotBeResolved(refusals []string, sent int) error {
|
|||||||
// consumer and refused on the provider would leave one end holding a credential the other has
|
// consumer and refused on the provider would leave one end holding a credential the other has
|
||||||
// never heard of — which is the state this whole mechanism exists to make impossible.
|
// never heard of — which is the state this whole mechanism exists to make impossible.
|
||||||
func sendTo(ctx context.Context, open *stores, names []string) error {
|
func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||||
|
_, err := sendToEach(ctx, open, names)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendToEach is sendTo, answering the machines it sent: those named, and before them the machine
|
||||||
|
// holding the bus when its user list must go first (novox/hq issue 249) — so a caller that waits for
|
||||||
|
// the machines it sent waits for that one too.
|
||||||
|
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
ident, err := openIdentity(ctx)
|
ident, err := openIdentity(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer ident.Close()
|
defer ident.Close()
|
||||||
|
|
||||||
gens, err := generators(ctx, open)
|
gens, err := generators(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
holder, behind, err := brokerBehind(ctx, open, names)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names = brokerFirst(names, holder, behind)
|
||||||
|
|
||||||
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
||||||
// converge, which flips the node and then sends it — is not made to wait on itself.
|
// converge, which flips the node and then sends it — is not made to wait on itself.
|
||||||
ctx, release, err := holdNodes(ctx, open, names)
|
ctx, release, err := holdNodes(ctx, open, names)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
|
|
||||||
@@ -703,33 +959,29 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
|||||||
sending = append(sending, readyNode{name, declared})
|
sending = append(sending, readyNode{name, declared})
|
||||||
}
|
}
|
||||||
if len(refusals) > 0 {
|
if len(refusals) > 0 {
|
||||||
return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
return nil, fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
||||||
len(refusals), strings.Join(refusals, "\n\n"))
|
len(refusals), strings.Join(refusals, "\n\n"))
|
||||||
}
|
}
|
||||||
|
|
||||||
server, err := connectLink(ctx, nil, nil, nil)
|
server, err := connectLink(ctx, nil, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
for _, s := range sending {
|
|
||||||
body, err := s.declared.Body()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := recordSent(ctx, inv, s.node, body); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
|
||||||
}
|
|
||||||
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
|
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
|
||||||
// same records the bus's accounts are, so what a runtime serves and what its account may are one
|
// same records the bus's accounts are, so what a runtime serves and what its account may are one
|
||||||
// composition. Issued after the declaration, because the runtime it is for arrives with it.
|
// composition. **Issued before the declarations** (novox/hq issue 249): the runtime the
|
||||||
return issueMemberships(ctx, open, server, sending)
|
// membership is for arrives with the declaration, and a membership waits for it on the bus; the
|
||||||
|
// code arriving first was refused its own state until somebody pushed.
|
||||||
|
if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: " "}, holder, sending); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sent := make([]string, 0, len(sending))
|
||||||
|
for _, s := range sending {
|
||||||
|
sent = append(sent, s.node)
|
||||||
|
}
|
||||||
|
return sent, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// issueMemberships publishes the membership of every module on the machines just sent.
|
// issueMemberships publishes the membership of every module on the machines just sent.
|
||||||
@@ -747,11 +999,26 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// The declarations are sent and recorded by now; a membership that cannot be issued is said
|
// **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The
|
||||||
// and does not unsay them. Every runtime without one serves the shape it derives (ADR 0160), so
|
// raise at start asserts them too, but a module registered and assigned since would otherwise have
|
||||||
// the push stands, the first failure is named once, and the next push tries again.
|
// its bucket only after the control plane next restarts — found the first time a module declared
|
||||||
issued, failed := 0, 0
|
// state: its bundle asked for a bucket that did not exist. Idempotent and cheap.
|
||||||
var first error
|
//
|
||||||
|
// **A failure here is the send's failure** (novox/hq issue 249). It was said and the push stood,
|
||||||
|
// because the declarations were already away; they are sent after this now — all but the bus's
|
||||||
|
// own machine, sent before it (deliver) — and a module whose state does not exist is a module
|
||||||
|
// that fails its start, so they are not sent and the caller tries again rather than reporting the
|
||||||
|
// rollout done.
|
||||||
|
buckets, err := open.inventory.DeclaredBuckets(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the modules' state could not be read, so no bucket was asserted: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||||
|
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||||
|
}
|
||||||
|
// Every membership is tried, and the first failure named once.
|
||||||
|
issued := 0
|
||||||
|
refused := map[string]error{}
|
||||||
for _, s := range sent {
|
for _, s := range sent {
|
||||||
node := s.node
|
node := s.node
|
||||||
for _, d := range records.Assigned[node] {
|
for _, d := range records.Assigned[node] {
|
||||||
@@ -772,10 +1039,9 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
||||||
if first == nil {
|
if refused[node] == nil {
|
||||||
first = err
|
refused[node] = fmt.Errorf("%s: %w", d.Module, err)
|
||||||
}
|
}
|
||||||
failed++
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
issued++
|
issued++
|
||||||
@@ -784,9 +1050,10 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
|||||||
if issued > 0 {
|
if issued > 0 {
|
||||||
fmt.Printf(" issued %d membership(s)\n", issued)
|
fmt.Printf(" issued %d membership(s)\n", issued)
|
||||||
}
|
}
|
||||||
if failed > 0 {
|
if len(refused) > 0 {
|
||||||
fmt.Printf(" %d membership(s) could not be issued; the first: %v — the machines keep what "+
|
// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
|
||||||
"they derive until the next push\n", failed, first)
|
// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
|
||||||
|
return &grantsRefused{nodes: refused}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -806,6 +1073,20 @@ func digestOf(body []byte) string {
|
|||||||
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
||||||
func wouldSend(ctx context.Context, open *stores,
|
func wouldSend(ctx context.Context, open *stores,
|
||||||
nodes []inventory.Node) (map[string]string, error) {
|
nodes []inventory.Node) (map[string]string, error) {
|
||||||
|
return wouldSendFrom(ctx, open, nodes, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// planned is one machine's plan as planFor answered it, for a caller that already asked.
|
||||||
|
type planned struct {
|
||||||
|
plan catalogue.Resolution
|
||||||
|
settings catalogue.SettingsBy
|
||||||
|
}
|
||||||
|
|
||||||
|
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
|
||||||
|
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
|
||||||
|
// to-be 45 Phase 0). A machine absent from plans is worked out here.
|
||||||
|
func wouldSendFrom(ctx context.Context, open *stores,
|
||||||
|
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
|
||||||
|
|
||||||
gens, err := generators(ctx, open)
|
gens, err := generators(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -813,9 +1094,12 @@ func wouldSend(ctx context.Context, open *stores,
|
|||||||
}
|
}
|
||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
plan, settings, err := planFor(ctx, open, n.Name)
|
known, have := plans[n.Name]
|
||||||
if err != nil {
|
plan, settings := known.plan, known.settings
|
||||||
continue
|
if !have {
|
||||||
|
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -874,55 +1158,44 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
// The mesh's own streams and consumers, the seats' work queues and their workers, and how every
|
||||||
|
// machine hears its declaration — one derivation, which the self-check reads as well (D6, D7).
|
||||||
|
names, err := assertBusObjects(ctx, inv, js)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
names := make([]string, 0, len(nodes))
|
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
|
||||||
for _, n := range nodes {
|
// whether it was cancelled the moment it took it.
|
||||||
names = append(names, n.Name)
|
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
||||||
}
|
|
||||||
if err := broker.Raise(js, names); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
|
||||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
// by hand, kept where a restart of this process does not take them.
|
||||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
return err
|
||||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
}
|
||||||
// consumer nothing had created (2026-09-28).
|
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
||||||
holders, err := seatHolders(ctx, inv)
|
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
||||||
|
// nothing declares any more is said and kept — what it holds is data.
|
||||||
|
buckets, err := inv.DeclaredBuckets(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
|
undeclared, err := broker.RaiseBuckets(js, buckets)
|
||||||
return err
|
|
||||||
}
|
|
||||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
|
||||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
|
||||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
|
||||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
|
||||||
records, err := inv.BusRecords(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
users, err := broker.Users(records)
|
if len(undeclared) > 0 {
|
||||||
|
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||||
|
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||||
|
}
|
||||||
|
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||||
|
hearing, err := moduleConsumerCount(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
hearing := 0
|
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||||
for _, p := range users {
|
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||||
consumer, needed := broker.ConsumerFor(p)
|
|
||||||
if !needed {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := js.EnsureConsumer(consumer); err != nil {
|
|
||||||
return fmt.Errorf("how %s on %s hears what it consumes: %w", p.Module, p.Node, err)
|
|
||||||
}
|
|
||||||
hearing++
|
|
||||||
}
|
|
||||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, and %d module(s) "+
|
|
||||||
"can hear what they consume\n", broker.BareAddress(address), len(names), hearing)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -984,7 +1257,11 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, e
|
|||||||
// never wrote it down: status read "applied, current" over a machine that had just been sent
|
// never wrote it down: status read "applied, current" over a machine that had just been sent
|
||||||
// something else. What was sent was sent; the record of it must not depend on the sender living
|
// something else. What was sent was sent; the record of it must not depend on the sender living
|
||||||
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
|
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
|
||||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte) (string, error) {
|
//
|
||||||
|
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||||
|
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||||
|
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||||
|
builds map[string]string) (string, error) {
|
||||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
record, err := inv.NodeByName(kept, node)
|
record, err := inv.NodeByName(kept, node)
|
||||||
@@ -992,8 +1269,37 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
digest := digestOf(body)
|
digest := digestOf(body)
|
||||||
if err := inv.RecordSent(kept, record.ID, digest); err != nil {
|
if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return digest, nil
|
return digest, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reportUnheldPushed says what a push's machines lack of the seats their modules depend on
|
||||||
|
// (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody
|
||||||
|
// is looking at, and one line per machine otherwise — a list per machine across the mesh is the
|
||||||
|
// hundred lines that buried the one that mattered. Nothing for a machine that lacks nothing.
|
||||||
|
func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[string][]catalogue.Unheld) {
|
||||||
|
for _, node := range asked {
|
||||||
|
lines := unheld[node]
|
||||||
|
if len(lines) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if named {
|
||||||
|
fmt.Fprintf(w, "\n%s has %d unmet seat dependenc(ies) (novox/hq ADR 0207):\n", node, len(lines))
|
||||||
|
for _, u := range lines {
|
||||||
|
fmt.Fprintf(w, " %s\n", u)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// controllerProcess names this serving process among controllers: the machine, the process and when
|
||||||
|
// it started — what a call kept on the bus carries, so the next controller can tell a call this one
|
||||||
|
// left running from one it is running itself (novox/hq to-be 45 §6).
|
||||||
|
func controllerProcess() string {
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,709 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||||
|
//
|
||||||
|
// Seen whole — what waits, what runs where and for how long, what was handed out as often as it
|
||||||
|
// may be and never settled — and changed through the controller: an ask cancelled, the queue
|
||||||
|
// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's
|
||||||
|
// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`).
|
||||||
|
//
|
||||||
|
// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that
|
||||||
|
// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
|
||||||
|
// rather than waiting for ever on an answer nobody will give.
|
||||||
|
|
||||||
|
// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
|
||||||
|
// than the holder's worst case (its two passes removing containers and its wait between, 50s).
|
||||||
|
const (
|
||||||
|
holderAsks = 15 * time.Second
|
||||||
|
killAnswer = 75 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
|
||||||
|
func dialTheBus() (*broker.JetStream, error) {
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot reach the bus: %w", err)
|
||||||
|
}
|
||||||
|
return js, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// queueCommand prints every ask in the build seat's work queue.
|
||||||
|
func queueCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("queue", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "the queue as JSON")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
q, err := link.ReadQueue(ctx, js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
body, err := json.MarshalIndent(q, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Print(queueText(q, time.Now()))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
|
||||||
|
// Never what an ask carries as `held` — that is every artifact the mesh has built.
|
||||||
|
func queueText(q link.Queue, now time.Time) string {
|
||||||
|
var b strings.Builder
|
||||||
|
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||||
|
fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead))
|
||||||
|
ago := func(t time.Time) string {
|
||||||
|
if t.IsZero() {
|
||||||
|
return "asked at an unknown time"
|
||||||
|
}
|
||||||
|
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
|
||||||
|
}
|
||||||
|
what := func(a link.QueuedAsk) string {
|
||||||
|
s := a.Repository
|
||||||
|
if a.Path != "" {
|
||||||
|
s += " at " + a.Path
|
||||||
|
}
|
||||||
|
if a.Ref != "" {
|
||||||
|
s += " on " + a.Ref
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
if len(running) > 0 {
|
||||||
|
fmt.Fprintln(&b, "\nin flight:")
|
||||||
|
for _, a := range running {
|
||||||
|
where := "taken, not yet said where"
|
||||||
|
switch {
|
||||||
|
case a.On != "":
|
||||||
|
where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
|
||||||
|
case a.Was != "":
|
||||||
|
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(waiting) > 0 {
|
||||||
|
fmt.Fprintln(&b, "\nwaiting:")
|
||||||
|
for _, a := range waiting {
|
||||||
|
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(dead) > 0 {
|
||||||
|
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
|
||||||
|
for _, a := range dead {
|
||||||
|
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case len(q.Asks) == 0:
|
||||||
|
fmt.Fprintln(&b, "nothing is asked of it")
|
||||||
|
default:
|
||||||
|
fmt.Fprintln(&b, "\n`cancel <id>` drops a waiting or dead ask, `clear` every waiting one, `kill <id>` ends one in flight")
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancelCommand drops one waiting or dead ask.
|
||||||
|
func cancelCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("cancel <build id>")
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
q, err := link.ReadQueue(ctx, js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ask, found := q.Find(args[0])
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+
|
||||||
|
"what came of it", args[0], seat)
|
||||||
|
}
|
||||||
|
said, err := cancelAsk(ctx, js, open, seat, ask)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
|
||||||
|
//
|
||||||
|
// **In this order, and each step for a reason** (novox/hq ADR 0219):
|
||||||
|
// 1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
|
||||||
|
// is running — that is `kill`, on the machine running it;
|
||||||
|
// 2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
|
||||||
|
// 3. for a waiting ask, the worker is read again: one handed out since the queue was read was
|
||||||
|
// taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
|
||||||
|
// read the mark first and ends it as cancelled, or is building it;
|
||||||
|
// 4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
|
||||||
|
// handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
|
||||||
|
// at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
|
||||||
|
// that took it before the mark is building it, and only `kill` ends that;
|
||||||
|
// 5. the message is deleted by its sequence;
|
||||||
|
// 6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
|
||||||
|
func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
|
||||||
|
if ask.State == link.AskInFlight && ask.On != "" {
|
||||||
|
return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
|
||||||
|
"ends the build where it runs", ask.ID, ask.On, ask.ID)
|
||||||
|
}
|
||||||
|
if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
withdraw := func() {
|
||||||
|
if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||||
|
switch ask.State {
|
||||||
|
case link.AskWaiting:
|
||||||
|
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||||
|
withdraw()
|
||||||
|
return "", err
|
||||||
|
} else if taken {
|
||||||
|
withdraw()
|
||||||
|
return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
|
||||||
|
"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
|
||||||
|
"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
|
||||||
|
}
|
||||||
|
case link.AskInFlight:
|
||||||
|
if started, err := startedSince(ctx, js, seat, ask); err != nil {
|
||||||
|
withdraw()
|
||||||
|
return "", err
|
||||||
|
} else if started != "" {
|
||||||
|
withdraw()
|
||||||
|
return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
|
||||||
|
"ends it there", ask.ID, started, ask.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
|
||||||
|
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
|
||||||
|
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||||
|
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||||
|
}
|
||||||
|
recordCancelled(ctx, open, ask)
|
||||||
|
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
|
||||||
|
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
|
||||||
|
// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
|
||||||
|
var holderLooks = 5 * time.Second
|
||||||
|
|
||||||
|
// startedSince waits out a holder's look at the cancelled set and says the machine that started the
|
||||||
|
// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
|
||||||
|
// a start of a build.
|
||||||
|
func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", ctx.Err()
|
||||||
|
case <-time.After(holderLooks):
|
||||||
|
}
|
||||||
|
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||||
|
if !ask.AskedAt.IsZero() {
|
||||||
|
since = ask.AskedAt.Add(-time.Minute)
|
||||||
|
}
|
||||||
|
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
s, ok := heard.Started[ask.ID]
|
||||||
|
if !ok || heard.Outcomes[ask.ID] {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
at, _ := time.Parse(time.RFC3339Nano, s.At)
|
||||||
|
if ask.Started.IsZero() || at.After(ask.Started) {
|
||||||
|
return s.On, nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// takenSince is whether the worker has handed out the ask at this sequence.
|
||||||
|
func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
|
||||||
|
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
|
||||||
|
if errors.Is(err, nats.ErrConsumerNotFound) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("cannot read the worker of the queue again: %w", err)
|
||||||
|
}
|
||||||
|
return info.Delivered.Stream >= seq, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded,
|
||||||
|
// then the plan that asked for it — by the id it asked with, or by repository and path.
|
||||||
|
func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) {
|
||||||
|
r := ask.Request
|
||||||
|
result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref,
|
||||||
|
Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand}
|
||||||
|
_ = builds{open.inventory, open}.Built(ctx, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// clearCommand cancels every waiting ask, and with --dead every dead one too.
|
||||||
|
func clearCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("clear", flag.ContinueOnError)
|
||||||
|
dead := set.Bool("dead", false, "the dead asks too")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
said, err := clearQueue(ctx, js, open, seat, *dead)
|
||||||
|
fmt.Print(said)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight.
|
||||||
|
func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) {
|
||||||
|
q, err := link.ReadQueue(ctx, js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
cancelled, refused := 0, 0
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
said, err := cancelAsk(ctx, js, open, seat, a)
|
||||||
|
if err != nil {
|
||||||
|
refused++
|
||||||
|
fmt.Fprintf(&b, " %s: %v\n", a.ID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cancelled++
|
||||||
|
fmt.Fprintf(&b, " %s\n", said)
|
||||||
|
}
|
||||||
|
what := "waiting"
|
||||||
|
if dead {
|
||||||
|
what = "waiting and dead"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what)
|
||||||
|
if refused > 0 {
|
||||||
|
fmt.Fprintf(&b, ", %d could not be", refused)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(&b)
|
||||||
|
if n := len(q.Of(link.AskInFlight)); n > 0 {
|
||||||
|
fmt.Fprintf(&b, "%d in flight, left running: `kill <id>` ends one where it runs\n", n)
|
||||||
|
}
|
||||||
|
if n := len(q.Of(link.AskDead)); n > 0 && !dead {
|
||||||
|
fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n)
|
||||||
|
}
|
||||||
|
if refused > 0 {
|
||||||
|
return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused)
|
||||||
|
}
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rebuildCommand asks the module's current source again — or, given a build's id, that build's
|
||||||
|
// repository, path and ref — under a new id.
|
||||||
|
func rebuildCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("rebuild <module | build id>")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var source buildSource
|
||||||
|
var path, ref, module string
|
||||||
|
if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
|
||||||
|
return err
|
||||||
|
} else if found {
|
||||||
|
source, module = sourceOfBuild(b, entries)
|
||||||
|
path, ref = b.Path, b.Ref
|
||||||
|
// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
|
||||||
|
// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
|
||||||
|
// that commit out over everything since. Building that commit again is `replay`, which says
|
||||||
|
// what it would do and refuses to register it while anything newer is asked or registered.
|
||||||
|
if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
|
||||||
|
ref = followedBranch(e.Source.Ref)
|
||||||
|
follows := ref
|
||||||
|
if follows == "" {
|
||||||
|
follows = "the repository's default branch"
|
||||||
|
}
|
||||||
|
fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
|
||||||
|
"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
|
||||||
|
}
|
||||||
|
} else if e, known := entryNamed(entries, args[0]); known {
|
||||||
|
// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
|
||||||
|
source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||||
|
path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
|
||||||
|
// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
|
||||||
|
if module != "" {
|
||||||
|
joined, said, err := joinAPlan(ctx, open, module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if joined {
|
||||||
|
fmt.Println(said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
id, err := askABuild(ctx, source, path, ref)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("rebuild asked as %s\n", id)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// entryNamed is the catalogue's entry for a module.
|
||||||
|
func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.Manifest.Module == name {
|
||||||
|
return e, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inventory.Entry{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
|
||||||
|
// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
|
||||||
|
// (ADR 0111) — else the repository as it was cloned.
|
||||||
|
func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) {
|
||||||
|
for _, e := range entries {
|
||||||
|
if (b.Module != "" && e.Manifest.Module == b.Module) ||
|
||||||
|
(b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) {
|
||||||
|
return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return buildSource{Repository: b.Repository}, b.Module
|
||||||
|
}
|
||||||
|
|
||||||
|
// replayCommand asks a recorded build's repository and path again at the commit it built.
|
||||||
|
func replayCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("replay", flag.ContinueOnError)
|
||||||
|
register := set.Bool("register", false, "register what it builds, as any build is")
|
||||||
|
older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("replay <build id> [--register [--older]]")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
b, found, err := inv.BuildByID(ctx, positionals[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no build %s is recorded", positionals[0])
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
source, module := sourceOfBuild(b, entries)
|
||||||
|
var history []inventory.Build
|
||||||
|
if module != "" {
|
||||||
|
if history, err = inv.Builds(ctx, module, 100); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// What is asked of the module and not yet answered, when the replay would be registered.
|
||||||
|
var outstanding []string
|
||||||
|
if *register {
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
|
||||||
|
js.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(replaySaid(b, module, id, *register))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
|
||||||
|
//
|
||||||
|
// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
|
||||||
|
// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
|
||||||
|
// older commit is newer than everything since, and would roll that older commit out as the module's
|
||||||
|
// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
|
||||||
|
// --register says otherwise, and --register is refused when a newer build of a different commit is
|
||||||
|
// registered, unless --older says that is the point.
|
||||||
|
//
|
||||||
|
// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
|
||||||
|
// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
|
||||||
|
// and their builds — made from newer source — would be recorded and never registered (issue 219
|
||||||
|
// orders by ask), the plan waiting on them sending the older commit instead.
|
||||||
|
func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
|
||||||
|
outstanding []string) error {
|
||||||
|
if b.Commit == "" {
|
||||||
|
return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
|
||||||
|
"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
|
||||||
|
}
|
||||||
|
if older && !register {
|
||||||
|
return errors.New("--older only says what --register may do; a dry run registers nothing")
|
||||||
|
}
|
||||||
|
if register && len(outstanding) > 0 {
|
||||||
|
return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
|
||||||
|
"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
|
||||||
|
orNone(module), strings.Join(outstanding, "; "), b.ID)
|
||||||
|
}
|
||||||
|
if !register || older {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, h := range history {
|
||||||
|
if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.AskedOrAt().After(b.AskedOrAt()) {
|
||||||
|
return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
|
||||||
|
"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
|
||||||
|
"without --register looks at it; --register --older registers it anyway",
|
||||||
|
module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome.
|
||||||
|
func replaySaid(b inventory.Build, module, id string, register bool) string {
|
||||||
|
what := b.Repository
|
||||||
|
if module != "" {
|
||||||
|
what = module
|
||||||
|
}
|
||||||
|
said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id)
|
||||||
|
if !register {
|
||||||
|
return said + "\n a dry run: the outcome is looked at and not taken in — nothing is recorded or " +
|
||||||
|
"registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it"
|
||||||
|
}
|
||||||
|
return said + "\n registered when it is built, as the module's current version — newer than every build " +
|
||||||
|
"asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it"
|
||||||
|
}
|
||||||
|
|
||||||
|
// killCommand finds the machine running a build and asks its holder to end it.
|
||||||
|
func killCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("kill <build id>")
|
||||||
|
}
|
||||||
|
id := args[0]
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||||
|
if at, ok := link.BuildAskedAt(id); ok {
|
||||||
|
since = at.Add(-time.Minute)
|
||||||
|
}
|
||||||
|
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
started, ok := heard.Started[id]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id)
|
||||||
|
}
|
||||||
|
if heard.Outcomes[id] {
|
||||||
|
return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
|
||||||
|
}
|
||||||
|
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return printHolderAnswer(started.On, answer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pauseCommand asks one machine's holder, or every holder's, to pause or resume.
|
||||||
|
func pauseCommand(ctx context.Context, verb string, args []string) error {
|
||||||
|
if len(args) > 1 {
|
||||||
|
return fmt.Errorf("%s [node]", verb)
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
nodes := args
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
if nodes, err = buildSeatHolders(ctx, seat); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
failed := 0
|
||||||
|
for _, node := range nodes {
|
||||||
|
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("%s: %v\n", node, err)
|
||||||
|
failed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := printHolderAnswer(node, answer); err != nil {
|
||||||
|
failed++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failed > 0 {
|
||||||
|
return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// printHolderAnswer prints what a holder said, or its refusal as the command's failure.
|
||||||
|
func printHolderAnswer(node string, answer link.Answer) error {
|
||||||
|
if answer.Error != "" {
|
||||||
|
fmt.Printf("%s: %s\n", node, answer.Error)
|
||||||
|
return errors.New(answer.Error)
|
||||||
|
}
|
||||||
|
var said struct {
|
||||||
|
Said string `json:"said"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" {
|
||||||
|
fmt.Printf("%s: %s\n", node, said.Said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", node, string(answer.Result))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSeatHolders is every machine an assigned module holding the build seat runs on.
|
||||||
|
func buildSeatHolders(ctx context.Context, seat string) ([]string, error) {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return holdersAmong(entries, seat), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each.
|
||||||
|
func holdersAmong(entries []inventory.Entry, seat string) []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.Manifest.ClaimsSeat(seat) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, n := range e.On {
|
||||||
|
if !seen[n] {
|
||||||
|
seen[n] = true
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
|
||||||
|
// by repository and path, and every open plan holding it not yet built.
|
||||||
|
func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
|
||||||
|
var out []string
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
if repositoryMatches(a.Repository, repository) && a.Path == path {
|
||||||
|
out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if module == "" {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, p := range plans {
|
||||||
|
if !p.Open() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, tier := range p.Tiers {
|
||||||
|
for _, m := range tier {
|
||||||
|
if m == module {
|
||||||
|
if st := p.Modules[m]; st == nil || st.State != "built" {
|
||||||
|
out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,772 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js
|
||||||
|
// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14294 \
|
||||||
|
// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
|
||||||
|
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused'
|
||||||
|
|
||||||
|
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
|
||||||
|
func asksRecorded(t *testing.T) *[]string {
|
||||||
|
t.Helper()
|
||||||
|
var asked []string
|
||||||
|
was := askABuild
|
||||||
|
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||||
|
id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond))
|
||||||
|
asked = append(asked, source.Repository+"#"+id)
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { askABuild = was })
|
||||||
|
return &asked
|
||||||
|
}
|
||||||
|
|
||||||
|
// twoTiers registers two modules, b standing on a, each with a source a plan asks.
|
||||||
|
func twoTiers(t *testing.T, open *stores) {
|
||||||
|
t.Helper()
|
||||||
|
for _, name := range []string{"a", "b"} {
|
||||||
|
if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed plan is retried: its failed module asked again under a new id, the plan building at that
|
||||||
|
// tier, and when that build comes in the plan goes on and asks its next tier.
|
||||||
|
func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
twoTiers(t, open)
|
||||||
|
before := time.Now().UTC().Add(-time.Hour)
|
||||||
|
failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||||
|
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Note: "a failed to build in tier 0",
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||||
|
Why: link.KilledByHand}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
said, err := retryPlan(ctx, open, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := p.Modules["a"]
|
||||||
|
if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
|
||||||
|
t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, a.Build) {
|
||||||
|
t.Errorf("retry does not say the new id: %q", said)
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
|
||||||
|
planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
|
||||||
|
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
|
||||||
|
t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
|
||||||
|
}
|
||||||
|
asking, _ := link.BuildAskedAt(a.Build)
|
||||||
|
planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
|
||||||
|
p, err = open.inventory.PlanByID(ctx, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
|
||||||
|
t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
|
||||||
|
}
|
||||||
|
if len(*asked) != 2 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What retry refuses, and says why.
|
||||||
|
func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
plan := func(id, state string, created time.Time) inventory.Plan {
|
||||||
|
return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee",
|
||||||
|
Created: created, State: state, Tiers: [][]string{{"a"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||||
|
}
|
||||||
|
failed := plan("plan-1", inventory.PlanFailed, at)
|
||||||
|
for _, c := range []struct {
|
||||||
|
p inventory.Plan
|
||||||
|
others []inventory.Plan
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{plan("plan-d", inventory.PlanDone, at), nil, "is done"},
|
||||||
|
{plan("plan-s", inventory.PlanSuperseded, at), nil, "was"},
|
||||||
|
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
||||||
|
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
||||||
|
} {
|
||||||
|
err := retryRefusal(c.p, c.others)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||||
|
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stopped := failed
|
||||||
|
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||||
|
stopped.Note = "a stopped at its first machine"
|
||||||
|
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||||
|
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||||
|
}
|
||||||
|
// Another branch's newer plan, an older one, and a failed one do not supersede it.
|
||||||
|
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||||
|
other.Branch = "release"
|
||||||
|
if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
|
||||||
|
plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
|
||||||
|
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
|
||||||
|
func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
twoTiers(t, open)
|
||||||
|
before := time.Now().UTC().Add(-time.Hour)
|
||||||
|
failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||||
|
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Note: "a failed to build in tier 0",
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
|
||||||
|
t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
|
||||||
|
if err := rebuildCommand(ctx, []string{"b"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
|
||||||
|
t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
|
||||||
|
}
|
||||||
|
if len(*asked) != 2 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
|
||||||
|
func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
|
||||||
|
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||||
|
newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||||
|
if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
|
||||||
|
t.Fatal("joined a failed plan a newer open one supersedes")
|
||||||
|
}
|
||||||
|
if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
|
||||||
|
t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
|
||||||
|
}
|
||||||
|
built := failed
|
||||||
|
built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||||
|
if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
|
||||||
|
t.Fatal("joined a plan that has the module built")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// replay is a dry run unless registered, and registering an older commit than one registered since
|
||||||
|
// is refused unless --older says it is meant (novox/hq issue 207).
|
||||||
|
func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
|
||||||
|
asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
|
||||||
|
newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
|
||||||
|
history := []inventory.Build{newer, old}
|
||||||
|
|
||||||
|
if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
|
||||||
|
t.Errorf("a dry run was refused: %v", err)
|
||||||
|
}
|
||||||
|
err := replayRefusal(old, "a", history, true, false, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
|
||||||
|
t.Errorf("registering an older commit than the one registered was not refused: %v", err)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
|
||||||
|
t.Errorf("--register --older was refused: %v", err)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
|
||||||
|
t.Errorf("registering the newest build again was refused: %v", err)
|
||||||
|
}
|
||||||
|
// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
|
||||||
|
same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
|
||||||
|
broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
|
||||||
|
if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
|
||||||
|
t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
|
||||||
|
t.Error("a build that recorded no commit was replayed")
|
||||||
|
}
|
||||||
|
if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
|
||||||
|
t.Error("--older without --register was taken")
|
||||||
|
}
|
||||||
|
// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
|
||||||
|
// holding it unbuilt, would be replaced by a replay asked now (issue 219).
|
||||||
|
q := link.Queue{Asks: []link.QueuedAsk{
|
||||||
|
{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
|
||||||
|
{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
|
||||||
|
{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
|
||||||
|
}}
|
||||||
|
plans := []inventory.Plan{
|
||||||
|
{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||||
|
{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
|
||||||
|
{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||||
|
}
|
||||||
|
outstanding := outstandingFor("a", "novox/a", "", q, plans)
|
||||||
|
if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
|
||||||
|
t.Fatalf("outstanding: %v", outstanding)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
|
||||||
|
t.Errorf("registered over an outstanding ask: %v", err)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
|
||||||
|
t.Errorf("a dry run was refused for what is outstanding: %v", err)
|
||||||
|
}
|
||||||
|
if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
|
||||||
|
t.Errorf("a dry replay does not say what it is: %q", said)
|
||||||
|
}
|
||||||
|
if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
|
||||||
|
t.Errorf("a registered replay does not say what it does: %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
|
||||||
|
// paused on some holders only is not a reason the plan is waiting.
|
||||||
|
func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
asked := now.Add(-12 * time.Minute)
|
||||||
|
p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding,
|
||||||
|
Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||||
|
|
||||||
|
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
|
||||||
|
line := planLineWith(p, now, all)
|
||||||
|
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a plan on a paused seat reads %q", line)
|
||||||
|
}
|
||||||
|
if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
||||||
|
t.Errorf("status --json says %+v", st)
|
||||||
|
}
|
||||||
|
if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
|
||||||
|
t.Errorf("a plan waiting on a paused seat counted late")
|
||||||
|
}
|
||||||
|
|
||||||
|
longAgo := now.Add(-25 * time.Hour)
|
||||||
|
forgotten := p
|
||||||
|
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
|
||||||
|
if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a plan paused over a day reads %q", line)
|
||||||
|
}
|
||||||
|
|
||||||
|
some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
|
||||||
|
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
|
||||||
|
t.Fatalf("%+v", some)
|
||||||
|
}
|
||||||
|
if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
|
||||||
|
}
|
||||||
|
if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
|
||||||
|
t.Errorf("status --json: %+v", st)
|
||||||
|
}
|
||||||
|
// A plan rolling out, or with nothing asked, is not waiting on the seat.
|
||||||
|
rolling := p
|
||||||
|
rolling.State = inventory.PlanRolling
|
||||||
|
if _, paused := pausedWaiting(rolling, all, now); paused {
|
||||||
|
t.Error("a rolling plan reads as waiting on the build seat")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The queue's verbs are the controller seat's, each to the command it names.
|
||||||
|
func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"queue", nil, []string{"queue"}},
|
||||||
|
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
||||||
|
{"clear", nil, []string{"clear"}},
|
||||||
|
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
||||||
|
{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
|
||||||
|
{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
|
||||||
|
{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
|
||||||
|
{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
|
||||||
|
{"pause", nil, []string{"pause"}},
|
||||||
|
{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
|
||||||
|
{"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}},
|
||||||
|
} {
|
||||||
|
got, err := argvFor(c.verb, c.args)
|
||||||
|
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||||
|
t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := argvFor("cancel", nil); err == nil {
|
||||||
|
t.Error("cancel without an id was taken")
|
||||||
|
}
|
||||||
|
declared := map[string]bool{}
|
||||||
|
for _, v := range catalogue.ControllerVerbs {
|
||||||
|
declared[v.Name] = true
|
||||||
|
}
|
||||||
|
for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} {
|
||||||
|
if !declared[v] {
|
||||||
|
t.Errorf("%s is not a verb of the controller seat", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- against a real bus -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller
|
||||||
|
// pointed at it, and a function that asks the seat one build.
|
||||||
|
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
t.Setenv(broker.NATSVar, url)
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
|
||||||
|
Emits: []string{"started", "built", "log.*", "paused.*"}}
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||||
|
if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
|
||||||
|
link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||||
|
_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
|
||||||
|
_ = js.Context().PurgeStream(broker.EventsStream)
|
||||||
|
})
|
||||||
|
ask := func(id, repository string) link.BuildRequest {
|
||||||
|
r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
|
||||||
|
body, _ := json.Marshal(r)
|
||||||
|
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
return js, ask
|
||||||
|
}
|
||||||
|
|
||||||
|
// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows.
|
||||||
|
func deadOne(t *testing.T, js *broker.JetStream) {
|
||||||
|
t.Helper()
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||||
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
for i := 0; i < worker.MaxDeliver; i++ {
|
||||||
|
msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("delivery %d: %v", i+1, err)
|
||||||
|
}
|
||||||
|
_ = msgs[0].Nak()
|
||||||
|
}
|
||||||
|
// One more pull, as a holder always has one waiting: the server finds the ask past its deliveries
|
||||||
|
// then, and stops counting it pending.
|
||||||
|
if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 {
|
||||||
|
t.Fatalf("an ask past its deliveries was delivered again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan
|
||||||
|
// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for
|
||||||
|
// is still found.
|
||||||
|
func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
|
||||||
|
js, ask := aBuildQueue(t)
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
twoTiers(t, open)
|
||||||
|
id := link.NewBuildID(time.Now())
|
||||||
|
ask(id, "https://forge.example/novox/a.git")
|
||||||
|
asked, _ := link.BuildAskedAt(id)
|
||||||
|
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
|
||||||
|
t.Fatalf("the queue reads %+v", q)
|
||||||
|
}
|
||||||
|
if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
|
||||||
|
strings.Contains(text, "secret-ish") {
|
||||||
|
t.Errorf("the queue says:\n%s", text)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := cancelCommand(ctx, []string{id}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||||
|
t.Fatalf("the ask is still queued: %+v", q.Asks)
|
||||||
|
}
|
||||||
|
if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
|
||||||
|
t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
|
||||||
|
}
|
||||||
|
b, found, err := open.inventory.BuildByID(ctx, id)
|
||||||
|
if err != nil || !found || b.Failed != link.CancelledByHand {
|
||||||
|
t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
|
||||||
|
}
|
||||||
|
p, err := open.inventory.PlanByID(ctx, plan.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
|
||||||
|
t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
|
||||||
|
}
|
||||||
|
if err := cancelCommand(ctx, []string{id}); err == nil {
|
||||||
|
t.Error("an ask cancelled already was cancelled again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight.
|
||||||
|
func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
|
||||||
|
js, ask := aBuildQueue(t)
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
start := time.Now()
|
||||||
|
dead := link.NewBuildID(start)
|
||||||
|
ask(dead, "https://forge.example/novox/dead.git")
|
||||||
|
deadOne(t, js)
|
||||||
|
var waiting []string
|
||||||
|
for i := 1; i <= 2; i++ {
|
||||||
|
id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
|
||||||
|
waiting = append(waiting, id)
|
||||||
|
ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
|
||||||
|
}
|
||||||
|
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
|
||||||
|
t.Fatalf("the queue reads %+v", q)
|
||||||
|
}
|
||||||
|
said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
|
||||||
|
t.Errorf("clear said:\n%s", said)
|
||||||
|
}
|
||||||
|
q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
|
||||||
|
t.Fatalf("after clear the queue is %+v", q.Asks)
|
||||||
|
}
|
||||||
|
if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||||
|
t.Fatalf("clear --dead left %+v", q.Asks)
|
||||||
|
}
|
||||||
|
for _, id := range append(waiting, dead) {
|
||||||
|
if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
|
||||||
|
t.Errorf("%s is recorded as %+v", id, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ask in flight is not cancelled: kill ends it where it runs.
|
||||||
|
func TestCancelRefusesAnAskInFlight(t *testing.T) {
|
||||||
|
js, ask := aBuildQueue(t)
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
id := link.NewBuildID(time.Now())
|
||||||
|
ask(id, "https://forge.example/novox/a.git")
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||||
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a, _ := q.Find(id)
|
||||||
|
if a.State != link.AskInFlight || a.On != "ace" {
|
||||||
|
t.Fatalf("the taken ask reads %+v", a)
|
||||||
|
}
|
||||||
|
_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "kill "+id) {
|
||||||
|
t.Fatalf("an ask in flight was cancelled: %v", err)
|
||||||
|
}
|
||||||
|
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||||
|
t.Error("a refused cancel recorded an outcome")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kill finds the machine from the build's start and asks that machine's holder; pause asks the
|
||||||
|
// machine named. Each prints what the holder answered, and a refusal is the command's failure.
|
||||||
|
func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
||||||
|
js, _ := aBuildQueue(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := map[string]string{}
|
||||||
|
handlers := map[string]link.ToolHandler{
|
||||||
|
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||||
|
var args struct{ ID string }
|
||||||
|
_ = json.Unmarshal(raw, &args)
|
||||||
|
asked["kill"] = args.ID
|
||||||
|
if args.ID != "build-running" {
|
||||||
|
return nil, fmt.Errorf("ace is not building %s", args.ID)
|
||||||
|
}
|
||||||
|
return map[string]any{"said": "killed " + args.ID}, nil
|
||||||
|
},
|
||||||
|
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
asked["pause"] = "ace"
|
||||||
|
return map[string]any{"said": "ace is paused"}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer stop()
|
||||||
|
for _, id := range []string{"build-running", "build-other"} {
|
||||||
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := killCommand(ctx, []string{"build-running"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asked["kill"] != "build-running" {
|
||||||
|
t.Fatalf("the holder was asked %v", asked)
|
||||||
|
}
|
||||||
|
if err := killCommand(ctx, []string{"build-other"}); err == nil {
|
||||||
|
t.Error("the holder's refusal was not the command's")
|
||||||
|
}
|
||||||
|
if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
|
||||||
|
t.Errorf("a build nobody started: %v", err)
|
||||||
|
}
|
||||||
|
if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
|
||||||
|
t.Fatalf("pause: %v %v", err, asked)
|
||||||
|
}
|
||||||
|
if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
|
||||||
|
t.Error("a machine nothing answers on was resumed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||||
|
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||||
|
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
twoTiers(t, open)
|
||||||
|
var sentTo [][]string
|
||||||
|
was := sendRollout
|
||||||
|
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||||
|
sentTo = append(sentTo, names)
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = was })
|
||||||
|
|
||||||
|
long := time.Now().UTC().Add(-2 * time.Hour)
|
||||||
|
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||||
|
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||||
|
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, stopped); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A newer plan holding a refuses it: sending the older build would put it back.
|
||||||
|
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
|
||||||
|
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
|
||||||
|
t.Fatalf("retried under a newer plan: %v", err)
|
||||||
|
}
|
||||||
|
newer.State = inventory.PlanSuperseded
|
||||||
|
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
said, err := retryPlan(ctx, open, stopped.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
|
||||||
|
t.Fatalf("sent %v; said %q", sentTo, said)
|
||||||
|
}
|
||||||
|
p, err := open.inventory.PlanByID(ctx, stopped.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := p.Modules["a"]
|
||||||
|
if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
|
||||||
|
t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
|
||||||
|
}
|
||||||
|
// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
|
||||||
|
t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
|
||||||
|
// the plan, asked later, never answers it (novox/hq ADR 0219).
|
||||||
|
func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := time.Now().UTC().Add(-time.Minute)
|
||||||
|
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
|
||||||
|
p, _ := open.inventory.PlanByID(ctx, plan.ID)
|
||||||
|
if p.Modules["a"].State != "asked" {
|
||||||
|
t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
|
||||||
|
}
|
||||||
|
// From the records too: a later build of the module recorded is not the plan's.
|
||||||
|
recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
|
||||||
|
if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
|
||||||
|
t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
|
||||||
|
}
|
||||||
|
planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
|
||||||
|
if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
|
||||||
|
t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rebuild of a build made at a commit asks what the module follows now, never the commit.
|
||||||
|
func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
twoTiers(t, open)
|
||||||
|
var refs []string
|
||||||
|
was := askABuild
|
||||||
|
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||||
|
refs = append(refs, ref)
|
||||||
|
return was(c, source, path, ref)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
|
||||||
|
Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
|
||||||
|
t.Fatalf("asked %v at %v", *asked, refs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ask the worker counts out that no machine said it started is cancelled only if no start comes
|
||||||
|
// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
|
||||||
|
func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
|
||||||
|
js, ask := aBuildQueue(t)
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
was := holderLooks
|
||||||
|
holderLooks = 300 * time.Millisecond
|
||||||
|
t.Cleanup(func() { holderLooks = was })
|
||||||
|
id := link.NewBuildID(time.Now())
|
||||||
|
ask(id, "https://forge.example/novox/a.git")
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||||
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a, _ := q.Find(id)
|
||||||
|
if a.State != link.AskInFlight || a.On != "" {
|
||||||
|
t.Fatalf("the taken ask reads %+v", a)
|
||||||
|
}
|
||||||
|
// The holder that took it says it started, while the cancel waits.
|
||||||
|
go func() {
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
_, _ = js.Context().Publish(link.BuildStarted(), started)
|
||||||
|
}()
|
||||||
|
if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
|
||||||
|
t.Fatalf("a build that started was cancelled: %v", err)
|
||||||
|
}
|
||||||
|
if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
|
||||||
|
t.Error("the withdrawn cancel is still marked")
|
||||||
|
}
|
||||||
|
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||||
|
t.Error("a withdrawn cancel recorded an outcome")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ package main
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
@@ -73,6 +75,28 @@ type meshStatus struct {
|
|||||||
// alone. A document without this called a machine well while a predecessor's chain refused
|
// alone. A document without this called a machine well while a predecessor's chain refused
|
||||||
// what the mesh declared open.
|
// what the mesh declared open.
|
||||||
Filtered []machineFiltered `json:"filtered,omitempty"`
|
Filtered []machineFiltered `json:"filtered,omitempty"`
|
||||||
|
// Unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||||
|
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||||
|
// dependency is met. Reported, not refused, until the switch.
|
||||||
|
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||||
|
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
|
||||||
|
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
|
||||||
|
// HandActsUnread says why when it could not be read, rather than reading as none.
|
||||||
|
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
|
||||||
|
HandActsUnread string `json:"handActsUnread,omitempty"`
|
||||||
|
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
|
||||||
|
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
|
||||||
|
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
|
||||||
|
Conditions []conditions.Condition `json:"conditions"`
|
||||||
|
ConditionsUnread string `json:"conditionsUnread,omitempty"`
|
||||||
|
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
|
||||||
|
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
|
||||||
|
// provider says so. A document without it called the mesh well while the identity provider
|
||||||
|
// refused every consumer for a day (04-ISSUES/179).
|
||||||
|
Failing []conditions.Condition `json:"failing,omitempty"`
|
||||||
|
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
|
||||||
|
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
|
||||||
|
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||||
@@ -171,7 +195,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now())}
|
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
|
||||||
// In a stated order, so two readings of an unchanged mesh are the same document.
|
// In a stated order, so two readings of an unchanged mesh are the same document.
|
||||||
untakenNodes := make([]string, 0, len(asked.untaken))
|
untakenNodes := make([]string, 0, len(asked.untaken))
|
||||||
for name := range asked.untaken {
|
for name := range asked.untaken {
|
||||||
@@ -204,6 +228,14 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
|
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
out.Unheld = asked.unheld
|
||||||
|
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||||
|
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
|
||||||
|
if out.Conditions == nil {
|
||||||
|
out.Conditions = []conditions.Condition{}
|
||||||
|
}
|
||||||
|
out.Failing = providerStandings(asked.conditions)
|
||||||
|
out.Overflowing = asked.overflowing
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||||
Node: name, Problem: asked.refused[name]})
|
Node: name, Problem: asked.refused[name]})
|
||||||
|
|||||||
@@ -184,6 +184,7 @@ func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inv
|
|||||||
return inventory.Plan{
|
return inventory.Plan{
|
||||||
ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()),
|
ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()),
|
||||||
Repository: m.Owner + "/" + m.Repo,
|
Repository: m.Owner + "/" + m.Repo,
|
||||||
|
Branch: m.Base,
|
||||||
Commit: m.Commit,
|
Commit: m.Commit,
|
||||||
Created: time.Now().UTC(),
|
Created: time.Now().UTC(),
|
||||||
State: inventory.PlanBuilding,
|
State: inventory.PlanBuilding,
|
||||||
@@ -192,6 +193,57 @@ func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inv
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// supersededBy is what a newer plan takes over from the open plans it supersedes (novox/hq issue
|
||||||
|
// 254, ADR 0218): the modules they had not finished, and those plans closed as superseded.
|
||||||
|
//
|
||||||
|
// **A merge looked at no plan but its own.** Two merges of one repository a few minutes apart were
|
||||||
|
// two open plans asking for the same modules, each sending machines what it built; and a plan that
|
||||||
|
// would never move again — waiting on a report that could not come, at 97b1b2b — stayed open for
|
||||||
|
// ever beside the newer ones, read as work in progress by everyone who looked. The newer merge is the
|
||||||
|
// newer intent for that repository and branch, so its plan takes over: every open plan of the same
|
||||||
|
// repository and branch **created before it** — by the time the plans were made, never by comparing
|
||||||
|
// commits, which have no order of their own — gives up the modules it had not built, and those are
|
||||||
|
// planned again in the newer plan beside what the newer merge moved.
|
||||||
|
//
|
||||||
|
// "Not built" is a module not yet asked, or asked and not answered; **and a module built and not
|
||||||
|
// yet sent to its machines**, where its policy rolls it out: closed, the older plan would never send
|
||||||
|
// it, and the catalogue announces no move for a rebuild (issue 189), so the newer plan builds and
|
||||||
|
// sends it. A build the older plan asked still finishes and registers as any build does — ordered by
|
||||||
|
// when it was asked (issue 219), so the newer plan's ask, made later, is the one that stands.
|
||||||
|
//
|
||||||
|
// A plan with no branch recorded is from before branches were kept, and is superseded by the next
|
||||||
|
// plan of its repository: what it had not built is folded in, so nothing is lost by it.
|
||||||
|
func supersededBy(newer inventory.Plan, open []inventory.Plan, rollsOut func(string) bool) ([]string, []inventory.Plan) {
|
||||||
|
folded := map[string]bool{}
|
||||||
|
var closed []inventory.Plan
|
||||||
|
for _, old := range open {
|
||||||
|
if old.ID == newer.ID || !old.Open() || !strings.EqualFold(old.Repository, newer.Repository) ||
|
||||||
|
(old.Branch != "" && old.Branch != newer.Branch) || !old.Created.Before(newer.Created) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var took []string
|
||||||
|
for name, s := range old.Modules {
|
||||||
|
if s == nil || s.State != "built" || (s.SentAt == nil && rollsOut(name)) {
|
||||||
|
folded[name] = true
|
||||||
|
took = append(took, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(took)
|
||||||
|
old.State = inventory.PlanSuperseded
|
||||||
|
old.Note = fmt.Sprintf("superseded at tier %d by %s (%s at %s)", old.Tier, newer.ID, newer.Repository, short(newer.Commit))
|
||||||
|
if len(took) > 0 {
|
||||||
|
old.Note += "; " + strings.Join(took, ", ") + " planned there again"
|
||||||
|
}
|
||||||
|
closed = append(closed, old)
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(folded))
|
||||||
|
for name := range folded {
|
||||||
|
out = append(out, name)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, closed
|
||||||
|
}
|
||||||
|
|
||||||
// gates is what the next tier needs running from this one: a module of the tier that a later
|
// gates is what the next tier needs running from this one: a module of the tier that a later
|
||||||
// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by
|
// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by
|
||||||
// the machines running it before the next tier is asked. A base an image stands on need only be
|
// the machines running it before the next tier is asked. A base an image stands on need only be
|
||||||
@@ -230,6 +282,22 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// applied says whether every machine running the module has reported since the module was built.
|
// applied says whether every machine running the module has reported since the module was built.
|
||||||
|
// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
|
||||||
|
// after that machine was sent the build (novox/hq issue 256).
|
||||||
|
func appliedEach(module string, since func(node string) time.Time, running []string, reports []inventory.Reported) (bool, []string) {
|
||||||
|
at := map[string]*time.Time{}
|
||||||
|
for _, r := range reports {
|
||||||
|
at[r.Node] = r.At
|
||||||
|
}
|
||||||
|
var waiting []string
|
||||||
|
for _, n := range running {
|
||||||
|
if t := at[n]; t == nil || t.Before(since(n)) {
|
||||||
|
waiting = append(waiting, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(waiting) == 0, waiting
|
||||||
|
}
|
||||||
|
|
||||||
func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
|
func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
|
||||||
at := map[string]*time.Time{}
|
at := map[string]*time.Time{}
|
||||||
for _, r := range reports {
|
for _, r := range reports {
|
||||||
@@ -256,37 +324,52 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
|
|||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
byName[e.Manifest.Module] = e
|
byName[e.Manifest.Module] = e
|
||||||
}
|
}
|
||||||
now := time.Now().UTC()
|
|
||||||
for _, name := range p.Tiers[p.Tier] {
|
for _, name := range p.Tiers[p.Tier] {
|
||||||
state := p.Modules[name]
|
askModule(ctx, p, name, byName)
|
||||||
if state == nil {
|
|
||||||
state = &inventory.PlanModule{}
|
|
||||||
p.Modules[name] = state
|
|
||||||
}
|
|
||||||
e, known := byName[name]
|
|
||||||
if !known {
|
|
||||||
state.State = "failed"
|
|
||||||
state.Why = "no longer in the catalogue"
|
|
||||||
p.State = inventory.PlanFailed
|
|
||||||
p.Note = name + " is no longer in the catalogue"
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
|
||||||
fmt.Printf(" tier %d: ", p.Tier)
|
|
||||||
// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
|
|
||||||
if err := buildOne(ctx, source, e.Source.Path, followedBranch(e.Source.Ref), 0); err != nil {
|
|
||||||
state.State = "failed"
|
|
||||||
state.Why = err.Error()
|
|
||||||
p.State = inventory.PlanFailed
|
|
||||||
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
state.State = "asked"
|
|
||||||
state.AskedAt = &now
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A
|
||||||
|
// variable so a test of what a plan does around an ask needs no build machine.
|
||||||
|
var askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
|
||||||
|
return buildOneAsked(ctx, source, path, ref, 0, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
|
||||||
|
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
|
||||||
|
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
state := p.Modules[name]
|
||||||
|
if state == nil {
|
||||||
|
state = &inventory.PlanModule{}
|
||||||
|
p.Modules[name] = state
|
||||||
|
}
|
||||||
|
e, known := byName[name]
|
||||||
|
if !known {
|
||||||
|
state.State = "failed"
|
||||||
|
state.Why = "no longer in the catalogue"
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = name + " is no longer in the catalogue"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||||
|
fmt.Printf(" tier %d: ", p.Tier)
|
||||||
|
// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
|
||||||
|
id, err := askABuild(ctx, source, e.Source.Path, followedBranch(e.Source.Ref))
|
||||||
|
if err != nil {
|
||||||
|
state.State = "failed"
|
||||||
|
state.Why = err.Error()
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
state.State = "asked"
|
||||||
|
state.AskedAt = &now
|
||||||
|
state.Build = id
|
||||||
|
state.Why, state.Commit, state.BuiltAt = "", "", nil
|
||||||
|
}
|
||||||
|
|
||||||
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
|
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
|
||||||
// advances what that completes. Called from the daemon's take-in of every outcome.
|
// advances what that completes. Called from the daemon's take-in of every outcome.
|
||||||
//
|
//
|
||||||
@@ -295,7 +378,7 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
|
|||||||
// the later plan's answer — it stood on the bases from before the later plan's merge, and taking it
|
// the later plan's answer — it stood on the bases from before the later plan's merge, and taking it
|
||||||
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
|
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
|
||||||
// build's request time is not known, and such an outcome is taken as before.
|
// build's request time is not known, and such an outcome is taken as before.
|
||||||
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time) {
|
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
||||||
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
||||||
@@ -331,7 +414,17 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
|||||||
state = &inventory.PlanModule{}
|
state = &inventory.PlanModule{}
|
||||||
p.Modules[module] = state
|
p.Modules[module] = state
|
||||||
}
|
}
|
||||||
if askedBefore(asked, state.AskedAt) {
|
// **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module
|
||||||
|
// is, as before, when it was asked at or after the plan's ask (issue 219).
|
||||||
|
// **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR
|
||||||
|
// 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's
|
||||||
|
// build, made from other source, and settling the plan with it would send that. A plan from
|
||||||
|
// before ids were kept is matched as it was: by when the build was asked (issue 219).
|
||||||
|
if state.Build != "" {
|
||||||
|
if state.Build != id {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else if askedBefore(asked, state.AskedAt) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if failed != "" {
|
if failed != "" {
|
||||||
@@ -339,6 +432,10 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
|||||||
state.Why = failed
|
state.Why = failed
|
||||||
p.State = inventory.PlanFailed
|
p.State = inventory.PlanFailed
|
||||||
p.Note = fmt.Sprintf("%s failed to build in tier %d", module, p.Tier)
|
p.Note = fmt.Sprintf("%s failed to build in tier %d", module, p.Tier)
|
||||||
|
sayUnsent(p, func(m string) bool {
|
||||||
|
u, err := inv.UpgradeOf(ctx, m)
|
||||||
|
return err == nil && u.RollOut
|
||||||
|
})
|
||||||
} else {
|
} else {
|
||||||
state.State = "built"
|
state.State = "built"
|
||||||
state.BuiltAt = &now
|
state.BuiltAt = &now
|
||||||
@@ -400,8 +497,17 @@ func advanceHeld(ctx context.Context, open *stores) {
|
|||||||
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
|
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("%s: %v\n", p.ID, err)
|
fmt.Printf("%s: %v\n", p.ID, err)
|
||||||
|
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||||
|
// the state is left as it was and the step is tried again on the next tick.
|
||||||
|
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||||
|
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||||
|
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||||
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
if p.State == inventory.PlanFailed {
|
||||||
|
sayUnsent(p, rollsOut)
|
||||||
|
}
|
||||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||||
break
|
break
|
||||||
@@ -442,6 +548,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
||||||
// outcome, whoever was listening.
|
// outcome, whoever was listening.
|
||||||
recorded := map[string][]inventory.Build{}
|
recorded := map[string][]inventory.Build{}
|
||||||
|
byID := map[string]inventory.Build{}
|
||||||
for _, m := range tier {
|
for _, m := range tier {
|
||||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||||
builds, err := inv.Builds(ctx, m, 5)
|
builds, err := inv.Builds(ctx, m, 5)
|
||||||
@@ -449,9 +556,19 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
recorded[m] = builds
|
recorded[m] = builds
|
||||||
|
// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
|
||||||
|
if s.Build != "" {
|
||||||
|
b, found, err := inv.BuildByID(ctx, s.Build)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
byID[s.Build] = b
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if settleFromRecords(p, tier, recorded) {
|
if settleFromRecords(p, tier, recorded, byID) {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
// Asked: wait for every build.
|
// Asked: wait for every build.
|
||||||
@@ -470,6 +587,15 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
// a module that packages another repository's source, keeps its commit; the catalogue announces
|
// a module that packages another repository's source, keeps its commit; the catalogue announces
|
||||||
// no move for it and its machines would keep the old image until somebody pushed (novox/hq
|
// no move for it and its machines would keep the old image until somebody pushed (novox/hq
|
||||||
// issue 189). A module whose policy records is built and left, as its policy says.
|
// issue 189). A module whose policy records is built and left, as its policy says.
|
||||||
|
//
|
||||||
|
// **One machine first, unless the module's policy says together** (novox/hq issue 249, ADR
|
||||||
|
// 0218). The plan sent every machine running the module at once, and the operator's policy —
|
||||||
|
// one at a time, stopping at the first that fails, which an announced upgrade honours — was not
|
||||||
|
// read here at all: a module whose new declarations broke it broke everywhere in the same
|
||||||
|
// minute. Now the first machine is sent, the plan records it and waits for that machine's report
|
||||||
|
// after the send to say it applied what it was sent; only then are the rest sent. A first machine
|
||||||
|
// that fails or refuses stops the module's rollout and the plan with it, the rest untouched.
|
||||||
|
var pending []string
|
||||||
for _, m := range tier {
|
for _, m := range tier {
|
||||||
state := p.Modules[m]
|
state := p.Modules[m]
|
||||||
if state == nil || state.SentAt != nil || !rollsOut(m) {
|
if state == nil || state.SentAt != nil || !rollsOut(m) {
|
||||||
@@ -479,17 +605,64 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
policy, err := inv.UpgradeOf(ctx, m)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
var reports []inventory.Reported
|
||||||
|
if !policy.Together {
|
||||||
|
// Read for the choice of the first machine as well as for its report.
|
||||||
|
if reports, err = inv.LastReports(ctx); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
state.SentAt = &now
|
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
|
||||||
if len(running) == 0 {
|
switch {
|
||||||
|
case step.failed != "":
|
||||||
|
state.Why = step.failed
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = fmt.Sprintf("%s stopped at its first machine in tier %d: %s; %s left as it was",
|
||||||
|
m, p.Tier, step.failed, orNone(strings.Join(step.rest, ", ")))
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
case step.waiting != "":
|
||||||
|
pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Local().Format("15:04")))
|
||||||
|
continue
|
||||||
|
case len(step.send) == 0:
|
||||||
|
// No machine runs it: nothing to send, and nothing to wait for.
|
||||||
|
state.SentAt = &now
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := sendTo(ctx, open, running); err != nil {
|
// What sendToEach answers, not what was asked: the machine holding the bus is sent before
|
||||||
return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(running, ", "), p.Tier, err)
|
// the first when its user list must change (issue 249), and the plan waits for it too.
|
||||||
|
sent, err := sendToEach(ctx, open, step.send)
|
||||||
|
if err != nil {
|
||||||
|
// Not marked sent, so the next step tries again (issue 249): a grant that could not be
|
||||||
|
// issued is a send that did not happen.
|
||||||
|
return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(step.send, ", "), p.Tier, err)
|
||||||
}
|
}
|
||||||
fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(running, ", "))
|
if step.first {
|
||||||
|
state.First = sent
|
||||||
|
state.FirstAt = &now
|
||||||
|
p.State = inventory.PlanRolling
|
||||||
|
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first", p.Tier, m, strings.Join(sent, ", "))
|
||||||
|
fmt.Printf("%s: tier %d built; sent %s to %s first, the rest once it reports it applied\n",
|
||||||
|
p.ID, p.Tier, m, strings.Join(sent, ", "))
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
state.SentAt = &now
|
||||||
|
fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(sent, ", "))
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
if len(pending) > 0 {
|
||||||
|
note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(pending, "; ") +
|
||||||
|
" to report it applied before the rest are sent"
|
||||||
|
changed := p.State != inventory.PlanRolling || p.Note != note
|
||||||
|
p.State = inventory.PlanRolling
|
||||||
|
p.Note = note
|
||||||
|
return changed, nil
|
||||||
|
}
|
||||||
// And wait for what the next tier needs running.
|
// And wait for what the next tier needs running.
|
||||||
needed := gates(*p, edges, rollsOut)
|
needed := gates(*p, edges, rollsOut)
|
||||||
if len(needed) > 0 {
|
if len(needed) > 0 {
|
||||||
@@ -516,10 +689,24 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
if state.BuiltAt != nil {
|
if state.BuiltAt != nil {
|
||||||
since = *state.BuiltAt
|
since = *state.BuiltAt
|
||||||
}
|
}
|
||||||
if state.SentAt != nil && state.SentAt.After(since) {
|
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
||||||
since = *state.SentAt
|
// a module is sent twice — the first machine, then the rest — and SentAt is the second.
|
||||||
|
// Asked of every machine, the first machine's report, made between the two sends, read as
|
||||||
|
// older than the build, and the gate waited for a report it already had, for ever.
|
||||||
|
sinceFor := func(node string) time.Time {
|
||||||
|
at := since
|
||||||
|
sent := state.SentAt
|
||||||
|
for _, n := range state.First {
|
||||||
|
if n == node {
|
||||||
|
sent = state.FirstAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if sent != nil && sent.After(at) {
|
||||||
|
at = *sent
|
||||||
|
}
|
||||||
|
return at
|
||||||
}
|
}
|
||||||
if ok, on := applied(m, since, running, reports); !ok {
|
if ok, on := appliedEach(m, sinceFor, running, reports); !ok {
|
||||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -540,6 +727,110 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rolloutStep is what a plan does next with one built module's machines (novox/hq issue 249).
|
||||||
|
type rolloutStep struct {
|
||||||
|
// send is the machines to send now; first, whether they are the first machine's send.
|
||||||
|
send []string
|
||||||
|
first bool
|
||||||
|
// waiting names the first machines whose report the rest wait for.
|
||||||
|
waiting string
|
||||||
|
// failed says how a first machine did not take it; rest is what is then left alone.
|
||||||
|
failed string
|
||||||
|
rest []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// nextRollout is the next step of one module's rollout in a plan (novox/hq issue 249, ADR 0218).
|
||||||
|
//
|
||||||
|
// Together, every machine running it at once, as the policy says. Otherwise one machine first — the
|
||||||
|
// first by name among those that have reported within the bound, so a laptop that is away is not
|
||||||
|
// the one the rest wait on; the first by name when none has; the same choice on every controller and
|
||||||
|
// every resume — and the rest once each machine the first send reached reports, about the
|
||||||
|
// declaration it was last sent, that it applied it. Compared by the store's own record of what was
|
||||||
|
// sent (Reported.Current), never by this controller's clock against the machine's.
|
||||||
|
//
|
||||||
|
// **A first machine that fails, refuses, or does not report within the bound stops the rollout
|
||||||
|
// there** (ADR 0218 §2), naming the machine; the rest are not sent. A wait with no end is not a
|
||||||
|
// rollout: it held the plan open for ever, read as work in progress (issue 254).
|
||||||
|
func nextRollout(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported,
|
||||||
|
now time.Time, bound time.Duration) rolloutStep {
|
||||||
|
if len(running) == 0 {
|
||||||
|
return rolloutStep{}
|
||||||
|
}
|
||||||
|
if together {
|
||||||
|
return rolloutStep{send: running}
|
||||||
|
}
|
||||||
|
byNode := map[string]inventory.Reported{}
|
||||||
|
for _, r := range reports {
|
||||||
|
byNode[r.Node] = r
|
||||||
|
}
|
||||||
|
if s.FirstAt == nil {
|
||||||
|
sorted := append([]string{}, running...)
|
||||||
|
sort.Strings(sorted)
|
||||||
|
for _, n := range sorted {
|
||||||
|
if r, said := byNode[n]; said && r.At != nil && now.Sub(*r.At) <= bound {
|
||||||
|
return rolloutStep{send: []string{n}, first: true}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rolloutStep{send: sorted[:1], first: true}
|
||||||
|
}
|
||||||
|
sentFirst := map[string]bool{}
|
||||||
|
for _, n := range s.First {
|
||||||
|
sentFirst[n] = true
|
||||||
|
}
|
||||||
|
var rest []string
|
||||||
|
for _, n := range running {
|
||||||
|
if !sentFirst[n] {
|
||||||
|
rest = append(rest, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var waiting, failed []string
|
||||||
|
for _, n := range s.First {
|
||||||
|
r, said := byNode[n]
|
||||||
|
// Only a report about what it was last sent says anything about this build.
|
||||||
|
if !said || r.At == nil || !r.Current {
|
||||||
|
waiting = append(waiting, n)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch r.Outcome {
|
||||||
|
case inventory.OutcomeApplied:
|
||||||
|
case inventory.OutcomeFailed, inventory.OutcomeRefused:
|
||||||
|
failed = append(failed, n+" "+r.Outcome+" what it was sent")
|
||||||
|
default:
|
||||||
|
waiting = append(waiting, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(failed) > 0 {
|
||||||
|
return rolloutStep{failed: strings.Join(failed, "; "), rest: rest}
|
||||||
|
}
|
||||||
|
if len(waiting) > 0 {
|
||||||
|
if now.Sub(*s.FirstAt) > bound {
|
||||||
|
return rolloutStep{failed: fmt.Sprintf("%s did not report it applied within %s",
|
||||||
|
strings.Join(waiting, ", "), bound), rest: rest}
|
||||||
|
}
|
||||||
|
return rolloutStep{waiting: strings.Join(waiting, ", ")}
|
||||||
|
}
|
||||||
|
return rolloutStep{send: rest}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayUnsent adds to an ended plan's note the modules it built and never sent (novox/hq issue 249).
|
||||||
|
// An announced move of a module a plan held was left to that plan; a plan that ends without
|
||||||
|
// sending it — failed elsewhere, or closed by hand — would leave its machines behind with nothing
|
||||||
|
// saying so. A module whose rollout stopped at its first machine is not among them: that stop was
|
||||||
|
// the point. Said once.
|
||||||
|
func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
|
||||||
|
var unsent []string
|
||||||
|
for name, s := range p.Modules {
|
||||||
|
if s != nil && s.State == "built" && s.SentAt == nil && s.FirstAt == nil && rollsOut(name) {
|
||||||
|
unsent = append(unsent, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unsent) == 0 || strings.Contains(p.Note, "built and never sent") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sort.Strings(unsent)
|
||||||
|
p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
|
||||||
|
}
|
||||||
|
|
||||||
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
|
||||||
func planTicker(ctx context.Context, open *stores) {
|
func planTicker(ctx context.Context, open *stores) {
|
||||||
advancePlans(ctx, open)
|
advancePlans(ctx, open)
|
||||||
@@ -556,15 +847,24 @@ func planTicker(ctx context.Context, open *stores) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// planLine is one plan as `status` says it.
|
// planLine is one plan as `status` says it.
|
||||||
func planLine(p inventory.Plan, now time.Time) string {
|
func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) }
|
||||||
|
|
||||||
|
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
|
||||||
|
// waiting on builds nobody will take until a person resumes the seat says so, and is not late.
|
||||||
|
func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
|
||||||
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
|
||||||
switch p.State {
|
switch p.State {
|
||||||
case inventory.PlanDone:
|
case inventory.PlanDone:
|
||||||
return fmt.Sprintf("%s %s done, %d tier(s)", p.Repository, short(p.Commit), len(p.Tiers))
|
return fmt.Sprintf("%s %s done, %d tier(s)", p.Repository, short(p.Commit), len(p.Tiers))
|
||||||
case inventory.PlanFailed:
|
case inventory.PlanFailed:
|
||||||
return fmt.Sprintf("%s %s FAILED at %s: %s", p.Repository, short(p.Commit), where, p.Note)
|
return fmt.Sprintf("%s %s FAILED at %s: %s", p.Repository, short(p.Commit), where, p.Note)
|
||||||
|
case inventory.PlanSuperseded:
|
||||||
|
return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
|
||||||
|
}
|
||||||
|
since := now.Sub(p.Updated).Round(time.Second)
|
||||||
|
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||||
|
return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
|
||||||
}
|
}
|
||||||
since := now.Sub(p.Updated).Round(time.Minute)
|
|
||||||
late := ""
|
late := ""
|
||||||
if since > planWaitBound {
|
if since > planWaitBound {
|
||||||
late = " — LATE"
|
late = " — LATE"
|
||||||
@@ -578,20 +878,49 @@ func planLine(p inventory.Plan, now time.Time) string {
|
|||||||
|
|
||||||
// planFailedBuild marks the module a failed build was for when the result names no module: by the
|
// planFailedBuild marks the module a failed build was for when the result names no module: by the
|
||||||
// repository and path the plan's modules were asked at.
|
// repository and path the plan's modules were asked at.
|
||||||
|
//
|
||||||
|
// **By the id first** (novox/hq ADR 0219): a plan keeps the id it asked each module under, so an
|
||||||
|
// outcome that never learnt its module's name — cancelled, killed, failed at the clone — is matched
|
||||||
|
// to the module it was asked for exactly. Repository and path remain for a plan from before ids
|
||||||
|
// were kept.
|
||||||
func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) {
|
func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) {
|
||||||
|
asked, _ := link.BuildAskedAt(result.ID)
|
||||||
|
if plans, err := open.inventory.OpenPlans(ctx); err == nil {
|
||||||
|
if module := moduleAskedAs(plans, result.ID); module != "" {
|
||||||
|
planBuilt(ctx, open, module, result.Commit, result.Failed, asked, result.ID)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
entries, err := open.inventory.Catalogued(ctx)
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path {
|
if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path {
|
||||||
asked, _ := link.BuildAskedAt(result.ID)
|
planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked, result.ID)
|
||||||
planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// moduleAskedAs is the module an open plan's current tier asked for under this id, or nothing.
|
||||||
|
func moduleAskedAs(plans []inventory.Plan, id string) string {
|
||||||
|
if id == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
for _, p := range plans {
|
||||||
|
if p.Tier >= len(p.Tiers) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.Build == id {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
func repositoryMatches(a, b string) bool {
|
func repositoryMatches(a, b string) bool {
|
||||||
trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) }
|
trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) }
|
||||||
return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a))
|
return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a))
|
||||||
@@ -610,7 +939,7 @@ type planStatus struct {
|
|||||||
Late bool `json:"late"`
|
Late bool `json:"late"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
|
func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus {
|
||||||
out := make([]planStatus, 0, len(plans))
|
out := make([]planStatus, 0, len(plans))
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
|
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
|
||||||
@@ -621,6 +950,10 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
|
|||||||
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
|
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
|
||||||
}
|
}
|
||||||
ps.Late = now.Sub(p.Updated) > planWaitBound
|
ps.Late = now.Sub(p.Updated) > planWaitBound
|
||||||
|
// Paused is a person's decision, not lateness (novox/hq ADR 0219).
|
||||||
|
if waiting, paused := pausedWaiting(p, pause, now); paused {
|
||||||
|
ps.Waiting, ps.Late = waiting, false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
out = append(out, ps)
|
out = append(out, ps)
|
||||||
}
|
}
|
||||||
@@ -628,12 +961,15 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
|
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
|
||||||
func openPlans(plans []inventory.Plan) ([]inventory.Plan, int) {
|
func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) {
|
||||||
var open []inventory.Plan
|
var open []inventory.Plan
|
||||||
late := 0
|
late := 0
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
if p.Open() {
|
if p.Open() {
|
||||||
open = append(open, p)
|
open = append(open, p)
|
||||||
|
if _, paused := pausedWaiting(p, pause, time.Now()); paused {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if time.Since(p.Updated) > planWaitBound {
|
if time.Since(p.Updated) > planWaitBound {
|
||||||
late++
|
late++
|
||||||
}
|
}
|
||||||
@@ -650,10 +986,18 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
||||||
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
||||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||||
|
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||||
|
why := addHandActFlags(set)
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||||
|
// Refused before anything is opened: a repair by hand says why.
|
||||||
|
if err := why.require("plans " + positionals[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -666,7 +1010,7 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s — %s\n", p.ID, planLine(p, now))
|
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
|
||||||
for i, tier := range p.Tiers {
|
for i, tier := range p.Tiers {
|
||||||
marker := " "
|
marker := " "
|
||||||
if i == p.Tier && p.Open() {
|
if i == p.Tier && p.Open() {
|
||||||
@@ -681,6 +1025,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if s.Commit != "" {
|
if s.Commit != "" {
|
||||||
state += " from " + short(s.Commit)
|
state += " from " + short(s.Commit)
|
||||||
}
|
}
|
||||||
|
if s.Build != "" && s.State != "built" {
|
||||||
|
state += " (" + s.Build + ")"
|
||||||
|
}
|
||||||
if s.Why != "" {
|
if s.Why != "" {
|
||||||
state += ": " + s.Why
|
state += ": " + s.Why
|
||||||
}
|
}
|
||||||
@@ -693,7 +1040,28 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if *whatIf != "" {
|
if *whatIf != "" {
|
||||||
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
|
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
|
||||||
}
|
}
|
||||||
if len(positionals) == 2 && positionals[0] == "stop" {
|
// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
|
||||||
|
if len(positionals) == 2 && positionals[0] == "retry" {
|
||||||
|
said, err := retryPlan(ctx, open, positionals[1])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one
|
||||||
|
// waiting on a report that cannot come — so it stops reading as work in progress. Marked failed
|
||||||
|
// with who ended it; what it asked still builds and registers.
|
||||||
|
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||||
|
how := "stopped"
|
||||||
|
if positionals[0] == "close" {
|
||||||
|
how = "closed"
|
||||||
|
}
|
||||||
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
p, err := inv.PlanByID(ctx, positionals[1])
|
p, err := inv.PlanByID(ctx, positionals[1])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -701,26 +1069,18 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if !p.Open() {
|
if !p.Open() {
|
||||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||||
}
|
}
|
||||||
|
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||||
p.State = inventory.PlanFailed
|
p.State = inventory.PlanFailed
|
||||||
p.Note = "stopped by hand at tier " + fmt.Sprint(p.Tier)
|
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||||
release, err := inv.HoldPlans(ctx, true)
|
sayUnsent(&p, func(m string) bool {
|
||||||
if err != nil {
|
u, err := inv.UpgradeOf(ctx, m)
|
||||||
return err
|
return err == nil && u.RollOut
|
||||||
}
|
})
|
||||||
defer release()
|
|
||||||
if p, err = inv.PlanByID(ctx, positionals[1]); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !p.Open() {
|
|
||||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
|
||||||
}
|
|
||||||
p.State = inventory.PlanFailed
|
|
||||||
p.Note = "stopped by hand at tier " + fmt.Sprint(p.Tier)
|
|
||||||
if err := inv.SavePlan(ctx, p); err != nil {
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("%s stopped at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
||||||
p.ID, p.Tier, len(p.Tiers))
|
p.ID, how, p.Tier, len(p.Tiers))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
plans, err := inv.RecentPlans(ctx, *limit)
|
plans, err := inv.RecentPlans(ctx, *limit)
|
||||||
@@ -731,8 +1091,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
fmt.Println("no merge has produced a plan yet")
|
fmt.Println("no merge has produced a plan yet")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
pause := buildSeatPause(ctx, inv, plans)
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
fmt.Printf("%-28s %s\n", p.ID, planLine(p, now))
|
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -795,7 +1156,13 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
|
|||||||
how := "built; its policy records, so nothing is sent"
|
how := "built; its policy records, so nothing is sent"
|
||||||
if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut {
|
if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut {
|
||||||
running, _ := inv.Running(ctx, name)
|
running, _ := inv.Running(ctx, name)
|
||||||
|
reports, _ := inv.LastReports(ctx)
|
||||||
how = "built, then sent to " + orNone(strings.Join(running, ", "))
|
how = "built, then sent to " + orNone(strings.Join(running, ", "))
|
||||||
|
// One machine first unless the policy says together (novox/hq issue 249).
|
||||||
|
if first := nextRollout(inventory.PlanModule{}, running, u.Together, reports, time.Now(), planWaitBound); first.first && len(running) > 1 {
|
||||||
|
how = fmt.Sprintf("built, then sent to %s first and to the rest once it has applied it",
|
||||||
|
first.send[0])
|
||||||
|
}
|
||||||
rolls[name] = how
|
rolls[name] = how
|
||||||
}
|
}
|
||||||
fmt.Printf(" %-22s %s\n", name, how)
|
fmt.Printf(" %-22s %s\n", name, how)
|
||||||
@@ -828,7 +1195,12 @@ func splitList(s string) []string {
|
|||||||
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
|
||||||
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
|
||||||
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
|
||||||
func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build) bool {
|
//
|
||||||
|
// The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR
|
||||||
|
// 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is
|
||||||
|
// found by nothing else.
|
||||||
|
func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build,
|
||||||
|
byID map[string]inventory.Build) bool {
|
||||||
changed := false
|
changed := false
|
||||||
for _, m := range tier {
|
for _, m := range tier {
|
||||||
s := p.Modules[m]
|
s := p.Modules[m]
|
||||||
@@ -837,6 +1209,10 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
|
|||||||
}
|
}
|
||||||
var outcome *inventory.Build
|
var outcome *inventory.Build
|
||||||
for i := range recorded[m] {
|
for i := range recorded[m] {
|
||||||
|
// Asked under an id, only that id's record answers (ADR 0219), and it is looked up below.
|
||||||
|
if s.Build != "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
b := recorded[m][i]
|
b := recorded[m][i]
|
||||||
if b.At.Before(*s.AskedAt) {
|
if b.At.Before(*s.AskedAt) {
|
||||||
break
|
break
|
||||||
@@ -848,6 +1224,9 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
|
|||||||
}
|
}
|
||||||
outcome = &b
|
outcome = &b
|
||||||
}
|
}
|
||||||
|
if own, found := byID[s.Build]; s.Build != "" && found {
|
||||||
|
outcome = &own
|
||||||
|
}
|
||||||
if outcome == nil {
|
if outcome == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
|||||||
// Only a build from before the ask: not this ask's outcome.
|
// Only a build from before the ask: not this ask's outcome.
|
||||||
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
|
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
|
||||||
}
|
}
|
||||||
if !settleFromRecords(&p, p.Tiers[0], records) {
|
if !settleFromRecords(&p, p.Tiers[0], records, nil) {
|
||||||
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
|
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
|
||||||
}
|
}
|
||||||
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
|
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
|
||||||
@@ -162,13 +162,13 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
|||||||
late := map[string][]inventory.Build{"postgres": {
|
late := map[string][]inventory.Build{"postgres": {
|
||||||
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
|
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
|
||||||
}}
|
}}
|
||||||
if settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "asked" {
|
if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
|
||||||
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
|
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
|
||||||
}
|
}
|
||||||
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
|
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
|
||||||
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
|
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
|
||||||
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
|
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
|
||||||
if !settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "built" ||
|
if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
|
||||||
r.Modules["postgres"].Commit != "4bcd5f73" {
|
r.Modules["postgres"].Commit != "4bcd5f73" {
|
||||||
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
|
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
|
||||||
}
|
}
|
||||||
@@ -176,8 +176,39 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
|||||||
// A failure recorded after the ask fails the plan, as hearing it would have.
|
// A failure recorded after the ask fails the plan, as hearing it would have.
|
||||||
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
|
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
|
||||||
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
|
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
|
||||||
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}})
|
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
|
||||||
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
|
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
|
||||||
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
|
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The first machine's report, made between the send to it and the send to the rest, opens the gate for it:
|
||||||
|
// each machine is judged from its own send, not from the last one (novox/hq issue 256).
|
||||||
|
func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
|
||||||
|
built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC)
|
||||||
|
firstSent := built.Add(31 * time.Second)
|
||||||
|
firstReported := built.Add(43 * time.Second)
|
||||||
|
restSent := built.Add(58 * time.Second)
|
||||||
|
restReported := built.Add(74 * time.Second)
|
||||||
|
reports := []inventory.Reported{
|
||||||
|
{Node: "ace", At: &firstReported},
|
||||||
|
{Node: "g14", At: &restReported},
|
||||||
|
}
|
||||||
|
since := func(node string) time.Time {
|
||||||
|
if node == "ace" {
|
||||||
|
return firstSent
|
||||||
|
}
|
||||||
|
return restSent
|
||||||
|
}
|
||||||
|
if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok {
|
||||||
|
t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting)
|
||||||
|
}
|
||||||
|
// The old reading, every machine from the last send, is what held the plan.
|
||||||
|
if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok {
|
||||||
|
t.Fatal("the single-moment reading should hold the first machine back")
|
||||||
|
}
|
||||||
|
early := built.Add(10 * time.Second)
|
||||||
|
if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" {
|
||||||
|
t.Fatal("a report from before the machine was sent opened the gate")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
|
|||||||
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||||
// would bury the ones that matter under a list nobody can act on.
|
// would bury the ones that matter under a list nobody can act on.
|
||||||
func speaksOnTheBus(m catalogue.Manifest) bool {
|
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only
|
||||||
|
// once that machine has reported it applied; a module whose policy says together goes everywhere at
|
||||||
|
// once, as before.
|
||||||
|
func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) {
|
||||||
|
running := []string{"novox", "ace", "g14"}
|
||||||
|
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
now := sentAt.Add(5 * time.Minute)
|
||||||
|
bound := 30 * time.Minute
|
||||||
|
after := sentAt.Add(time.Minute)
|
||||||
|
next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep {
|
||||||
|
return nextRollout(s, running, together, reports, now, bound)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Together: every machine at once.
|
||||||
|
if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first {
|
||||||
|
t.Fatalf("a together policy did not send every machine at once: %+v", step)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise the first by name, alone, when none has reported lately.
|
||||||
|
step := next(inventory.PlanModule{}, running, false, nil)
|
||||||
|
if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) {
|
||||||
|
t.Fatalf("the first send was %+v, wanted ace alone", step)
|
||||||
|
}
|
||||||
|
|
||||||
|
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||||
|
report := func(outcome string, current bool) []inventory.Reported {
|
||||||
|
return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current},
|
||||||
|
{Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No report yet, or one about an older declaration than it was last sent: wait.
|
||||||
|
for what, reports := range map[string][]inventory.Reported{
|
||||||
|
"no report": nil,
|
||||||
|
"a report about older": report(inventory.OutcomeApplied, false),
|
||||||
|
} {
|
||||||
|
step := next(state, running, false, reports)
|
||||||
|
if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" {
|
||||||
|
t.Errorf("%s: %+v, wanted to wait for ace", what, step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Applied what it was last sent: the rest, and only the rest.
|
||||||
|
step = next(state, running, false, report(inventory.OutcomeApplied, true))
|
||||||
|
if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) {
|
||||||
|
t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failed or refused: stop, the rest untouched.
|
||||||
|
for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} {
|
||||||
|
step := next(state, running, false, report(outcome, true))
|
||||||
|
if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) ||
|
||||||
|
!reflect.DeepEqual(step.rest, []string{"novox", "g14"}) {
|
||||||
|
t.Errorf("a first machine that %s it: %+v", outcome, step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The machine holding the bus went with the first send: the rest wait for it too, and it is
|
||||||
|
// not sent again.
|
||||||
|
both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt}
|
||||||
|
half := report(inventory.OutcomeApplied, true)
|
||||||
|
if step := next(both, running, false, half); step.waiting != "novox" {
|
||||||
|
t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step)
|
||||||
|
}
|
||||||
|
all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true})
|
||||||
|
if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||||
|
t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One machine, or none: nothing is waited for that cannot come.
|
||||||
|
if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first {
|
||||||
|
t.Fatalf("a module nothing runs was sent: %+v", step)
|
||||||
|
}
|
||||||
|
if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" {
|
||||||
|
t.Fatalf("a module on one machine waited for more: %+v", step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there,
|
||||||
|
// naming the machine and the bound; the rest are left alone.
|
||||||
|
func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) {
|
||||||
|
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||||
|
step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute)
|
||||||
|
if !strings.Contains(step.failed, "ace did not report it applied within 30m") ||
|
||||||
|
!reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 {
|
||||||
|
t.Fatalf("a silent first machine: %+v", step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first machine is the first by name among those heard from lately: a laptop that is away is
|
||||||
|
// not the one the rest wait on. When none has been heard from, the first by name.
|
||||||
|
func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour)
|
||||||
|
reports := []inventory.Reported{
|
||||||
|
{Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately},
|
||||||
|
}
|
||||||
|
step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute)
|
||||||
|
if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||||
|
t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An announced move of a module an open plan is still rolling out is left to the plan: sending it
|
||||||
|
// here as well put the bundle on every machine at once (novox/hq issue 249).
|
||||||
|
func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) {
|
||||||
|
sent := time.Now()
|
||||||
|
plans := []inventory.Plan{
|
||||||
|
{ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}},
|
||||||
|
{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||||
|
"agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}},
|
||||||
|
}
|
||||||
|
if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" {
|
||||||
|
t.Fatalf("a module the plan is rolling out was not left to it: %q", got)
|
||||||
|
}
|
||||||
|
for _, m := range []string{"gitea", "keycloak"} {
|
||||||
|
if got := rolledOutByAPlan(plans, m); got != "" {
|
||||||
|
t.Errorf("%s, which no plan will send, was left to %s", m, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan that ends without sending what it built says so, with the remedy; a module whose rollout
|
||||||
|
// stopped at its first machine is not among them.
|
||||||
|
func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) {
|
||||||
|
at := time.Now()
|
||||||
|
p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1",
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"agent": {State: "built"},
|
||||||
|
"stopped": {State: "built", First: []string{"ace"}, FirstAt: &at},
|
||||||
|
"sent": {State: "built", SentAt: &at},
|
||||||
|
"notes": {State: "built"},
|
||||||
|
"later": {},
|
||||||
|
}}
|
||||||
|
rollsOut := func(m string) bool { return m != "notes" }
|
||||||
|
sayUnsent(&p, rollsOut)
|
||||||
|
sayUnsent(&p, rollsOut)
|
||||||
|
if p.Note != "closed by hand at tier 1; built and never sent: agent — `push --behind` sends them" {
|
||||||
|
t.Fatalf("the note reads %q", p.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
)
|
)
|
||||||
|
|
||||||
// rotateCommand replaces a credential and moves both ends together.
|
// rotateCommand replaces a credential and moves both ends together.
|
||||||
@@ -33,12 +35,16 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||||
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||||
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||||
|
// One consuming module rather than every module on the machine. A machine runs many consumers
|
||||||
|
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
|
||||||
|
// issue 268) is no reason to restart every other one on the machine.
|
||||||
|
module := set.String("module", "", "only this consuming module's credential")
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(positionals) != 1 {
|
if len(positionals) != 1 {
|
||||||
return errors.New("rotate <provision> [--consumer <machine>]")
|
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
|
||||||
}
|
}
|
||||||
provision := positionals[0]
|
provision := positionals[0]
|
||||||
|
|
||||||
@@ -53,6 +59,12 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
holders = ofModule(holders, *module)
|
||||||
|
if len(holders) == 0 && *module != "" {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
|
||||||
|
"says what a machine holds", *module, onMachine(*only), provision)
|
||||||
|
}
|
||||||
if len(holders) == 0 {
|
if len(holders) == 0 {
|
||||||
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||||
// and a rotation somebody believes happened is worse than one they know did not.
|
// and a rotation somebody believes happened is worse than one they know did not.
|
||||||
@@ -116,6 +128,27 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ofModule is the holders whose consuming module is this one; all of them when none is named.
|
||||||
|
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
|
||||||
|
if module == "" {
|
||||||
|
return holders
|
||||||
|
}
|
||||||
|
var out []inventory.Holder
|
||||||
|
for _, h := range holders {
|
||||||
|
if h.ConsumerModule == module {
|
||||||
|
out = append(out, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func onMachine(machine string) string {
|
||||||
|
if machine == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return " on " + machine
|
||||||
|
}
|
||||||
|
|
||||||
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
||||||
func asLocal(local string) string {
|
func asLocal(local string) string {
|
||||||
if local == "" {
|
if local == "" {
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
|
||||||
|
// 268): a module that leaked its database password is no reason to restart every other consumer.
|
||||||
|
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
|
||||||
|
holders := []inventory.Holder{
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
|
||||||
|
}
|
||||||
|
got := ofModule(holders, "letta")
|
||||||
|
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
|
||||||
|
t.Fatalf("narrowed to letta: %+v", got)
|
||||||
|
}
|
||||||
|
if len(ofModule(holders, "")) != 3 {
|
||||||
|
t.Fatal("no module named narrowed anyway")
|
||||||
|
}
|
||||||
|
if len(ofModule(holders, "absent")) != 0 {
|
||||||
|
t.Fatal("a module holding nothing matched")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq ADR 0207 at the controller's acts: `assign` refuses a module whose resources a
|
||||||
|
// seat nothing on the node holds applies, `unassign` refuses taking the last holder from under its
|
||||||
|
// dependents, and `status` reports what composition does not yet refuse.
|
||||||
|
|
||||||
|
func serviceManagerHolder() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "systemd", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
|
||||||
|
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}},
|
||||||
|
Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func packageManagerHolder() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "pacman", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.PackageManagerSeat, Scope: catalogue.ScopeNode}},
|
||||||
|
Resources: []map[string]any{{"id": "refresh", "type": "service", "unit": "pacman-refresh.timer"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func aDaemon() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "sshd", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "sshd", "type": "service", "unit": "sshd.service"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, serviceManagerHolder())
|
||||||
|
register(t, open, packageManagerHolder())
|
||||||
|
register(t, open, aDaemon())
|
||||||
|
|
||||||
|
_, err := assign(ctx, open, "laptop", "sshd")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("sshd went onto a machine nothing holds the service manager of")
|
||||||
|
}
|
||||||
|
for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q:\n%v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assigned, err := open.inventory.Assigned(ctx, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if contains(assigned, "sshd") {
|
||||||
|
t.Fatalf("a refused assignment was kept: %v", assigned)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The holders depend on each other, so neither goes on alone — and both go on in one act.
|
||||||
|
if _, err := assign(ctx, open, "laptop", "systemd"); err == nil {
|
||||||
|
t.Fatal("systemd went on alone though its package needs a package manager")
|
||||||
|
}
|
||||||
|
if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil {
|
||||||
|
t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said)
|
||||||
|
}
|
||||||
|
if said, err := assign(ctx, open, "laptop", "sshd"); err != nil {
|
||||||
|
t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) {
|
||||||
|
argv, err := argvFor("assign", map[string]any{"node": "laptop", "module": "systemd, pacman"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Join(argv, " ") != "assign laptop systemd pacman" {
|
||||||
|
t.Errorf("the seat's assign became %v", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, serviceManagerHolder())
|
||||||
|
register(t, open, packageManagerHolder())
|
||||||
|
register(t, open, aDaemon())
|
||||||
|
if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := unassign(ctx, open, "laptop", "systemd")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("the service manager came off a machine still running services")
|
||||||
|
}
|
||||||
|
for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not name %q:\n%v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assigned, _ := open.inventory.Assigned(ctx, "laptop")
|
||||||
|
if !contains(assigned, "systemd") {
|
||||||
|
t.Fatalf("a refused unassignment took the module off anyway: %v", assigned)
|
||||||
|
}
|
||||||
|
if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil {
|
||||||
|
t.Fatalf("a dependent could not come off: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) {
|
||||||
|
// The mesh as it ran before the switch (novox/hq ADR 0207 §4).
|
||||||
|
defer catalogue.EnforcingSeatDependencies(false)()
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, serviceManagerHolder())
|
||||||
|
register(t, open, aDaemon())
|
||||||
|
// Assigned straight into the store: a machine whose modules predate the rule, which is every
|
||||||
|
// machine on the day it ships.
|
||||||
|
if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, refused := asked.refused["laptop"]; refused {
|
||||||
|
t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"])
|
||||||
|
}
|
||||||
|
if asked.well() {
|
||||||
|
t.Error("a mesh with an unheld dependency reads as all well")
|
||||||
|
}
|
||||||
|
got := printed(t, func() error { return printStatus(asked) })
|
||||||
|
for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Errorf("status does not say %q:\n%s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Unheld []catalogue.Unheld `json:"unheld"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat {
|
||||||
|
t.Errorf("the document's unheld is %+v", doc.Unheld)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -29,11 +29,13 @@ type seatHolder struct {
|
|||||||
|
|
||||||
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
|
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
|
||||||
type seatRow struct {
|
type seatRow struct {
|
||||||
Seat string `json:"seat"`
|
Seat string `json:"seat"`
|
||||||
Scope string `json:"scope"`
|
Scope string `json:"scope"`
|
||||||
Delivers string `json:"delivers,omitempty"`
|
Delivers string `json:"delivers,omitempty"`
|
||||||
Decision string `json:"decision"`
|
Decision string `json:"decision"`
|
||||||
Holders []seatHolder `json:"holders"`
|
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||||
|
Replicated bool `json:"replicated,omitempty"`
|
||||||
|
Holders []seatHolder `json:"holders"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||||
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
|||||||
rows := make([]seatRow, 0, len(seats))
|
rows := make([]seatRow, 0, len(seats))
|
||||||
for _, s := range seats {
|
for _, s := range seats {
|
||||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||||
Holders: []seatHolder{}}
|
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||||
seen := map[seatHolder]bool{}
|
seen := map[seatHolder]bool{}
|
||||||
for _, h := range held {
|
for _, h := range held {
|
||||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||||
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if len(args) == 3 && args[1] == "--to" {
|
if len(args) == 3 && args[1] == "--to" {
|
||||||
return handOver(ctx, args[0], args[2])
|
return handOver(ctx, args[0], args[2], false)
|
||||||
}
|
}
|
||||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
|
if len(args) == 3 && args[1] == "--add" {
|
||||||
|
return handOver(ctx, args[0], args[2], true)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
|
||||||
|
"seat <name> --add <node>/<module>")
|
||||||
}
|
}
|
||||||
|
|
||||||
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
|
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
|
||||||
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
|
|||||||
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
|
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
|
||||||
// and refusing to record a handover to a module the node has not started would make the handover
|
// and refusing to record a handover to a module the node has not started would make the handover
|
||||||
// impossible to do before the switch instead of as the switch.
|
// impossible to do before the switch instead of as the switch.
|
||||||
func handOver(ctx context.Context, seatName, to string) error {
|
//
|
||||||
|
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
|
||||||
|
// records the named assignment as a further holder and leaves every holder on record as it is. A
|
||||||
|
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
|
||||||
|
// the one named, as it always did.
|
||||||
|
func handOver(ctx context.Context, seatName, to string, adding bool) error {
|
||||||
nodeName, module, ok := strings.Cut(to, "/")
|
nodeName, module, ok := strings.Cut(to, "/")
|
||||||
if !ok || nodeName == "" || module == "" {
|
if !ok || nodeName == "" || module == "" {
|
||||||
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
||||||
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
|
|||||||
if !known {
|
if !known {
|
||||||
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
|
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
|
||||||
}
|
}
|
||||||
|
if adding && !seat.Replicated {
|
||||||
|
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
|
||||||
|
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
|
||||||
|
}
|
||||||
assigned, err := inv.Assigned(ctx, nodeName)
|
assigned, err := inv.Assigned(ctx, nodeName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
|
|||||||
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
|
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
|
||||||
}
|
}
|
||||||
var was string
|
var was string
|
||||||
|
var held []string
|
||||||
holdings, err := inv.Holdings(ctx)
|
holdings, err := inv.Holdings(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
|
|||||||
for _, h := range holdings {
|
for _, h := range holdings {
|
||||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
|
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
|
||||||
was = h.Node
|
was = h.Node
|
||||||
|
held = append(held, h.Node)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
|
|||||||
} else if err := catalogue.CanHold(*m, seat); err != nil {
|
} else if err := catalogue.CanHold(*m, seat); err != nil {
|
||||||
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
|
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
|
||||||
}
|
}
|
||||||
|
if adding {
|
||||||
|
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
|
||||||
|
strings.Join(held, ", "))
|
||||||
|
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,9 +9,11 @@ import (
|
|||||||
"github.com/nats-io/nats.go/micro"
|
"github.com/nats-io/nats.go/micro"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -36,19 +38,194 @@ type verbAnswer struct {
|
|||||||
|
|
||||||
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
|
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
|
||||||
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
|
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
|
||||||
|
//
|
||||||
|
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
|
||||||
|
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
|
||||||
|
// the verb declares but did not use for the command line it composed — given beside another that
|
||||||
|
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
|
||||||
|
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
|
||||||
|
// not take that" would have stopped it before anything was sent.
|
||||||
func argvFor(verb string, args map[string]any) ([]string, error) {
|
func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||||
str := func(key string) string {
|
a, err := readArguments(verb, args)
|
||||||
v, _ := args[key].(string)
|
if err != nil {
|
||||||
return strings.TrimSpace(v)
|
return nil, err
|
||||||
}
|
}
|
||||||
need := func(keys ...string) error {
|
argv, err := a.commandLine()
|
||||||
for _, k := range keys {
|
if len(a.misread) > 0 {
|
||||||
if str(k) == "" {
|
// The table and the command line disagree: the verb reads an argument no caller can see
|
||||||
return fmt.Errorf("%s needs %q", verb, k)
|
// in its schema, so no caller could ever pass it.
|
||||||
|
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
|
||||||
|
"table and its command lines disagree", verb, quoteAll(a.misread))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if unused := a.unused(); len(unused) > 0 {
|
||||||
|
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
|
||||||
|
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
|
||||||
|
// command line was composed from.
|
||||||
|
type verbArguments struct {
|
||||||
|
verb string
|
||||||
|
given map[string]string
|
||||||
|
used map[string]bool
|
||||||
|
declared map[string]bool
|
||||||
|
misread []string // arguments the command line read that the schema does not declare: a bug here
|
||||||
|
}
|
||||||
|
|
||||||
|
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
|
||||||
|
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
|
||||||
|
// wrote.
|
||||||
|
func controllerVerb(name string) (catalogue.Verb, bool) {
|
||||||
|
for _, v := range catalogue.ControllerVerbs {
|
||||||
|
if v.Name == name {
|
||||||
|
return v, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return catalogue.Verb{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredArguments are a schema's properties, and which of them are switches.
|
||||||
|
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
|
||||||
|
switches = map[string]bool{}
|
||||||
|
props, _ := v.Input["properties"].(map[string]any)
|
||||||
|
for name, p := range props {
|
||||||
|
names = append(names, name)
|
||||||
|
desc, _ := p.(map[string]any)
|
||||||
|
switch enum := desc["enum"].(type) {
|
||||||
|
case []string:
|
||||||
|
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
||||||
|
case []any:
|
||||||
|
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names, switches
|
||||||
|
}
|
||||||
|
|
||||||
|
// readArguments refuses what the verb does not take, before anything is composed.
|
||||||
|
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
|
||||||
|
v, known := controllerVerb(verb)
|
||||||
|
if !known {
|
||||||
|
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
||||||
|
}
|
||||||
|
names, switches := declaredArguments(v)
|
||||||
|
declared := map[string]bool{}
|
||||||
|
for _, n := range names {
|
||||||
|
declared[n] = true
|
||||||
|
}
|
||||||
|
takes := "none"
|
||||||
|
if len(names) > 0 {
|
||||||
|
takes = quoteAll(names)
|
||||||
|
}
|
||||||
|
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
|
||||||
|
keys := make([]string, 0, len(args))
|
||||||
|
for k := range args {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
if !declared[k] {
|
||||||
|
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
|
||||||
|
}
|
||||||
|
var value string
|
||||||
|
switch x := args[k].(type) {
|
||||||
|
case nil:
|
||||||
|
continue
|
||||||
|
case string:
|
||||||
|
value = strings.TrimSpace(x)
|
||||||
|
case bool:
|
||||||
|
if !switches[k] {
|
||||||
|
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
|
||||||
|
}
|
||||||
|
value = fmt.Sprint(x)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
|
||||||
|
}
|
||||||
|
if switches[k] {
|
||||||
|
switch value {
|
||||||
|
case "true":
|
||||||
|
case "false", "":
|
||||||
|
continue // said and off: the same as not given, and nothing passed over
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
if value != "" {
|
||||||
|
a.given[k] = value
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// str is one argument's value, marked as used.
|
||||||
|
func (a *verbArguments) str(key string) string {
|
||||||
|
if !a.declared[key] {
|
||||||
|
a.misread = append(a.misread, key)
|
||||||
|
}
|
||||||
|
a.used[key] = true
|
||||||
|
return a.given[key]
|
||||||
|
}
|
||||||
|
|
||||||
|
// on is a switch, marked as used.
|
||||||
|
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
|
||||||
|
|
||||||
|
// need refuses a call missing a required argument, in the verb's own words.
|
||||||
|
func (a *verbArguments) need(keys ...string) error {
|
||||||
|
for _, k := range keys {
|
||||||
|
if a.str(k) == "" {
|
||||||
|
return fmt.Errorf("%s needs %q", a.verb, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// unused are the arguments given that the command line was not composed from.
|
||||||
|
func (a *verbArguments) unused() []string {
|
||||||
|
var out []string
|
||||||
|
for k := range a.given {
|
||||||
|
if !a.used[k] {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *verbArguments) usedGiven() []string {
|
||||||
|
var out []string
|
||||||
|
for k := range a.given {
|
||||||
|
if a.used[k] {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
if len(out) == 0 {
|
||||||
|
return []string{"nothing"}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func quoteAll(xs []string) string {
|
||||||
|
q := make([]string, len(xs))
|
||||||
|
for i, x := range xs {
|
||||||
|
if x == "nothing" {
|
||||||
|
q[i] = x
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
q[i] = fmt.Sprintf("%q", x)
|
||||||
|
}
|
||||||
|
return strings.Join(q, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
|
||||||
|
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
|
||||||
|
func (a *verbArguments) commandLine() ([]string, error) {
|
||||||
|
verb, str, on, need := a.verb, a.str, a.on, a.need
|
||||||
switch verb {
|
switch verb {
|
||||||
case "command":
|
case "command":
|
||||||
// The generic verb: the command line as given, split as a shell would split it, with
|
// The generic verb: the command line as given, split as a shell would split it, with
|
||||||
@@ -64,7 +241,15 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
|||||||
if len(argv) == 0 {
|
if len(argv) == 0 {
|
||||||
return nil, errors.New("command names no command")
|
return nil, errors.New("command names no command")
|
||||||
}
|
}
|
||||||
|
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||||
|
// it says why, as it would through its own verb.
|
||||||
|
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||||
|
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||||
|
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||||
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
|
case "tools":
|
||||||
|
return nil, errors.New("tools is answered from the records, not by a command")
|
||||||
case "status":
|
case "status":
|
||||||
return []string{"status", "--json"}, nil
|
return []string{"status", "--json"}, nil
|
||||||
case "nodes":
|
case "nodes":
|
||||||
@@ -82,10 +267,14 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
|||||||
if id := str("log"); id != "" {
|
if id := str("log"); id != "" {
|
||||||
return []string{"builds", "--log", id}, nil
|
return []string{"builds", "--log", id}, nil
|
||||||
}
|
}
|
||||||
if m := str("module"); m != "" {
|
argv := []string{"builds"}
|
||||||
return []string{"builds", m}, nil
|
if n := str("limit"); n != "" {
|
||||||
|
argv = append(argv, "-n", n)
|
||||||
}
|
}
|
||||||
return []string{"builds"}, nil
|
if m := str("module"); m != "" {
|
||||||
|
argv = append(argv, m)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
case "plans":
|
case "plans":
|
||||||
if r := str("repository"); r != "" {
|
if r := str("repository"); r != "" {
|
||||||
argv := []string{"plans", "--what-if", r}
|
argv := []string{"plans", "--what-if", r}
|
||||||
@@ -97,23 +286,80 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
|||||||
}
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
}
|
}
|
||||||
if id := str("stop"); id != "" {
|
for _, act := range []string{"stop", "close", "retry"} {
|
||||||
return []string{"plans", "stop", id}, nil
|
if id := str(act); id != "" {
|
||||||
|
argv := []string{"plans", act, id}
|
||||||
|
if act == "retry" {
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
|
||||||
|
if w := str("why"); w != "" {
|
||||||
|
argv = append(argv, "--why", w)
|
||||||
|
}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
argv = append(argv, "--cause", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if id := str("id"); id != "" {
|
if id := str("id"); id != "" {
|
||||||
return []string{"plans", id}, nil
|
return []string{"plans", id}, nil
|
||||||
}
|
}
|
||||||
return []string{"plans"}, nil
|
argv := []string{"plans"}
|
||||||
|
if n := str("limit"); n != "" {
|
||||||
|
argv = append(argv, "-n", n)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
// The build queue (novox/hq ADR 0219).
|
||||||
|
case "queue":
|
||||||
|
return []string{"queue"}, nil
|
||||||
|
case "cancel", "kill":
|
||||||
|
if err := need("id"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{verb, str("id")}, nil
|
||||||
|
case "clear":
|
||||||
|
if on("dead") {
|
||||||
|
return []string{"clear", "--dead"}, nil
|
||||||
|
}
|
||||||
|
return []string{"clear"}, nil
|
||||||
|
case "rebuild":
|
||||||
|
if err := need("what"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"rebuild", str("what")}, nil
|
||||||
|
case "replay":
|
||||||
|
if err := need("id"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"replay", str("id")}
|
||||||
|
if on("register") {
|
||||||
|
argv = append(argv, "--register")
|
||||||
|
}
|
||||||
|
if on("older") {
|
||||||
|
argv = append(argv, "--older")
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "pause", "resume":
|
||||||
|
if n := str("node"); n != "" {
|
||||||
|
return []string{verb, n}, nil
|
||||||
|
}
|
||||||
|
return []string{verb}, nil
|
||||||
case "plan":
|
case "plan":
|
||||||
if err := need("node"); err != nil {
|
if err := need("node"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if on("files") {
|
||||||
|
return []string{"plan", str("node"), "--files"}, nil
|
||||||
|
}
|
||||||
return []string{"plan", str("node"), "--json"}, nil
|
return []string{"plan", str("node"), "--json"}, nil
|
||||||
case "assign", "unassign":
|
case "assign", "unassign":
|
||||||
if err := need("node", "module"); err != nil {
|
if err := need("node", "module"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return []string{verb, str("node"), str("module")}, nil
|
// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
|
||||||
|
// the seats that apply resources depend on each other and go on together.
|
||||||
|
return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
|
||||||
case "pin":
|
case "pin":
|
||||||
if err := need("node", "provision", "from", "module"); err != nil {
|
if err := need("node", "provision", "from", "module"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -128,23 +374,118 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
|||||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||||
if n := str("node"); n != "" {
|
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
||||||
return []string{"push", n, "--wait", "0"}, nil
|
if err := need("why"); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
||||||
}
|
}
|
||||||
return []string{"push", "--behind", "--wait", "0"}, nil
|
why := []string{"--why", str("why")}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
why = append(why, "--cause", c)
|
||||||
|
}
|
||||||
|
if n := str("node"); n != "" {
|
||||||
|
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||||
|
// a machine and asking for every machine behind are two requests, and guessing one
|
||||||
|
// would push a machine nobody named, or not push one somebody did.
|
||||||
|
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
||||||
|
}
|
||||||
|
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||||
|
// first, so a caller who meant one machine reads that it was not one.
|
||||||
|
on("behind")
|
||||||
|
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
||||||
|
case "hand-act":
|
||||||
|
if err := need("what", "why", "cause"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
|
||||||
|
if c := str("condition"); c != "" {
|
||||||
|
argv = append(argv, "--condition", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "hand-acts":
|
||||||
|
argv := []string{"hand-acts", "--json"}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "durations":
|
||||||
|
argv := []string{"durations", "--json"}
|
||||||
|
if k := str("kind"); k != "" {
|
||||||
|
argv = append(argv, "--kind", k)
|
||||||
|
}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "conditions":
|
||||||
|
// One verb, four shapes, as `plans` (novox/hq to-be 45 §2): a silence, one condition, the
|
||||||
|
// history, or the open ones filtered.
|
||||||
|
if key := str("silence"); key != "" {
|
||||||
|
if err := need("for", "why"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"conditions", "silence", key, "--for", str("for"), "--why", str("why")}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
argv = append(argv, "--cause", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if on("history") {
|
||||||
|
argv := []string{"conditions", "history", "--json"}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
if k := str("key"); k != "" {
|
||||||
|
argv = append(argv, "--key", k)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if k := str("key"); k != "" {
|
||||||
|
return []string{"conditions", "show", k, "--json"}, nil
|
||||||
|
}
|
||||||
|
argv := []string{"conditions", "--json"}
|
||||||
|
for _, filter := range []string{"scope", "severity", "machine"} {
|
||||||
|
if v := str(filter); v != "" {
|
||||||
|
argv = append(argv, "--"+filter, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "doctor":
|
||||||
|
which := 0
|
||||||
|
argv := []string{"doctor"}
|
||||||
|
for _, sub := range []string{"run", "probes", "signals"} {
|
||||||
|
if on(sub) {
|
||||||
|
which++
|
||||||
|
argv = append(argv, sub)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if which > 1 {
|
||||||
|
return nil, errors.New("doctor answers one of run, probes or signals at a time")
|
||||||
|
}
|
||||||
|
return append(argv, "--json"), nil
|
||||||
case "rotate":
|
case "rotate":
|
||||||
if p := str("provision"); p != "" {
|
if p := str("provision"); p != "" {
|
||||||
argv := []string{"rotate", p}
|
argv := []string{"rotate", p}
|
||||||
if c := str("consumer"); c != "" {
|
if c := str("consumer"); c != "" {
|
||||||
argv = append(argv, "--consumer", c)
|
argv = append(argv, "--consumer", c)
|
||||||
}
|
}
|
||||||
|
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
|
||||||
|
// and secret stay the other shape's, and are refused as passed over.
|
||||||
|
if m := str("module"); m != "" {
|
||||||
|
argv = append(argv, "--module", m)
|
||||||
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
}
|
}
|
||||||
if str("node") != "" && str("module") != "" && str("secret") != "" {
|
_, node := a.given["node"]
|
||||||
|
_, module := a.given["module"]
|
||||||
|
_, secret := a.given["secret"]
|
||||||
|
if node || module || secret {
|
||||||
|
if err := need("node", "module", "secret"); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
|
||||||
|
}
|
||||||
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
||||||
}
|
}
|
||||||
// Half of either shape: the command says its usage, which names both shapes, and that is
|
// Neither shape: the command says its usage, which names both, and that is the answer the
|
||||||
// the answer the caller needs.
|
// caller needs.
|
||||||
return []string{"rotate"}, nil
|
return []string{"rotate"}, nil
|
||||||
case "settings":
|
case "settings":
|
||||||
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
||||||
@@ -152,15 +493,16 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
|||||||
if err := need("module"); err != nil {
|
if err := need("module"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
argv := []string{"settings", "set", str("module")}
|
var argv []string
|
||||||
switch {
|
if on("clear") {
|
||||||
case str("clear") == "true":
|
|
||||||
argv = []string{"settings", "clear", str("module")}
|
argv = []string{"settings", "clear", str("module")}
|
||||||
case str("values") != "":
|
} else {
|
||||||
argv = append(argv, str("values"))
|
argv = []string{"settings", "set", str("module")}
|
||||||
|
// Neither values nor clear: the command says its usage, which names both.
|
||||||
|
if v := str("values"); v != "" {
|
||||||
|
argv = append(argv, v)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Neither values nor clear: the command says its usage, which names both, and that is the
|
|
||||||
// answer the caller needs — the same as `rotate` given half of either shape.
|
|
||||||
if n := str("node"); n != "" {
|
if n := str("node"); n != "" {
|
||||||
argv = append(argv, "--node", n)
|
argv = append(argv, "--node", n)
|
||||||
}
|
}
|
||||||
@@ -192,11 +534,35 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
|||||||
}
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
|
||||||
|
verb, catalogue.ControllerSeatName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true}
|
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||||
|
"hand-acts": true, "durations": true, "conditions": true, "doctor": true}
|
||||||
|
|
||||||
|
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||||
|
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||||
|
func repairingCommand(argv []string) string {
|
||||||
|
switch {
|
||||||
|
case argv[0] == "push":
|
||||||
|
return "push"
|
||||||
|
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close"):
|
||||||
|
return "plans " + argv[1]
|
||||||
|
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||||
|
return "broker consumer-reset"
|
||||||
|
case argv[0] == "hand-act":
|
||||||
|
return "hand-act record"
|
||||||
|
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
||||||
|
return "conditions silence"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func isWhyFlag(word string) bool {
|
||||||
|
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
||||||
|
}
|
||||||
|
|
||||||
// runVerb runs this binary with the given command line and gathers what it said.
|
// runVerb runs this binary with the given command line and gathers what it said.
|
||||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||||
@@ -208,6 +574,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
|||||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||||
// from a shell in this container would have, because it is that.
|
// from a shell in this container would have, because it is that.
|
||||||
cmd.Env = os.Environ()
|
cmd.Env = os.Environ()
|
||||||
|
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||||
|
caller := link.CallerIn(ctx)
|
||||||
|
if caller == "" {
|
||||||
|
caller = "a seat call whose caller the bus did not name"
|
||||||
|
}
|
||||||
|
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||||
@@ -244,8 +616,35 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
handlers := map[string]link.ToolHandler{}
|
handlers := map[string]link.ToolHandler{}
|
||||||
for _, v := range seat.Serves {
|
for _, v := range seat.Serves {
|
||||||
verb := v.Name
|
verb := v.Name
|
||||||
if verb == "tools" {
|
if inProcess[verb] {
|
||||||
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
|
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
||||||
|
args := map[string]any{}
|
||||||
|
if len(bytes.TrimSpace(raw)) > 0 {
|
||||||
|
if err := json.Unmarshal(raw, &args); err != nil {
|
||||||
|
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Refused like any verb's: what a verb does not take is not ignored.
|
||||||
|
a, err := readArguments(verb, args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if verb == "calls" {
|
||||||
|
return callsAnswer(link.Calls, a.given["call"])
|
||||||
|
}
|
||||||
|
if verb == "doctor" {
|
||||||
|
// From the serving controller, which runs the self-check and hears the signals
|
||||||
|
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
|
||||||
|
argv, err := a.commandLine()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sub := ""
|
||||||
|
if len(argv) > 2 {
|
||||||
|
sub = argv[1]
|
||||||
|
}
|
||||||
|
return doctorAnswer(ctx, sub)
|
||||||
|
}
|
||||||
return seatTools(), nil
|
return seatTools(), nil
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -284,12 +683,85 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if verb == "status" && statusFrom != nil {
|
||||||
|
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||||
|
return statusFrom.answer(ctx)
|
||||||
|
}
|
||||||
|
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||||
|
// Whatever it did, `status` is composed again once it has.
|
||||||
|
defer statusFrom.nudge()
|
||||||
|
}
|
||||||
|
if answersFirst(argv) {
|
||||||
|
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||||
|
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||||
|
link.Acknowledge(ctx)
|
||||||
|
}
|
||||||
return runVerb(ctx, argv)
|
return runVerb(ctx, argv)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return handlers, behind, nil
|
return handlers, behind, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||||
|
func actsOnAPlan(args map[string]any) bool {
|
||||||
|
for _, act := range []string{"stop", "close", "retry"} {
|
||||||
|
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||||
|
// the records, `calls` from what this process served.
|
||||||
|
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
|
||||||
|
|
||||||
|
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||||
|
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||||
|
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
|
||||||
|
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
|
||||||
|
func answersFirst(argv []string) bool {
|
||||||
|
return len(argv) > 0 && argv[0] == "push"
|
||||||
|
}
|
||||||
|
|
||||||
|
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
||||||
|
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
|
||||||
|
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
|
||||||
|
// the reason said beside it.
|
||||||
|
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||||
|
if id != "" {
|
||||||
|
c, ok, err := log.Get(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
||||||
|
"bus could not be read: %w", id, err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
if log.IsDurable() {
|
||||||
|
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
|
||||||
|
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
||||||
|
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
recent, err := log.Recent()
|
||||||
|
for i := range recent {
|
||||||
|
recent[i].Answer = nil
|
||||||
|
}
|
||||||
|
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
|
||||||
|
if log.IsDurable() {
|
||||||
|
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
|
||||||
|
broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||||
|
} else {
|
||||||
|
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
answer["unread"] = err.Error()
|
||||||
|
}
|
||||||
|
return answer, nil
|
||||||
|
}
|
||||||
|
|
||||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||||
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
|
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
|
||||||
func seatTools() map[string]any {
|
func seatTools() map[string]any {
|
||||||
@@ -310,9 +782,10 @@ func seatTools() map[string]any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
|
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
|
||||||
// verb runnable rather than that it happens to want the arguments the check guessed.
|
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
|
||||||
|
// more, since an argument a verb does not declare is refused.
|
||||||
func sampleArguments(v catalogue.Verb) map[string]any {
|
func sampleArguments(v catalogue.Verb) map[string]any {
|
||||||
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
|
sample := map[string]any{}
|
||||||
switch required := v.Input["required"].(type) {
|
switch required := v.Input["required"].(type) {
|
||||||
case []string:
|
case []string:
|
||||||
for _, k := range required {
|
for _, k := range required {
|
||||||
|
|||||||
@@ -0,0 +1,373 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"go/ast"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The schemas the controller serves, verb by verb, as the console receives them: from the
|
||||||
|
// announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244).
|
||||||
|
func servedSchemas(t *testing.T) map[string]catalogue.Verb {
|
||||||
|
t.Helper()
|
||||||
|
handlers, behind, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(behind) != 0 {
|
||||||
|
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
|
||||||
|
}
|
||||||
|
served := map[string]catalogue.Verb{}
|
||||||
|
for _, e := range seatAnnouncement(handlers).Endpoints {
|
||||||
|
var input map[string]any
|
||||||
|
if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil {
|
||||||
|
t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err)
|
||||||
|
}
|
||||||
|
served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input}
|
||||||
|
}
|
||||||
|
if len(served) != len(catalogue.ControllerVerbs) {
|
||||||
|
t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs))
|
||||||
|
}
|
||||||
|
return served
|
||||||
|
}
|
||||||
|
|
||||||
|
// subsetsOf is every subset of the names, the empty one included.
|
||||||
|
func subsetsOf(names []string) [][]string {
|
||||||
|
var out [][]string
|
||||||
|
for mask := 0; mask < 1<<len(names); mask++ {
|
||||||
|
var s []string
|
||||||
|
for i, n := range names {
|
||||||
|
if mask&(1<<i) != 0 {
|
||||||
|
s = append(s, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func argumentsFor(subset []string, switches map[string]bool) map[string]any {
|
||||||
|
args := map[string]any{}
|
||||||
|
for _, n := range subset {
|
||||||
|
if switches[n] {
|
||||||
|
args[n] = "true"
|
||||||
|
} else {
|
||||||
|
args[n] = "x-" + n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
|
// saidOutright are the switches that say the default aloud, so the line is the same without them —
|
||||||
|
// on purpose, and only these.
|
||||||
|
var saidOutright = map[string]string{
|
||||||
|
"push": "behind", // the whole mesh is what no machine means; behind lets a caller say they meant it
|
||||||
|
}
|
||||||
|
|
||||||
|
// **No argument a caller gives is passed over** (novox/hq issue 244). For every verb served and
|
||||||
|
// every combination of the arguments its schema declares, the verb either refuses the call, or
|
||||||
|
// composes a command line that each given argument changed: taking any one away changes the line
|
||||||
|
// or makes it refused. An argument the line is the same without is one the verb ignored — the
|
||||||
|
// shape of the push that named a machine and pushed every machine behind.
|
||||||
|
func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
|
||||||
|
for name, v := range servedSchemas(t) {
|
||||||
|
if inProcess[name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
names, switches := declaredArguments(v)
|
||||||
|
for _, subset := range subsetsOf(names) {
|
||||||
|
args := argumentsFor(subset, switches)
|
||||||
|
argv, err := argvFor(name, args)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(err.Error(), "does not declare") {
|
||||||
|
t.Errorf("%s %v: %v", name, args, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, dropped := range subset {
|
||||||
|
fewer := map[string]any{}
|
||||||
|
for k, val := range args {
|
||||||
|
if k != dropped {
|
||||||
|
fewer[k] = val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
without, err := argvFor(name, fewer)
|
||||||
|
if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped {
|
||||||
|
t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with
|
||||||
|
// what it requires and one argument more; and `node` in particular, for every verb whose schema
|
||||||
|
// does not take a machine.
|
||||||
|
func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
||||||
|
for name, v := range servedSchemas(t) {
|
||||||
|
names, _ := declaredArguments(v)
|
||||||
|
strangers := []string{"no-such-argument"}
|
||||||
|
if !contains(names, "node") {
|
||||||
|
strangers = append(strangers, "node")
|
||||||
|
}
|
||||||
|
for _, stranger := range strangers {
|
||||||
|
args := sampleArguments(v)
|
||||||
|
args[stranger] = "x"
|
||||||
|
_, err := argvFor(name, args)
|
||||||
|
if inProcess[name] {
|
||||||
|
_, err = readArguments(name, args)
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
|
||||||
|
t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil {
|
||||||
|
t.Error("a switch took a word that is neither true nor false, and would have read it as false")
|
||||||
|
}
|
||||||
|
if _, err := argvFor("node", map[string]any{"node": 7}); err == nil {
|
||||||
|
t.Error("a number was taken as a machine's name, or as no machine")
|
||||||
|
}
|
||||||
|
if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" {
|
||||||
|
t.Errorf("a switch given as JSON true: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
||||||
|
// naming one beside behind is refused rather than one of the two guessed.
|
||||||
|
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||||
|
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
|
||||||
|
t.Fatalf("a named push: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
||||||
|
argv, err := argvFor("push", args)
|
||||||
|
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
|
||||||
|
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), `"behind"`) {
|
||||||
|
t.Fatalf("a named push with behind was taken: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) {
|
||||||
|
t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// commandFlags is every flag set this package's commands parse, by the name the set is made with,
|
||||||
|
// and the flags defined on it — read from the source, so a flag added to a command is seen here
|
||||||
|
// without anyone remembering to.
|
||||||
|
func commandFlags(t *testing.T) map[string][]string {
|
||||||
|
t.Helper()
|
||||||
|
files, err := filepath.Glob("*.go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fset := token.NewFileSet()
|
||||||
|
out := map[string][]string{}
|
||||||
|
for _, f := range files {
|
||||||
|
if strings.HasSuffix(f, "_test.go") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
file, err := parser.ParseFile(fset, f, nil, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, decl := range file.Decls {
|
||||||
|
fn, ok := decl.(*ast.FuncDecl)
|
||||||
|
if !ok || fn.Body == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sets := map[string]string{} // variable → the set's name
|
||||||
|
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
||||||
|
if assign, ok := n.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 && len(assign.Rhs) == 1 {
|
||||||
|
if call, ok := assign.Rhs[0].(*ast.CallExpr); ok && isSelector(call.Fun, "flag", "NewFlagSet") {
|
||||||
|
if id, ok := assign.Lhs[0].(*ast.Ident); ok {
|
||||||
|
if name, ok := stringLit(call.Args[0]); ok {
|
||||||
|
sets[id.Name] = name
|
||||||
|
out[name] = append(out[name], []string{}...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
call, ok := n.(*ast.CallExpr)
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
recv, ok := sel.X.(*ast.Ident)
|
||||||
|
if !ok || sets[recv.Name] == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
arg := 0
|
||||||
|
switch sel.Sel.Name {
|
||||||
|
case "Bool", "String", "Int", "Int64", "Uint", "Uint64", "Float64", "Duration", "Func", "BoolFunc", "TextVar":
|
||||||
|
case "BoolVar", "StringVar", "IntVar", "Int64Var", "UintVar", "Uint64Var", "Float64Var", "DurationVar", "Var":
|
||||||
|
arg = 1
|
||||||
|
default:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if arg < len(call.Args) {
|
||||||
|
if flagName, ok := stringLit(call.Args[arg]); ok {
|
||||||
|
out[sets[recv.Name]] = append(out[sets[recv.Name]], flagName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func isSelector(e ast.Expr, pkg, name string) bool {
|
||||||
|
sel, ok := e.(*ast.SelectorExpr)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
id, ok := sel.X.(*ast.Ident)
|
||||||
|
return ok && id.Name == pkg && sel.Sel.Name == name
|
||||||
|
}
|
||||||
|
|
||||||
|
func stringLit(e ast.Expr) (string, bool) {
|
||||||
|
lit, ok := e.(*ast.BasicLit)
|
||||||
|
if !ok || lit.Kind != token.STRING {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
s, err := strconv.Unquote(lit.Value)
|
||||||
|
return s, err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// accountedFlags are the flags of a verb's command that are not an argument of the same name:
|
||||||
|
// carried by an argument named otherwise ("=argument"), or set by the verb itself, or withheld from
|
||||||
|
// the named verb on purpose — each with why. `command` reaches every flag of the binary regardless.
|
||||||
|
var accountedFlags = map[string]map[string]string{
|
||||||
|
"status": {"json": "set by the verb: the answer is data"},
|
||||||
|
"seats": {"json": "set by the verb: the answer is data"},
|
||||||
|
"queue": {"json": "not set: the verb answers the table a person reads"},
|
||||||
|
"plan": {"json": "set by the verb unless files is asked"},
|
||||||
|
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
|
||||||
|
"build": {
|
||||||
|
"wait": "set by the verb to 0: the id follows the build (issue 176)",
|
||||||
|
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
|
||||||
|
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
|
||||||
|
"behind": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||||
|
"on": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||||
|
},
|
||||||
|
"builds": {"n": "=limit"},
|
||||||
|
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||||
|
"durations": {
|
||||||
|
"json": "set by the verb: the answer is data",
|
||||||
|
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||||
|
},
|
||||||
|
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
||||||
|
// from the source, so a flag added to a command — or a verb added whose command takes flags —
|
||||||
|
// fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And
|
||||||
|
// a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody
|
||||||
|
// reads.
|
||||||
|
func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
|
||||||
|
flags := commandFlags(t)
|
||||||
|
if len(flags["push"]) == 0 || len(flags["plan"]) == 0 {
|
||||||
|
t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags)
|
||||||
|
}
|
||||||
|
reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
|
||||||
|
served := servedSchemas(t)
|
||||||
|
for name, v := range served {
|
||||||
|
if inProcess[name] || name == "command" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
names, switches := declaredArguments(v)
|
||||||
|
for _, subset := range subsetsOf(names) {
|
||||||
|
argv, err := argvFor(name, argumentsFor(subset, switches))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// The flag set is named by the longest run of leading words that names one.
|
||||||
|
var words []string
|
||||||
|
for _, w := range argv {
|
||||||
|
if strings.HasPrefix(w, "-") {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
words = append(words, w)
|
||||||
|
}
|
||||||
|
for n := len(words); n > 0; n-- {
|
||||||
|
if _, has := flags[strings.Join(words[:n], " ")]; has {
|
||||||
|
if reached[name] == nil {
|
||||||
|
reached[name] = map[string]bool{}
|
||||||
|
}
|
||||||
|
reached[name][strings.Join(words[:n], " ")] = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
used := map[string]map[string]bool{}
|
||||||
|
verbs := make([]string, 0, len(reached))
|
||||||
|
for verb := range reached {
|
||||||
|
verbs = append(verbs, verb)
|
||||||
|
}
|
||||||
|
sort.Strings(verbs)
|
||||||
|
for _, verb := range verbs {
|
||||||
|
declared, _ := declaredArguments(served[verb])
|
||||||
|
for set := range reached[verb] {
|
||||||
|
if used[set] == nil {
|
||||||
|
used[set] = map[string]bool{}
|
||||||
|
}
|
||||||
|
for _, flagName := range flags[set] {
|
||||||
|
why, accounted := accountedFlags[set][flagName]
|
||||||
|
switch {
|
||||||
|
case accounted && strings.HasPrefix(why, "="):
|
||||||
|
used[set][flagName] = true
|
||||||
|
if !contains(declared, strings.TrimPrefix(why, "=")) {
|
||||||
|
t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare",
|
||||||
|
verb, flagName, set, strings.TrimPrefix(why, "="))
|
||||||
|
}
|
||||||
|
case accounted:
|
||||||
|
used[set][flagName] = true
|
||||||
|
case contains(declared, flagName):
|
||||||
|
default:
|
||||||
|
t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+
|
||||||
|
"or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for set, fs := range accountedFlags {
|
||||||
|
for flagName := range fs {
|
||||||
|
if !used[set][flagName] {
|
||||||
|
t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every verb's required arguments are properties of its schema: a schema that requires what it
|
||||||
|
// does not describe is the uncallable verb of issue 244 from the other side.
|
||||||
|
func TestEveryRequiredArgumentIsDescribed(t *testing.T) {
|
||||||
|
for name, v := range servedSchemas(t) {
|
||||||
|
names, _ := declaredArguments(v)
|
||||||
|
for k := range sampleArguments(v) {
|
||||||
|
if !contains(names, k) {
|
||||||
|
t.Errorf("%s requires %q and does not describe it", name, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,29 +10,6 @@ import (
|
|||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
|
|
||||||
// schema names and no other (novox/hq ADR 0154, ADR 0035).
|
|
||||||
func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
|
|
||||||
for _, v := range catalogue.ControllerVerbs {
|
|
||||||
if v.Name == "tools" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
args := map[string]any{}
|
|
||||||
props, _ := v.Input["properties"].(map[string]any)
|
|
||||||
for name := range props {
|
|
||||||
args[name] = "x"
|
|
||||||
}
|
|
||||||
argv, err := argvFor(v.Name, args)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("%s: %v", v.Name, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if argv[0] == "" {
|
|
||||||
t.Errorf("%s: empty command", v.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// `builds` given a build's id reads that build's log from the bus rather than listing builds
|
// `builds` given a build's id reads that build's log from the bus rather than listing builds
|
||||||
// (novox/hq ADR 0157).
|
// (novox/hq ADR 0157).
|
||||||
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
|
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
|
||||||
@@ -66,13 +43,22 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
|||||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
||||||
t.Fatalf("a pair credential: %v", argv)
|
t.Fatalf("a pair credential: %v", argv)
|
||||||
}
|
}
|
||||||
|
argv, _ = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "module": "letta"})
|
||||||
|
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace --module letta" {
|
||||||
|
t.Fatalf("one consuming module's pair credential: %v", argv)
|
||||||
|
}
|
||||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
||||||
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
||||||
t.Fatalf("an own secret: %v", argv)
|
t.Fatalf("an own secret: %v", argv)
|
||||||
}
|
}
|
||||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
|
if _, err := argvFor("rotate", map[string]any{"node": "ace"}); err == nil || !strings.Contains(err.Error(), `"module"`) {
|
||||||
if strings.Join(argv, " ") != "rotate" {
|
t.Fatalf("half an own secret is refused, naming what it lacks: %v", err)
|
||||||
t.Fatalf("half an own secret falls to the command's usage: %v", argv)
|
}
|
||||||
|
if argv, _ := argvFor("rotate", nil); strings.Join(argv, " ") != "rotate" {
|
||||||
|
t.Fatalf("neither shape falls to the command's usage: %v", argv)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("rotate", map[string]any{"provision": "p", "node": "ace", "module": "m", "secret": "s"}); err == nil {
|
||||||
|
t.Fatal("both shapes at once were taken, and one of them passed over")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,8 +107,8 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
|||||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||||
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||||
if strings.Join(argv, " ") != "push one --wait 0" {
|
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
|
||||||
t.Fatalf("push waits: %v", argv)
|
t.Fatalf("push waits: %v", argv)
|
||||||
}
|
}
|
||||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||||
|
|||||||
@@ -31,6 +31,10 @@ type sendable struct {
|
|||||||
// the same composition as its received files, and every machine's private-network address.
|
// the same composition as its received files, and every machine's private-network address.
|
||||||
Received map[string]map[string][]catalogue.Contribution
|
Received map[string]map[string][]catalogue.Contribution
|
||||||
Mesh []string
|
Mesh []string
|
||||||
|
// BusUsers is the bus's user list this declaration carries, empty for every machine but the one
|
||||||
|
// holding the bus; not sent apart from the file it is in. Its digest is recorded once sent, so
|
||||||
|
// whether that machine must go first is read from the list alone (novox/hq issue 249).
|
||||||
|
BusUsers string
|
||||||
// LeftOut is every module of the machine's set left out of this declaration because a stored
|
// LeftOut is every module of the machine's set left out of this declaration because a stored
|
||||||
// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
|
// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
|
||||||
// keeps that module's held things and touches none of its containers; a machine is told
|
// keeps that module's held things and touches none of its containers; a machine is told
|
||||||
@@ -39,6 +43,13 @@ type sendable struct {
|
|||||||
LeftOut []string
|
LeftOut []string
|
||||||
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||||
leftOutWhy map[string]string
|
leftOutWhy map[string]string
|
||||||
|
// withheld is every consumer this machine's grants leave out, because its identity overflows the
|
||||||
|
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
|
||||||
|
withheld []catalogue.Overflow
|
||||||
|
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||||
|
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||||
|
// Composed only on the send path; nil records that it is not known.
|
||||||
|
Builds map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
|
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
|
||||||
|
|||||||
@@ -0,0 +1,485 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The signals table (novox/hq to-be 45 §3, ADR 0227 rule 5), compiled in.
|
||||||
|
//
|
||||||
|
// **Every signal the core expects has a watchdog; its absence is a condition.** A row says what is
|
||||||
|
// expected, from whom, after what, within what bound, and what is raised when it does not come. One
|
||||||
|
// table, read three ways: by the watchdog loop (watchdogs.go), which runs every row's watch over the
|
||||||
|
// facts it gathered; by `doctor signals`, which says the age of every row's newest signal; and by the
|
||||||
|
// test generated from it (signals_test.go), which suppresses each signal in turn and asserts its
|
||||||
|
// condition — so a row added without a watch, or a watch that does not fire, fails the build.
|
||||||
|
//
|
||||||
|
// A row not watched yet says why, and in which phase it will be: a watchdog of a signal nothing emits
|
||||||
|
// would be a condition that can only cry wolf. Bounds marked provisional are set from what Phase 0
|
||||||
|
// measured (`durations`) and corrected in Phase 1's first live week; a corrected bound is a change to
|
||||||
|
// this table, reviewed like code.
|
||||||
|
|
||||||
|
// signalRow is one row of the table.
|
||||||
|
type signalRow struct {
|
||||||
|
Row string
|
||||||
|
Signal string
|
||||||
|
Emitter string
|
||||||
|
Trigger string
|
||||||
|
Bound string
|
||||||
|
Kind string
|
||||||
|
Severity conditions.Severity
|
||||||
|
// Phase is the phase of to-be 45 its watchdog is built in.
|
||||||
|
Phase int
|
||||||
|
// Deferred says why it is not watched yet; empty for a row that is.
|
||||||
|
Deferred string
|
||||||
|
// needs is the part of the facts the row reads: an error there is the row blind, which is
|
||||||
|
// itself said (probe-failed) rather than read as nothing wrong.
|
||||||
|
needs func(f *signalFacts) error
|
||||||
|
// watch is what is wrong now, from the facts.
|
||||||
|
watch func(f *signalFacts) []conditions.Observation
|
||||||
|
// newest is the time of the newest signal of this row, for `doctor signals`; zero when none.
|
||||||
|
newest func(f *signalFacts) time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bounds, provisional where to-be 45 says so.
|
||||||
|
const (
|
||||||
|
// heartbeatEvery is the interval a node-engine or node tools that say none have always used.
|
||||||
|
heartbeatEvery = 60 * time.Second
|
||||||
|
// heartbeatsMissed is how many intervals may pass in silence (S1, S11).
|
||||||
|
heartbeatsMissed = 3
|
||||||
|
// controlNodeUrgentAfter is how long the control node may be silent before it is urgent (S1).
|
||||||
|
controlNodeUrgentAfter = 30 * time.Minute
|
||||||
|
// reportAtLeast is the least a machine is given to report a send (S2).
|
||||||
|
reportAtLeast = 2 * time.Minute
|
||||||
|
// tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at.
|
||||||
|
tierAtLeast = planWaitBound
|
||||||
|
// loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4).
|
||||||
|
loopDeafAfter = 2 * time.Minute
|
||||||
|
// askAtLeast is the least a build ask is given, and askDefault the bound while nothing is
|
||||||
|
// measured (S6): the build seat declares no timeout of its own.
|
||||||
|
askAtLeast = 20 * time.Minute
|
||||||
|
askDefault = time.Hour
|
||||||
|
// callDefault is the bound of a verb that declares none (S7); push's and build's are longer.
|
||||||
|
callDefault = 10 * time.Minute
|
||||||
|
// advisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9).
|
||||||
|
advisoryQuiet = time.Hour
|
||||||
|
// staleRefusalsAllowed in staleRefusalsWithin are what S13 lets pass from one writer.
|
||||||
|
staleRefusalsAllowed = 5
|
||||||
|
staleRefusalsWithin = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// callBounds are the verbs that may run longer than callDefault, and how long (S7).
|
||||||
|
var callBounds = map[string]time.Duration{
|
||||||
|
"push": 30 * time.Minute, "rotate": 30 * time.Minute, "assign": 15 * time.Minute,
|
||||||
|
"unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute,
|
||||||
|
}
|
||||||
|
|
||||||
|
// callBound is a verb's bound.
|
||||||
|
func callBound(verb string) time.Duration {
|
||||||
|
if b, ok := callBounds[verb]; ok {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
return callDefault
|
||||||
|
}
|
||||||
|
|
||||||
|
// signalsTable is the table, in to-be 45's order.
|
||||||
|
var signalsTable = []signalRow{
|
||||||
|
{Row: "S1", Signal: "machine heartbeat", Emitter: "node-engine", Trigger: "its interval",
|
||||||
|
Bound: "3 × the interval it says (60 s when it says none), provisional; not raised while the machine " +
|
||||||
|
"said it is asleep or shutting down (ADR 0211); urgent after 30 min for a control node",
|
||||||
|
Kind: "silent", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchHeartbeats,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard })
|
||||||
|
}},
|
||||||
|
{Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent",
|
||||||
|
Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep",
|
||||||
|
Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt })
|
||||||
|
}},
|
||||||
|
{Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered",
|
||||||
|
Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional; not while the " +
|
||||||
|
"build seat is paused under it",
|
||||||
|
Kind: "stalled", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.plans, func(p planFacts) time.Time { return p.entered })
|
||||||
|
}},
|
||||||
|
{Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller",
|
||||||
|
Trigger: "while its consumers have pending messages", Bound: "2 min",
|
||||||
|
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.loop.took }},
|
||||||
|
{Row: "S5", Signal: "a merge announced becomes a plan, or nothing reads it", Emitter: "announcer → controller",
|
||||||
|
Trigger: "each merge", Bound: "10 min (the catch-up pass of issue 266)",
|
||||||
|
Kind: "merge-not-acted", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.mergesErr }, watch: watchMerges,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.mergesPassed }},
|
||||||
|
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
|
||||||
|
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
|
||||||
|
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
|
||||||
|
Kind: "ask-lost", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.asksErr }, watch: watchAsks,
|
||||||
|
newest: func(f *signalFacts) time.Time { return newestOf(f.asks, func(a askFacts) time.Time { return a.since }) }},
|
||||||
|
{Row: "S7", Signal: "a call running → finished", Emitter: "controller", Trigger: "each call",
|
||||||
|
Bound: "the verb's bound: push, rotate and command 30 min, assign and unassign 15 min, doctor 3 min, " +
|
||||||
|
"any other 10 min",
|
||||||
|
Kind: "call-hung", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchCalls,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.calls, func(c link.Call) time.Time { return c.Started })
|
||||||
|
}},
|
||||||
|
{Row: "S8", Signal: "a provider's failing word repeated", Emitter: "provider",
|
||||||
|
Trigger: "every 15 min while failing (ADR 0224)", Bound: "30 min",
|
||||||
|
Kind: kindProviderSilent, Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.standingsErr }, watch: watchProviders,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
||||||
|
}},
|
||||||
|
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||||
|
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
||||||
|
Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
|
||||||
|
"once it exists again or the mesh no longer expects it",
|
||||||
|
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
||||||
|
}},
|
||||||
|
{Row: "S10", Signal: "the self-check's heartbeat", Emitter: "controller's doctor", Trigger: "every run",
|
||||||
|
Bound: "2 × its interval; watched from a second machine by mesh-watcher, and here as well",
|
||||||
|
Kind: "self-check-silent", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchSelfCheck,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.selfCheck.last }},
|
||||||
|
{Row: "S11", Signal: "node tools heartbeat", Emitter: "node tools", Trigger: "its interval",
|
||||||
|
Bound: "3 × the interval it says (60 s when it says none), provisional; only where node-tools is assigned",
|
||||||
|
Kind: "tools-silent", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchTools,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
|
||||||
|
}},
|
||||||
|
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
|
||||||
|
Bound: "15 s", Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
|
||||||
|
Deferred: "the lease is built in Phase 2 (to-be 45 §6): there is nothing renewed to watch yet, and a " +
|
||||||
|
"second controller is caught today by its consumers being bound (standingBy)"},
|
||||||
|
{Row: "S13", Signal: "stale refusals", Emitter: "every receiver (rule 2)", Trigger: "each refusal",
|
||||||
|
Bound: "more than 5 from one machine in 5 min", Kind: "stale-writer", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchStaleRefusals,
|
||||||
|
newest: func(f *signalFacts) time.Time { return time.Time{} }},
|
||||||
|
{Row: "S14", Signal: "facts snapshot exported", Emitter: "controller", Trigger: "daily",
|
||||||
|
Bound: "2 days", Kind: "facts-stale", Severity: conditions.Warning, Phase: 5,
|
||||||
|
Deferred: "the facts snapshot is built in Phase 5 (to-be 45 §9): nothing exports one yet"},
|
||||||
|
{Row: "S15", Signal: "a hand act with a cause already recorded", Emitter: "hand-act log",
|
||||||
|
Trigger: "each act", Bound: "the second within 14 days", Kind: "healer-wanted",
|
||||||
|
Severity: conditions.Warning, Phase: 3,
|
||||||
|
Deferred: "Phase 3 (to-be 45 §10): `hand-acts` lists repeated causes today; the condition comes with the healers"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// newestOf is the newest time among things.
|
||||||
|
func newestOf[T any](list []T, at func(T) time.Time) time.Time {
|
||||||
|
var newest time.Time
|
||||||
|
for _, x := range list {
|
||||||
|
if t := at(x); t.After(newest) {
|
||||||
|
newest = t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return newest
|
||||||
|
}
|
||||||
|
|
||||||
|
// ago is a duration as the summaries say it.
|
||||||
|
func ago(d time.Duration) string {
|
||||||
|
if d < time.Minute {
|
||||||
|
return d.Round(time.Second).String()
|
||||||
|
}
|
||||||
|
return d.Round(time.Minute).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
||||||
|
func heartbeatBound(every time.Duration) time.Duration {
|
||||||
|
if every <= 0 {
|
||||||
|
every = heartbeatEvery
|
||||||
|
}
|
||||||
|
return heartbeatsMissed * every
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if m.lastHeard.IsZero() || m.asleep() {
|
||||||
|
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bound := heartbeatBound(m.every)
|
||||||
|
silent := f.now.Sub(m.lastHeard)
|
||||||
|
if silent <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
severity := conditions.Warning
|
||||||
|
if m.control && silent > controlNodeUrgentAfter {
|
||||||
|
severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "silent",
|
||||||
|
Machine: m.name, Severity: severity,
|
||||||
|
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", m.name,
|
||||||
|
m.lastHeard.UTC().Format("2006-01-02 15:04 MST"), bound),
|
||||||
|
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchReports(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||||
|
continue // silent: S1 says it, and a silent machine reports nothing
|
||||||
|
}
|
||||||
|
bound := max(reportAtLeast, 3*m.lastApply)
|
||||||
|
waited := f.now.Sub(m.sentAt)
|
||||||
|
if waited <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name,
|
||||||
|
Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)",
|
||||||
|
m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||||
|
Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchPlans(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, p := range f.plans {
|
||||||
|
if p.paused {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
in := f.now.Sub(p.entered)
|
||||||
|
if in <= p.bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
|
||||||
|
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
|
||||||
|
ago(p.bound), p.waiting),
|
||||||
|
Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchLoop(f *signalFacts) []conditions.Observation {
|
||||||
|
if f.loop.pending == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
since := f.loop.took
|
||||||
|
if since.IsZero() {
|
||||||
|
since = f.started
|
||||||
|
}
|
||||||
|
if f.now.Sub(since) <= loopDeafAfter {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "controller", Kind: "controller-deaf",
|
||||||
|
Token: "deaf", Machine: f.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the controller's event loop has taken nothing for %s while its consumers hold %d "+
|
||||||
|
"message(s): reports, builds and merges are not being acted on", ago(f.now.Sub(since)), f.loop.pending),
|
||||||
|
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchMerges(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.merges {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, ID: m.Repo + "." + short(m.Commit),
|
||||||
|
Kind: "merge-not-acted", Token: "not-acted", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s/%s merged into %s (%s) was never handed to the controller by the bus; "+
|
||||||
|
"acted on late by the catch-up", m.Owner, m.Repo, m.Base, short(m.Commit)),
|
||||||
|
Said: fmt.Sprintf("announced %s, %s behind it", m.At.UTC().Format(time.RFC3339), readableList(m.Modules))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchAsks(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, a := range f.asks {
|
||||||
|
var said string
|
||||||
|
switch {
|
||||||
|
case a.state == link.AskDead:
|
||||||
|
said = fmt.Sprintf("the %s queue handed it out as often as it may and it was never settled", a.seat)
|
||||||
|
case a.state == link.AskInFlight && f.now.Sub(a.since) > a.bound:
|
||||||
|
said = fmt.Sprintf("in flight on %s for %s (bound %s)", orSomewhere(a.on), ago(f.now.Sub(a.since)), ago(a.bound))
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBuild, ID: a.id, Kind: "ask-lost",
|
||||||
|
Token: "lost", Machine: a.on, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the build %s of %s has no outcome: %s", a.id, a.what, said), Said: said})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSomewhere(node string) string {
|
||||||
|
if node == "" {
|
||||||
|
return "a machine that did not say which"
|
||||||
|
}
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchCalls(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range f.calls {
|
||||||
|
bound := callBound(c.Verb)
|
||||||
|
running := f.now.Sub(c.Started)
|
||||||
|
if running <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCall, ID: c.ID, Kind: "call-hung",
|
||||||
|
Token: "hung", Machine: f.host, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s.%s (call %s) has been running since %s, past its bound of %s", c.Seat, c.Verb,
|
||||||
|
c.ID, c.Started.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||||
|
Said: fmt.Sprintf("running %s, asked by %s", ago(running), orSomebody(c.Caller))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSomebody(caller string) string {
|
||||||
|
if caller == "" {
|
||||||
|
return "a caller the bus did not name"
|
||||||
|
}
|
||||||
|
return caller
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchProviders(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range f.standings {
|
||||||
|
quiet := f.now.Sub(c.LastObserved)
|
||||||
|
if quiet <= providerSaysAgainWithin {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module, node, consumer, ok := providerOf(c)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, ID: module + "." + node + "." + consumer,
|
||||||
|
Token: "silent", Kind: kindProviderSilent, Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped "+
|
||||||
|
"saying anything, so its last word is all the mesh has", module, node, consumer,
|
||||||
|
c.LastObserved.UTC().Format("2006-01-02 15:04 MST")),
|
||||||
|
Said: fmt.Sprintf("not said again for %s", ago(quiet))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, a := range f.advisories {
|
||||||
|
if f.now.Sub(a.Last) > advisoryQuiet {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||||
|
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||||
|
}
|
||||||
|
severity := conditions.Warning
|
||||||
|
times := ""
|
||||||
|
if a.Count > 1 {
|
||||||
|
times = fmt.Sprintf(" (%d times since %s)", a.Count, a.First.UTC().Format("15:04 MST"))
|
||||||
|
}
|
||||||
|
machine := ""
|
||||||
|
if a.ID == "controller" {
|
||||||
|
machine = f.host
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
|
||||||
|
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchSelfCheck(f *signalFacts) []conditions.Observation {
|
||||||
|
every := f.selfCheck.every
|
||||||
|
if every <= 0 {
|
||||||
|
every = doctorEvery
|
||||||
|
}
|
||||||
|
since := f.selfCheck.last
|
||||||
|
if since.IsZero() {
|
||||||
|
since = f.started
|
||||||
|
}
|
||||||
|
if f.now.Sub(since) <= 2*every {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "doctor", Kind: "self-check-silent",
|
||||||
|
Machine: f.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's "+
|
||||||
|
"invariants are not being checked", since.UTC().Format("2006-01-02 15:04 MST"), every),
|
||||||
|
Said: fmt.Sprintf("no run for %s", ago(f.now.Sub(since)))}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchTools(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if !m.tools || m.asleep() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bound := heartbeatBound(m.toolsEvery)
|
||||||
|
since := m.toolsHeard
|
||||||
|
if since.IsZero() {
|
||||||
|
// Not heard since this controller started: silent since then at the most.
|
||||||
|
since = f.toolsHeardFrom
|
||||||
|
}
|
||||||
|
silent := f.now.Sub(since)
|
||||||
|
if silent <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
heard := "not since this controller started at " + since.UTC().Format("2006-01-02 15:04 MST")
|
||||||
|
if !m.toolsHeard.IsZero() {
|
||||||
|
heard = "since " + m.toolsHeard.UTC().Format("2006-01-02 15:04 MST")
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "tools-silent",
|
||||||
|
Machine: m.name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the node tools on %s have not said they are there %s (bound %s): nothing can "+
|
||||||
|
"ask that machine anything", m.name, heard, bound),
|
||||||
|
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchStaleRefusals(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for node, n := range f.staleRefusals {
|
||||||
|
if n <= staleRefusalsAllowed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "stale-writer",
|
||||||
|
Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s refused %d declarations in %s as older than the one it holds: a controller "+
|
||||||
|
"is sending what it has moved past (which one is said once declarations carry an epoch, Phase 2)",
|
||||||
|
node, n, staleRefusalsWithin),
|
||||||
|
Said: fmt.Sprintf("%d stale refusals in %s", n, staleRefusalsWithin)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchedRows are the rows a watchdog runs for.
|
||||||
|
func watchedRows() []signalRow {
|
||||||
|
var out []signalRow
|
||||||
|
for _, r := range signalsTable {
|
||||||
|
if r.watch != nil {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// kindsOf is a row's condition kinds, one or several.
|
||||||
|
func kindsOf(r signalRow) []string {
|
||||||
|
var out []string
|
||||||
|
for _, k := range strings.Split(r.Kind, ",") {
|
||||||
|
out = append(out, strings.TrimSpace(k))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
|
||||||
|
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
|
||||||
|
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
|
||||||
|
// and the condition clears. A row that is not watched says why. A row added to the table without a
|
||||||
|
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
|
||||||
|
|
||||||
|
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
|
||||||
|
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
|
||||||
|
// running, the self-check a minute old.
|
||||||
|
func calm(now time.Time) *signalFacts {
|
||||||
|
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
||||||
|
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
||||||
|
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||||
|
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
||||||
|
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||||
|
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||||
|
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
||||||
|
mergesPassed: now.Add(-time.Minute),
|
||||||
|
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
||||||
|
lostConsumers: map[string]bool{}, staleRefusals: map[string]int{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// suppression is one row's signal held back: inside its bound, and past it.
|
||||||
|
type suppression struct{ inside, past func(f *signalFacts) }
|
||||||
|
|
||||||
|
// suppressions are every watched row's, by row.
|
||||||
|
var suppressions = map[string]suppression{
|
||||||
|
"S1": {
|
||||||
|
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
|
||||||
|
},
|
||||||
|
"S2": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S3": {
|
||||||
|
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
|
||||||
|
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
|
||||||
|
},
|
||||||
|
"S4": {
|
||||||
|
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
||||||
|
},
|
||||||
|
"S5": {
|
||||||
|
inside: func(f *signalFacts) {},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S6": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||||
|
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||||
|
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S7": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S8": {
|
||||||
|
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
||||||
|
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
||||||
|
},
|
||||||
|
"S9": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||||
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||||
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S10": {
|
||||||
|
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
|
||||||
|
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
|
||||||
|
},
|
||||||
|
"S11": {
|
||||||
|
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
|
||||||
|
},
|
||||||
|
"S13": {
|
||||||
|
inside: func(f *signalFacts) { f.staleRefusals = map[string]int{"anchor": 5} },
|
||||||
|
past: func(f *signalFacts) { f.staleRefusals = map[string]int{"anchor": 6} },
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// standingSaid is a provider's failing word last said at a moment.
|
||||||
|
func standingSaid(at time.Time) conditions.Condition {
|
||||||
|
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, row := range signalsTable {
|
||||||
|
t.Run(row.Row, func(t *testing.T) {
|
||||||
|
if seen[row.Row] {
|
||||||
|
t.Fatalf("%s is in the table twice", row.Row)
|
||||||
|
}
|
||||||
|
seen[row.Row] = true
|
||||||
|
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
|
||||||
|
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
|
||||||
|
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
|
||||||
|
}
|
||||||
|
if row.Deferred != "" {
|
||||||
|
if row.watch != nil || row.Phase <= 1 {
|
||||||
|
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
|
||||||
|
}
|
||||||
|
if _, has := suppressions[row.Row]; has {
|
||||||
|
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if row.watch == nil || row.needs == nil || row.newest == nil {
|
||||||
|
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
|
||||||
|
}
|
||||||
|
s, ok := suppressions[row.Row]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
|
||||||
|
}
|
||||||
|
if got := row.watch(calm(now)); len(got) != 0 {
|
||||||
|
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
|
||||||
|
}
|
||||||
|
inside := calm(now)
|
||||||
|
s.inside(inside)
|
||||||
|
if got := row.watch(inside); len(got) != 0 {
|
||||||
|
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
|
||||||
|
}
|
||||||
|
past := calm(now)
|
||||||
|
s.past(past)
|
||||||
|
got := row.watch(past)
|
||||||
|
if len(got) == 0 {
|
||||||
|
t.Fatalf("%s raised nothing past its bound", row.Row)
|
||||||
|
}
|
||||||
|
for _, o := range got {
|
||||||
|
if !slices.Contains(kindsOf(row), o.Kind) {
|
||||||
|
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
|
||||||
|
}
|
||||||
|
if o.Severity != row.Severity {
|
||||||
|
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(o.Summary) == "" {
|
||||||
|
t.Errorf("%s raised a condition that says nothing", row.Row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Through the store: raised past the bound, cleared when the signal returns.
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
told := &conditions.Told{}
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
||||||
|
Now: func() time.Time { return now }})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||||
|
w.see(t.Context(), past)
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil || len(open) != len(got) {
|
||||||
|
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
|
||||||
|
}
|
||||||
|
w.see(t.Context(), calm(now))
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for name := range suppressions {
|
||||||
|
if !seen[name] {
|
||||||
|
t.Errorf("a suppression for %s, which the table does not have", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
|
||||||
|
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
f := calm(now)
|
||||||
|
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
|
||||||
|
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
|
||||||
|
got := watchHeartbeats(f)
|
||||||
|
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
|
||||||
|
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
f := calm(now)
|
||||||
|
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
|
||||||
|
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
|
||||||
|
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
|
||||||
|
t.Fatalf("a sleeping machine raised %+v", got)
|
||||||
|
}
|
||||||
|
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
|
||||||
|
if got := watchHeartbeats(f); len(got) != 1 {
|
||||||
|
t.Fatalf("a woken machine silent past its bound raised %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
|
||||||
|
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
|
||||||
|
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||||
|
silent := calm(now)
|
||||||
|
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
|
||||||
|
w.see(t.Context(), silent)
|
||||||
|
blind := calm(now)
|
||||||
|
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
|
||||||
|
w.see(t.Context(), blind)
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range open {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
|
||||||
|
if !slices.Contains(keys, want) {
|
||||||
|
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.see(t.Context(), calm(now))
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("seeing again left open %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
|
||||||
|
// every machine silent.
|
||||||
|
func TestAControllerStandingBySaysNothing(t *testing.T) {
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
|
||||||
|
w.tick(t.Context())
|
||||||
|
if w.lastTick().IsZero() {
|
||||||
|
t.Fatal("a tick standing by was not counted")
|
||||||
|
}
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("%+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224) —
|
||||||
|
// **the condition store's first kind** (to-be 45 §2, ADR 0227).
|
||||||
|
//
|
||||||
|
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
||||||
|
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
||||||
|
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
||||||
|
// provider announces a consumer it has failed for minutes; the controller keeps it until the provider
|
||||||
|
// says it recovered; and `status`, its JSON and `node show` name it, breaking "all well". Unchanged in
|
||||||
|
// what it says and when; kept as a condition, `provider.<module>.<node>.<consumer>.failing`, rather
|
||||||
|
// than a row of its own, so it is said outward like every other fault and silenced like one.
|
||||||
|
|
||||||
|
// Kinds of the provider standing.
|
||||||
|
const (
|
||||||
|
kindProviderFailing = "provider-failing"
|
||||||
|
kindProviderSilent = "provider-silent"
|
||||||
|
// sourceProvisioner is what raised a standing: the provider's own event.
|
||||||
|
sourceProvisioner = "provisioner.failing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// providerSaysAgainWithin is how long a failing word stays current without being said again: twice
|
||||||
|
// the quarter of an hour a provider repeats it at (ADR 0224). Past it, S8.
|
||||||
|
const providerSaysAgainWithin = 30 * time.Minute
|
||||||
|
|
||||||
|
// standings keeps what providers say, as conditions.
|
||||||
|
type standings struct {
|
||||||
|
keeper func() *conditions.Keeper
|
||||||
|
}
|
||||||
|
|
||||||
|
// standingObservation is a provider's failing word as an observation: the provider, its machine and
|
||||||
|
// the consumer name it, so the same consumer failed again is the same condition.
|
||||||
|
func standingObservation(st link.Standing) conditions.Observation {
|
||||||
|
whom := st.Consumer
|
||||||
|
if st.Node != "" {
|
||||||
|
whom += " on " + st.Node
|
||||||
|
}
|
||||||
|
summary := fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode,
|
||||||
|
whom, orUnclassed(st.Class), st.Attempts, st.Since.UTC().Format("2006-01-02 15:04 MST"))
|
||||||
|
said := orUnclassed(st.Class)
|
||||||
|
if e := firstLine(st.Error); e != "" {
|
||||||
|
said += ": " + e
|
||||||
|
}
|
||||||
|
if st.Provider != "" {
|
||||||
|
said += fmt.Sprintf(" (provision %s, %d attempts)", st.Provider, st.Attempts)
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeProvider,
|
||||||
|
ID: st.Module + "." + st.ProviderNode + "." + st.Consumer,
|
||||||
|
Token: "failing", Kind: kindProviderFailing, Machine: st.ProviderNode, Also: alsoOn(st.Node, st.ProviderNode),
|
||||||
|
Severity: conditions.Warning, Summary: summary, Said: said, Source: sourceProvisioner}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stood keeps a provider's newest word: failing raises or observes its condition, recovered clears
|
||||||
|
// it. An error is the store away, and the link holds the message to be asked again — a recovery is
|
||||||
|
// said once, and dropping it would leave a consumer named failing that is fine.
|
||||||
|
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
||||||
|
k := s.keeper()
|
||||||
|
if k == nil {
|
||||||
|
return false, fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
|
||||||
|
}
|
||||||
|
o := standingObservation(st)
|
||||||
|
if !st.Failing {
|
||||||
|
why := "the provider says it recovered"
|
||||||
|
if st.Why != "" {
|
||||||
|
why += ": " + st.Why
|
||||||
|
}
|
||||||
|
cleared, err := k.Clear(ctx, o.Key(), why)
|
||||||
|
return cleared, storeAway(err)
|
||||||
|
}
|
||||||
|
_, err := k.Observe(ctx, o)
|
||||||
|
return false, storeAway(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// storeAway reads the condition store failing as the bus being away for the moment: the link holds the
|
||||||
|
// message and asks again, as it does for a store restarting (ADR 0083), rather than taking it unkept.
|
||||||
|
func storeAway(err error) error {
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%v: %w", err, link.ErrTryAgain)
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerStandings is every open provider-failing condition, from what is open.
|
||||||
|
func providerStandings(open []conditions.Condition) []conditions.Condition {
|
||||||
|
var out []conditions.Condition
|
||||||
|
for _, c := range open {
|
||||||
|
if c.Kind == kindProviderFailing {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerOf reads a standing's provider module and machine back from its key.
|
||||||
|
func providerOf(c conditions.Condition) (module, node, consumer string, ok bool) {
|
||||||
|
parts := strings.Split(c.Key, ".")
|
||||||
|
if len(parts) != 5 || parts[0] != conditions.ScopeProvider {
|
||||||
|
return "", "", "", false
|
||||||
|
}
|
||||||
|
return parts[1], parts[2], parts[3], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// unassignedProviders clears the standing of every provider no longer assigned where it ran.
|
||||||
|
//
|
||||||
|
// **A provider no longer assigned is not asked about** (ADR 0224 §4): nothing runs there to fail
|
||||||
|
// anybody, and nothing there will ever say it recovered. The observation that resolves it is the
|
||||||
|
// assignment. Assigned again, its first failure raises it again.
|
||||||
|
func unassignedProviders(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper,
|
||||||
|
open []conditions.Condition) error {
|
||||||
|
assigned := map[string]map[string]bool{}
|
||||||
|
for _, c := range providerStandings(open) {
|
||||||
|
module, node, _, ok := providerOf(c)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
on, asked := assigned[node]
|
||||||
|
if !asked {
|
||||||
|
modules, err := inv.Assigned(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, inventory.ErrNoSuchNode) {
|
||||||
|
modules = nil // a machine the mesh no longer knows runs nothing
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("what %s is assigned cannot be read: %w", node, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
on = map[string]bool{}
|
||||||
|
for _, m := range modules {
|
||||||
|
on[m] = true
|
||||||
|
}
|
||||||
|
assigned[node] = on
|
||||||
|
}
|
||||||
|
if !on[module] {
|
||||||
|
if _, err := k.Clear(ctx, c.Key, module+" is no longer assigned to "+node+
|
||||||
|
": nothing runs there to fail anybody"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orUnclassed(class string) string {
|
||||||
|
if class == "" {
|
||||||
|
return "failing"
|
||||||
|
}
|
||||||
|
return class
|
||||||
|
}
|
||||||
|
|
||||||
|
// alsoOn is a consumer's machine, when it is not the provider's.
|
||||||
|
func alsoOn(consumerNode, providerNode string) []string {
|
||||||
|
if consumerNode == "" || consumerNode == providerNode {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []string{consumerNode}
|
||||||
|
}
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224), kept as
|
||||||
|
// the condition store's first kind (to-be 45 §2). On 2026-10-05 the identity provider refused every
|
||||||
|
// consumer for a day and status called the mesh well (04-ISSUES/179): this is that day, told to the
|
||||||
|
// controller the way the provider now tells it.
|
||||||
|
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
|
||||||
|
Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
|
||||||
|
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||||
|
since := time.Now().Add(-23 * time.Hour)
|
||||||
|
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
||||||
|
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
||||||
|
Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
|
||||||
|
if _, err := kept.Stood(ctx, failing); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asked.well() {
|
||||||
|
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
||||||
|
}
|
||||||
|
said := printed(t, func() error { return printStatus(asked) })
|
||||||
|
for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing",
|
||||||
|
"idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} {
|
||||||
|
if !strings.Contains(said, want) {
|
||||||
|
t.Fatalf("status does not say %q:\n%s", want, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(said, "all doing what they were told") {
|
||||||
|
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(said, "1 open condition(s)") {
|
||||||
|
t.Fatalf("status does not lead with what is open:\n%s", said)
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Conditions []conditions.Condition `json:"conditions"`
|
||||||
|
Failing []conditions.Condition `json:"failing"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 ||
|
||||||
|
!strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") {
|
||||||
|
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||||
|
}
|
||||||
|
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||||
|
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||||
|
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recovered: gone, and the mesh may be well again as far as this is concerned.
|
||||||
|
failing.Failing = false
|
||||||
|
if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
|
||||||
|
t.Fatalf("%v %v", cleared, err)
|
||||||
|
}
|
||||||
|
asked, err = theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(asked.conditions) != 0 {
|
||||||
|
t.Fatalf("a recovered consumer is still named: %+v", asked.conditions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
||||||
|
// cleared on the next look, said as resolved by the assignment.
|
||||||
|
func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||||
|
if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||||
|
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
all, err := conditionsFrom.Open(ctx)
|
||||||
|
if err != nil || len(all) != 1 {
|
||||||
|
t.Fatalf("%+v %v", all, err)
|
||||||
|
}
|
||||||
|
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if all, _ := conditionsFrom.Open(ctx); len(all) != 0 {
|
||||||
|
t.Fatalf("%+v", all)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The store away holds a recovery** (ADR 0224 §3): a standing that cannot be kept is asked again.
|
||||||
|
func TestAStandingTheStoreCannotKeepIsAskedAgain(t *testing.T) {
|
||||||
|
kept := standings{keeper: func() *conditions.Keeper { return nil }}
|
||||||
|
if _, err := kept.Stood(t.Context(), link.Standing{Module: "idp", ProviderNode: "anchor", Consumer: "x"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), link.ErrTryAgain.Error()) {
|
||||||
|
t.Fatalf("answered %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
)
|
)
|
||||||
@@ -81,8 +82,12 @@ func printStatus(asked answers) error {
|
|||||||
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||||
behind, sources := asked.behind, asked.sources
|
behind, sources := asked.behind, asked.sources
|
||||||
|
|
||||||
|
// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
|
||||||
|
// first, silenced ones with when their silence ends.
|
||||||
|
printConditions(asked.conditions, asked.conditionsUnread, time.Now())
|
||||||
|
|
||||||
if len(asked.refused) > 0 {
|
if len(asked.refused) > 0 {
|
||||||
// First, above everything else. A machine that cannot be worked out is not running an old
|
// First of what follows. A machine that cannot be worked out is not running an old
|
||||||
// declaration — it has no declaration, and nothing below this line is about it.
|
// declaration — it has no declaration, and nothing below this line is about it.
|
||||||
var names []string
|
var names []string
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
@@ -138,14 +143,14 @@ func printStatus(asked answers) error {
|
|||||||
len(quiet), strings.Join(said, "\n "))
|
len(quiet), strings.Join(said, "\n "))
|
||||||
}
|
}
|
||||||
|
|
||||||
if open, late := openPlans(asked.plans); len(open) > 0 {
|
if open, late := openPlans(asked.plans, asked.paused); len(open) > 0 {
|
||||||
fmt.Printf("%d plan(s) open", len(open))
|
fmt.Printf("%d plan(s) open", len(open))
|
||||||
if late > 0 {
|
if late > 0 {
|
||||||
fmt.Printf(", %d waiting past %s", late, planWaitBound)
|
fmt.Printf(", %d waiting past %s", late, planWaitBound)
|
||||||
}
|
}
|
||||||
fmt.Println(":")
|
fmt.Println(":")
|
||||||
for _, p := range open {
|
for _, p := range open {
|
||||||
fmt.Printf(" %s\n", planLine(p, time.Now()))
|
fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.paused))
|
||||||
}
|
}
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
}
|
}
|
||||||
@@ -282,6 +287,45 @@ func printStatus(asked answers) error {
|
|||||||
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(asked.unheld) > 0 {
|
||||||
|
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
||||||
|
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
||||||
|
// holds, until every machine has its holders and the switch makes it a refusal.
|
||||||
|
fmt.Printf("%d module dependenc(ies) on a seat nothing on the machine holds (unheld, ADR 0207):\n",
|
||||||
|
len(asked.unheld))
|
||||||
|
for _, u := range asked.unheld {
|
||||||
|
holders := "no module in the catalogue claims it yet"
|
||||||
|
if len(u.Holders) > 0 {
|
||||||
|
holders = "could be held by " + strings.Join(u.Holders, ", ")
|
||||||
|
}
|
||||||
|
fmt.Printf(" %-12s %-24s %-24s %s\n", u.Node, u.Module, u.Seat, holders)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(asked.overflowing) > 0 {
|
||||||
|
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
|
||||||
|
// out of its provider's grants, so the module holds a login nothing created; the provider's
|
||||||
|
// machine is sent everything else rather than refused for one consumer elsewhere.
|
||||||
|
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
|
||||||
|
len(asked.overflowing))
|
||||||
|
for _, o := range asked.overflowing {
|
||||||
|
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
||||||
|
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
||||||
|
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
|
||||||
|
switch {
|
||||||
|
case asked.handActsUnread != "":
|
||||||
|
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
|
||||||
|
asked.handActsUnread)
|
||||||
|
case asked.handActs != nil && *asked.handActs > 0:
|
||||||
|
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
|
||||||
|
}
|
||||||
|
|
||||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||||
@@ -291,8 +335,9 @@ func printStatus(asked answers) error {
|
|||||||
|
|
||||||
if asked.well() {
|
if asked.well() {
|
||||||
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||||
// and getting here means every question was asked and answered.
|
// and getting here means every question was asked and answered. **No open conditions first**
|
||||||
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
|
// (novox/hq to-be 45 §2): it is what the sentence means now, silenced ones included.
|
||||||
|
fmt.Printf("no open conditions; %d machine(s), all doing what they were told, all heard from, running what "+
|
||||||
"the mesh would send them, and every module current with its source\n", len(nodes))
|
"the mesh would send them, and every module current with its source\n", len(nodes))
|
||||||
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
||||||
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
||||||
@@ -386,10 +431,48 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
}
|
}
|
||||||
|
// And which machines run a module whose resources a seat nothing there holds applies (novox/hq
|
||||||
|
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||||
|
// the private network is built from and should say nothing else; a machine that does not
|
||||||
|
// resolve is already in refused, and is passed over here.
|
||||||
|
plans := map[string]planned{}
|
||||||
|
for _, n := range out.nodes {
|
||||||
|
plan, settings, err := planFor(ctx, open, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
if unresolvable(err) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
plans[n.Name] = planned{plan, settings}
|
||||||
|
out.unheld = append(out.unheld, plan.Unheld...)
|
||||||
|
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||||
|
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||||
|
// resolution, as the provider's composition judges it.
|
||||||
|
out.overflowing = append(out.overflowing, plan.Overflowing()...)
|
||||||
|
}
|
||||||
|
// And every open condition (novox/hq to-be 45 §2): what the watchdogs, the self-check and the
|
||||||
|
// providers' own words say is wrong — a provider failing a consumer among them (ADR 0224). Kept on
|
||||||
|
// the bus; a process that cannot read them says so, and the mesh is then not called well.
|
||||||
|
if out.conditions, err = openConditions(ctx); err != nil {
|
||||||
|
out.conditionsUnread = err.Error()
|
||||||
|
}
|
||||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
}
|
}
|
||||||
|
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||||
|
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||||
|
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||||
|
// to read the log from; a process with none has no log to count.
|
||||||
|
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||||
|
n, unread := handActsThisWeek(ctx)
|
||||||
|
if unread != "" {
|
||||||
|
out.handActsUnread = unread
|
||||||
|
} else {
|
||||||
|
out.handActs = &n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// And which machines are not running what the mesh would send them. The same question as a
|
// And which machines are not running what the mesh would send them. The same question as a
|
||||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||||
@@ -401,7 +484,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||||
// reason is kept and reported as data; every question that does not depend on it is still
|
// reason is kept and reported as data; every question that does not depend on it is still
|
||||||
// answered.
|
// answered.
|
||||||
would, err := wouldSend(ctx, open, out.nodes)
|
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
out.network = err.Error()
|
out.network = err.Error()
|
||||||
would = map[string]string{}
|
would = map[string]string{}
|
||||||
@@ -496,7 +579,8 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
|||||||
func (a answers) well() bool {
|
func (a answers) well() bool {
|
||||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||||
len(a.filtered) == 0
|
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||||
|
len(a.conditions) == 0 && a.conditionsUnread == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// hostSplit is which machines report which host version, for every version more than one machine
|
// hostSplit is which machines report which host version, for every version more than one machine
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
|
||||||
|
// §4's D9 and §8's health of the controller).
|
||||||
|
//
|
||||||
|
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
|
||||||
|
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
|
||||||
|
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
|
||||||
|
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
|
||||||
|
// controller composes it in the background — at its start, after anything that changes what it says
|
||||||
|
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
|
||||||
|
// the last composition at once, saying when it was composed and how long that took. A caller who
|
||||||
|
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
|
||||||
|
|
||||||
|
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
|
||||||
|
// long a nudge waits for the next, so a push answered by four machines is composed once.
|
||||||
|
var (
|
||||||
|
statusEvery = time.Minute
|
||||||
|
statusSettle = 2 * time.Second
|
||||||
|
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
|
||||||
|
// good; the attempt is said as failed, and the last summary stands with its age.
|
||||||
|
statusComposeWithin = 2 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// statusSummary is the last composed `status --json`, and when.
|
||||||
|
type statusSummary struct {
|
||||||
|
compose func(context.Context) ([]byte, error)
|
||||||
|
// every, settle and within are the clocks above, read once when it is made.
|
||||||
|
every, settle, within time.Duration
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
body []byte
|
||||||
|
composedAt time.Time
|
||||||
|
took time.Duration
|
||||||
|
failed string
|
||||||
|
failedAt time.Time
|
||||||
|
started time.Time
|
||||||
|
first chan struct{} // closed when the first attempt ends, either way
|
||||||
|
|
||||||
|
nudged chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
|
||||||
|
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
|
||||||
|
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
|
||||||
|
// composed when asked, as at a shell.
|
||||||
|
var statusFrom *statusSummary
|
||||||
|
|
||||||
|
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
|
||||||
|
func (s *statusSummary) nudge() {
|
||||||
|
if s == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case s.nudged <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
|
||||||
|
func (s *statusSummary) keep(ctx context.Context) {
|
||||||
|
once := sync.Once{}
|
||||||
|
for {
|
||||||
|
s.composeOnce(ctx)
|
||||||
|
once.Do(func() { close(s.first) })
|
||||||
|
timer := time.NewTimer(s.every)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
timer.Stop()
|
||||||
|
return
|
||||||
|
case <-timer.C:
|
||||||
|
case <-s.nudged:
|
||||||
|
timer.Stop()
|
||||||
|
// Let what else is arriving arrive, then compose once for all of it.
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(s.settle):
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-s.nudged:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *statusSummary) composeOnce(ctx context.Context) {
|
||||||
|
start := time.Now()
|
||||||
|
asking, cancel := context.WithTimeout(ctx, s.within)
|
||||||
|
body, err := s.compose(asking)
|
||||||
|
cancel()
|
||||||
|
took := time.Since(start)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
s.failed, s.failedAt = err.Error(), time.Now()
|
||||||
|
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
|
||||||
|
"with its age\n", took.Round(time.Millisecond), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// answer is what the `status` verb answers: the last summary at once, the same document `status
|
||||||
|
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
|
||||||
|
// but never past the caller's window; a controller that has none says so and why, rather than
|
||||||
|
// answering an empty mesh as a well one.
|
||||||
|
func (s *statusSummary) answer(ctx context.Context) (any, error) {
|
||||||
|
wait := time.NewTimer(link.AnswerWithin - time.Second)
|
||||||
|
defer wait.Stop()
|
||||||
|
select {
|
||||||
|
case <-s.first:
|
||||||
|
case <-wait.C:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.body == nil {
|
||||||
|
why := "its first composition has not finished"
|
||||||
|
if s.failed != "" {
|
||||||
|
why = "it could not be composed: " + s.failed
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
|
||||||
|
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
|
||||||
|
}
|
||||||
|
var parsed any
|
||||||
|
_ = json.Unmarshal(s.body, &parsed)
|
||||||
|
out := map[string]any{
|
||||||
|
"output": string(s.body), "ok": true, "answer": parsed,
|
||||||
|
"composed": s.composedAt.UTC().Format(time.RFC3339),
|
||||||
|
"age": time.Since(s.composedAt).Round(time.Second).String(),
|
||||||
|
"composedIn": s.took.Round(time.Millisecond).String(),
|
||||||
|
"note": "composed by the serving controller at its start, after each report, build or act, and " +
|
||||||
|
"every minute; answered at once from the last composition",
|
||||||
|
}
|
||||||
|
if s.failed != "" && s.failedAt.After(s.composedAt) {
|
||||||
|
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
|
||||||
|
s.failedAt.UTC().Format(time.RFC3339), s.failed)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeStatus is `status --json`, composed in this process against its stores.
|
||||||
|
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||||
|
return func(ctx context.Context) ([]byte, error) {
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return statusAsJSON(asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
|
||||||
|
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
|
||||||
|
// controller composing for ever.
|
||||||
|
var readingVerbs = map[string]bool{
|
||||||
|
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||||
|
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||||
|
"doctor": true, "conditions": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||||
|
type nudgingListener struct {
|
||||||
|
link.Enrolment
|
||||||
|
summary *statusSummary
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||||
|
// A declaration refused as older than the one the machine holds is counted (novox/hq to-be 45 S13).
|
||||||
|
if link.IsStaleRefusal(report.Refused) {
|
||||||
|
link.StaleRefusals.Refused(report.Node, time.Now())
|
||||||
|
}
|
||||||
|
news, err := l.Enrolment.Heard(ctx, report)
|
||||||
|
if news {
|
||||||
|
l.summary.nudge()
|
||||||
|
}
|
||||||
|
return news, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// quickly shortens the summary's clocks for one test.
|
||||||
|
func quickly(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
every, settle := statusEvery, statusSettle
|
||||||
|
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
|
||||||
|
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
|
||||||
|
}
|
||||||
|
|
||||||
|
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
|
||||||
|
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
|
||||||
|
// verb answered "still running"; from the summary it answers at once, in full, saying when.
|
||||||
|
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
var composed atomic.Int32
|
||||||
|
slow := make(chan struct{})
|
||||||
|
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
|
||||||
|
if composed.Add(1) > 1 {
|
||||||
|
<-slow // every composition after the first outlasts any caller
|
||||||
|
}
|
||||||
|
return []byte(`{"wrong":[],"machines":4}`), nil
|
||||||
|
})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
defer close(slow)
|
||||||
|
go s.keep(ctx)
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
s.nudge() // a composition is under way and does not finish
|
||||||
|
start := time.Now()
|
||||||
|
got, err := s.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if took := time.Since(start); took > time.Second {
|
||||||
|
t.Fatalf("answer %d took %s", i+1, took)
|
||||||
|
}
|
||||||
|
m := got.(map[string]any)
|
||||||
|
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
|
||||||
|
t.Fatalf("answer %d was not in full: %v", i+1, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first composition is waited for, never past the caller's window; a controller with none yet
|
||||||
|
// says so rather than answering an empty mesh as a well one.
|
||||||
|
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
was := link.AnswerWithin
|
||||||
|
link.AnswerWithin = 1100 * time.Millisecond
|
||||||
|
t.Cleanup(func() { link.AnswerWithin = was })
|
||||||
|
never := make(chan struct{})
|
||||||
|
defer close(never)
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
start := time.Now()
|
||||||
|
_, err := s.answer(ctx)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
|
||||||
|
t.Fatalf("answered %v", err)
|
||||||
|
}
|
||||||
|
if took := time.Since(start); took > link.AnswerWithin {
|
||||||
|
t.Fatalf("waited %s, past the caller's window", took)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A nudge composes it again, once for several close together; a failed composition leaves the last
|
||||||
|
// summary standing and says it is the last that could be composed.
|
||||||
|
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
var composed atomic.Int32
|
||||||
|
fail := atomic.Bool{}
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) {
|
||||||
|
n := composed.Add(1)
|
||||||
|
if fail.Load() {
|
||||||
|
return nil, errors.New("the store did not answer")
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(map[string]any{"n": n})
|
||||||
|
return body, nil
|
||||||
|
})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
if _, err := s.answer(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.nudge()
|
||||||
|
s.nudge()
|
||||||
|
s.nudge()
|
||||||
|
waitFor(t, func() bool { return composed.Load() == 2 })
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if n := composed.Load(); n != 2 {
|
||||||
|
t.Fatalf("three nudges together composed %d times after the first", n-1)
|
||||||
|
}
|
||||||
|
fail.Store(true)
|
||||||
|
s.nudge()
|
||||||
|
waitFor(t, func() bool { return composed.Load() == 3 })
|
||||||
|
waitFor(t, func() bool {
|
||||||
|
got, err := s.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := got.(map[string]any)
|
||||||
|
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The seat's `status` answers from the summary when this process keeps one.
|
||||||
|
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
statusFrom = s
|
||||||
|
t.Cleanup(func() { statusFrom = nil })
|
||||||
|
handlers, _, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
|
||||||
|
t.Fatalf("answered %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitFor(t *testing.T, ok func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
|
for !ok() {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("never happened")
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 254, ADR 0218: a newer plan takes over what the older open plans of its repository
|
||||||
|
// and branch had not built, and closes them as superseded; another repository's plan, another
|
||||||
|
// branch's, and a plan made after it are left alone.
|
||||||
|
func TestANewerPlanSupersedesTheOlderOpenPlansOfItsRepository(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
sent := at.Add(time.Minute)
|
||||||
|
plan := func(id, repository, branch string, created time.Time, modules map[string]*inventory.PlanModule) inventory.Plan {
|
||||||
|
return inventory.Plan{ID: id, Repository: repository, Branch: branch, Commit: id + "-commit",
|
||||||
|
Created: created, State: inventory.PlanRolling, Modules: modules}
|
||||||
|
}
|
||||||
|
older := plan("plan-1", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{
|
||||||
|
"gitea": {State: "built", SentAt: &sent}, // done with: stays done
|
||||||
|
"keycloak": {State: "asked"}, // asked, not answered: folded
|
||||||
|
"plex": {}, // not yet asked: folded
|
||||||
|
"agent": {State: "built"}, // built, rolls out, not sent: folded
|
||||||
|
"notes": {State: "built"}, // built, records: nothing to send
|
||||||
|
})
|
||||||
|
stuck := plan("plan-0", "Novox/Mesh-Catalog", "", at.Add(-time.Hour), map[string]*inventory.PlanModule{
|
||||||
|
"runtime": {State: "asked"},
|
||||||
|
})
|
||||||
|
other := plan("plan-2", "novox/mesh-controller", "main", at, map[string]*inventory.PlanModule{"mesh-controller": {}})
|
||||||
|
release := plan("plan-3", "novox/mesh-catalog", "release", at, map[string]*inventory.PlanModule{"lemurs": {}})
|
||||||
|
later := plan("plan-5", "novox/mesh-catalog", "main", at.Add(2*time.Hour), map[string]*inventory.PlanModule{"later": {}})
|
||||||
|
done := plan("plan-6", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{"finished": {}})
|
||||||
|
done.State = inventory.PlanDone
|
||||||
|
|
||||||
|
newer := plan("plan-4", "novox/mesh-catalog", "main", at.Add(time.Hour), nil)
|
||||||
|
newer.Commit = "97b1b2b0c0ffee"
|
||||||
|
rollsOut := func(m string) bool { return m != "notes" }
|
||||||
|
folded, closed := supersededBy(newer, []inventory.Plan{stuck, older, other, release, later, done, newer}, rollsOut)
|
||||||
|
|
||||||
|
if want := []string{"agent", "keycloak", "plex", "runtime"}; !reflect.DeepEqual(folded, want) {
|
||||||
|
t.Fatalf("folded %v, wanted %v", folded, want)
|
||||||
|
}
|
||||||
|
var ids []string
|
||||||
|
for _, p := range closed {
|
||||||
|
ids = append(ids, p.ID)
|
||||||
|
if p.State != inventory.PlanSuperseded || p.Open() {
|
||||||
|
t.Errorf("%s was left %s", p.ID, p.State)
|
||||||
|
}
|
||||||
|
if !strings.Contains(p.Note, "plan-4") || !strings.Contains(p.Note, "97b1b2b0") {
|
||||||
|
t.Errorf("%s does not name the plan that superseded it: %q", p.ID, p.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if want := []string{"plan-0", "plan-1"}; !reflect.DeepEqual(ids, want) {
|
||||||
|
t.Fatalf("superseded %v, wanted %v — another repository, another branch, a later plan and a "+
|
||||||
|
"finished one are left alone", ids, want)
|
||||||
|
}
|
||||||
|
if other.State != inventory.PlanRolling {
|
||||||
|
t.Fatal("the plan handed in was changed in place")
|
||||||
|
}
|
||||||
|
if line := planLine(closed[1], time.Now()); !strings.Contains(line, "superseded") {
|
||||||
|
t.Fatalf("a superseded plan reads %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 254: a person closes a plan that will not move again, by its id.
|
||||||
|
func TestAPersonClosesAStuckPlan(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
|
||||||
|
Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Fatalf("a plan was closed by hand without saying why: %v", err)
|
||||||
|
}
|
||||||
|
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
|
||||||
|
!strings.Contains(closed.Note, "its report will not come") {
|
||||||
|
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||||
|
}
|
||||||
|
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
|
||||||
|
t.Fatal("a plan already closed was closed again")
|
||||||
|
}
|
||||||
|
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
|
||||||
|
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
|
||||||
|
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An act says what it changed about the node it acted on, and nothing about the rest of the mesh
|
||||||
|
// (novox/hq ADR 0207): after the seat dependencies shipped, every `push <node>` and `assign` printed
|
||||||
|
// every node's unmet dependencies, a hundred lines around the one about the module just assigned.
|
||||||
|
|
||||||
|
func aContainer(name string) catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: name, Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": name, "type": "container", "image": name}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, aContainer("web"))
|
||||||
|
register(t, open, aContainer("db"))
|
||||||
|
// anchor already lacks a runtime for db: true, and not this act's to say.
|
||||||
|
if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said, err := assign(ctx, open, "laptop", "web")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%v\n%s", err, said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) {
|
||||||
|
t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said)
|
||||||
|
}
|
||||||
|
for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} {
|
||||||
|
if strings.Contains(said, not) {
|
||||||
|
t.Errorf("the assignment says %q, which it did not change:\n%s", not, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) {
|
||||||
|
shelf := map[string]catalogue.Manifest{
|
||||||
|
"web": aContainer("web"),
|
||||||
|
"docker": {Module: "docker", Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat}},
|
||||||
|
Resources: []map[string]any{{"id": "d", "type": "container", "image": "dind"}}},
|
||||||
|
}
|
||||||
|
lines := unheldChange(shelf, "laptop", []string{"web"}, []string{"web", "docker"})
|
||||||
|
if len(lines) != 1 || !strings.Contains(lines[0], "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") {
|
||||||
|
t.Errorf("meeting a dependency said %v", lines)
|
||||||
|
}
|
||||||
|
// Taking the dependent off says nothing: the dependency went with its module.
|
||||||
|
if lines := unheldChange(shelf, "laptop", []string{"web"}, nil); len(lines) != 0 {
|
||||||
|
t.Errorf("unassigning the dependent said %v", lines)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPushSaysANamedNodesDependenciesAndCountsTheRest(t *testing.T) {
|
||||||
|
unheld := map[string][]catalogue.Unheld{
|
||||||
|
"anchor": {{Node: "anchor", Module: "db", Seat: catalogue.ContainerRuntimeSeat}},
|
||||||
|
"laptop": {{Node: "laptop", Module: "web", Seat: catalogue.ContainerRuntimeSeat},
|
||||||
|
{Node: "laptop", Module: "sshd", Seat: catalogue.ServiceManagerSeat}},
|
||||||
|
}
|
||||||
|
var named bytes.Buffer
|
||||||
|
reportUnheldPushed(&named, true, []string{"laptop"}, unheld)
|
||||||
|
got := named.String()
|
||||||
|
if !strings.Contains(got, "laptop has 2 unmet") || !strings.Contains(got, "web on laptop") ||
|
||||||
|
!strings.Contains(got, "sshd on laptop") || strings.Contains(got, "anchor") {
|
||||||
|
t.Errorf("a named push said:\n%s", got)
|
||||||
|
}
|
||||||
|
var all bytes.Buffer
|
||||||
|
reportUnheldPushed(&all, false, []string{"anchor", "laptop", "quiet"}, unheld)
|
||||||
|
want := "anchor: 1 unmet seat dependenc(ies) — see `status`\nlaptop: 2 unmet seat dependenc(ies) — see `status`\n"
|
||||||
|
if all.String() != want {
|
||||||
|
t.Errorf("a push to every node said:\n%s\nwant\n%s", all.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyTheServingControllerLogsEachChange(t *testing.T) {
|
||||||
|
read := func(f func()) string {
|
||||||
|
old := os.Stderr
|
||||||
|
r, w, _ := os.Pipe()
|
||||||
|
os.Stderr = w
|
||||||
|
f()
|
||||||
|
_ = w.Close()
|
||||||
|
os.Stderr = old
|
||||||
|
var b bytes.Buffer
|
||||||
|
_, _ = b.ReadFrom(r)
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
u := []catalogue.Unheld{{Node: "n1", Module: "web", Seat: catalogue.ContainerRuntimeSeat}}
|
||||||
|
if got := read(func() { logUnheld("n1", u) }); got != "" {
|
||||||
|
t.Errorf("a command logged:\n%s", got)
|
||||||
|
}
|
||||||
|
logUnheldChanges = true
|
||||||
|
defer func() { logUnheldChanges = false }()
|
||||||
|
if got := read(func() { logUnheld("n1", u) }); !strings.Contains(got, "web on n1") {
|
||||||
|
t.Errorf("the serving controller did not log a change:\n%s", got)
|
||||||
|
}
|
||||||
|
if got := read(func() { logUnheld("n1", u) }); got != "" {
|
||||||
|
t.Errorf("an unchanged report was logged again:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
+195
-38
@@ -5,8 +5,10 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
@@ -55,10 +57,26 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A plan that holds the module rolls it out, and this does not** (novox/hq issue 249, ADR
|
||||||
|
// 0218). A merge's plan builds the module and sends it one machine first, the rest once that one
|
||||||
|
// has applied it; this announcement arrives as the build registers, and sending here too — one
|
||||||
|
// machine after another without waiting for any to apply — put the new bundle on every machine in
|
||||||
|
// the same minute, whatever the plan was waiting for. A move no plan answers (a build asked by
|
||||||
|
// hand) is still this handler's.
|
||||||
|
if plans, err := inv.OpenPlans(ctx); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
} else if id := rolledOutByAPlan(plans, u.Module); id != "" {
|
||||||
|
// Said with its remedy: a plan that ends without sending it — failed, or closed by hand — leaves
|
||||||
|
// these machines behind, which `status` lists and `push --behind` sends (novox/hq issue 249).
|
||||||
|
fmt.Printf("%s moved to %s; %s rolls it out to %s — if that plan ends without sending it, "+
|
||||||
|
"`status` lists them as behind and `push --behind` sends it\n",
|
||||||
|
u.Module, shortCommit(u.Commit), id, readableList(on))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if decision.Together {
|
if decision.Together {
|
||||||
fmt.Printf("%s moved to %s; sending %s together\n",
|
fmt.Printf("%s moved to %s; sending %s together\n",
|
||||||
u.Module, shortCommit(u.Commit), readableList(on))
|
u.Module, shortCommit(u.Commit), readableList(on))
|
||||||
return sendTo(ctx, f.open, on)
|
return askAgainOnGrants(sendTo(ctx, f.open, on))
|
||||||
}
|
}
|
||||||
// One at a time, and stopping at the first that fails.
|
// One at a time, and stopping at the first that fails.
|
||||||
//
|
//
|
||||||
@@ -69,13 +87,34 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
|||||||
u.Module, shortCommit(u.Commit), readableList(on))
|
u.Module, shortCommit(u.Commit), readableList(on))
|
||||||
for _, node := range on {
|
for _, node := range on {
|
||||||
if err := sendTo(ctx, f.open, []string{node}); err != nil {
|
if err := sendTo(ctx, f.open, []string{node}); err != nil {
|
||||||
return fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
|
return askAgainOnGrants(fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
|
||||||
node, u.Module, err)
|
node, u.Module, err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249):
|
||||||
|
// an announcement handled by a send whose memberships could not be issued is held and redelivered,
|
||||||
|
// rather than taken as handled with the machines left on the old version.
|
||||||
|
func askAgainOnGrants(err error) error {
|
||||||
|
if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) {
|
||||||
|
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// rolledOutByAPlan is the open plan that will send a module's machines its new build — one holding
|
||||||
|
// the module that has not finished sending it — or empty when none will (novox/hq issue 249).
|
||||||
|
func rolledOutByAPlan(plans []inventory.Plan, module string) string {
|
||||||
|
for _, p := range plans {
|
||||||
|
if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || (s.SentAt == nil && s.State != "failed")) {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// readableList names machines the way a sentence does, because this is read by a person deciding
|
// readableList names machines the way a sentence does, because this is read by a person deciding
|
||||||
// whether an upgrade went where they expected.
|
// whether an upgrade went where they expected.
|
||||||
func readableList(names []string) string {
|
func readableList(names []string) string {
|
||||||
@@ -228,6 +267,11 @@ func notNow(err error) error {
|
|||||||
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
||||||
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
||||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||||
|
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
||||||
|
// what the bus did not hand over (novox/hq issue 266). Each judges against what the other wrote.
|
||||||
|
actingOnMerges.Lock()
|
||||||
|
defer actingOnMerges.Unlock()
|
||||||
|
|
||||||
inv := f.open.inventory
|
inv := f.open.inventory
|
||||||
entries, err := inv.Catalogued(ctx)
|
entries, err := inv.Catalogued(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -238,34 +282,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
return notNow(err)
|
return notNow(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Two kinds of module are affected by one merge, and they are affected differently.
|
from, packaging, already := mergeCandidates(m, entries, read)
|
||||||
//
|
|
||||||
// A module **built from** this repository and branch has moved: the mesh records the new commit
|
|
||||||
// as what its source now has, and only what the merge actually changed is rebuilt. A module that
|
|
||||||
// only **packages source from** it has not moved — its own source is somewhere else, at the
|
|
||||||
// commit it already records — so it is rebuilt and its record left alone. Writing this commit as
|
|
||||||
// its source would make it permanently behind a repository its manifest does not come from.
|
|
||||||
var from, packaging []inventory.Entry
|
|
||||||
already := 0
|
|
||||||
for _, e := range entries {
|
|
||||||
switch {
|
|
||||||
case sourceIs(e.Source, m):
|
|
||||||
if e.Source.BuiltFrom == m.Commit {
|
|
||||||
already++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// **A merge older than the last look at the source is history, not a move.** The forge
|
|
||||||
// announces what it finds merged, and an old merge surfacing late would otherwise move
|
|
||||||
// the recorded head backwards and rebuild everything built from that repository, once
|
|
||||||
// per old merge (2026-09-28).
|
|
||||||
if isHistory(m.MergedAt, e.Source.Seen) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
from = append(from, e)
|
|
||||||
case readsFrom(read[e.Manifest.Module], m):
|
|
||||||
packaging = append(packaging, e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(from) == 0 && len(packaging) == 0 {
|
if len(from) == 0 && len(packaging) == 0 {
|
||||||
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
||||||
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
|
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
|
||||||
@@ -323,6 +340,45 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
plan := planOfMerge(m, movedNames, edges)
|
plan := planOfMerge(m, movedNames, edges)
|
||||||
|
// **A newer plan supersedes the older open plans of this repository and branch** (novox/hq issue
|
||||||
|
// 254, ADR 0218): what they had not built is planned here again, and they are closed, so one plan
|
||||||
|
// works a repository's modules at a time and a stuck one ends at the next merge.
|
||||||
|
working, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
rollsOut := func(module string) bool {
|
||||||
|
u, err := inv.UpgradeOf(ctx, module)
|
||||||
|
return err == nil && u.RollOut
|
||||||
|
}
|
||||||
|
folded, superseded := supersededBy(plan, working, rollsOut)
|
||||||
|
if len(folded) > 0 {
|
||||||
|
held := map[string]bool{}
|
||||||
|
for _, e := range entries {
|
||||||
|
held[e.Manifest.Module] = true
|
||||||
|
}
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, name := range movedNames {
|
||||||
|
names[name] = true
|
||||||
|
}
|
||||||
|
var also []string
|
||||||
|
for _, name := range folded {
|
||||||
|
// One the catalogue no longer holds would fail the newer plan's ask; it is not this
|
||||||
|
// merge's to build.
|
||||||
|
if held[name] && !names[name] {
|
||||||
|
names[name] = true
|
||||||
|
movedNames = append(movedNames, name)
|
||||||
|
also = append(also, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(also) > 0 {
|
||||||
|
again := planOfMerge(m, movedNames, edges)
|
||||||
|
again.ID, again.Created = plan.ID, plan.Created
|
||||||
|
plan = again
|
||||||
|
fmt.Printf(" %s, left unbuilt by an older plan of %s, are planned here again\n",
|
||||||
|
strings.Join(also, ", "), plan.Repository)
|
||||||
|
}
|
||||||
|
}
|
||||||
if hasCycle(plan.Tiers, edges) {
|
if hasCycle(plan.Tiers, edges) {
|
||||||
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
||||||
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
||||||
@@ -330,6 +386,14 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
if err := inv.SavePlan(ctx, plan); err != nil {
|
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||||
return notNow(err)
|
return notNow(err)
|
||||||
}
|
}
|
||||||
|
// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
|
||||||
|
// both open, which the next merge settles, rather than neither.
|
||||||
|
for _, old := range superseded {
|
||||||
|
if err := inv.SavePlan(ctx, old); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
|
||||||
|
}
|
||||||
var tiers []string
|
var tiers []string
|
||||||
for i, t := range plan.Tiers {
|
for i, t := range plan.Tiers {
|
||||||
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
||||||
@@ -353,6 +417,57 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||||
|
var actingOnMerges sync.Mutex
|
||||||
|
|
||||||
|
// mergeCandidates is what one merge could move, judged against what the catalogue holds.
|
||||||
|
//
|
||||||
|
// Two kinds of module are affected by one merge, and they are affected differently.
|
||||||
|
//
|
||||||
|
// A module **built from** this repository and branch has moved: the mesh records the new commit as
|
||||||
|
// what its source now has, and only what the merge actually changed is rebuilt. A module that only
|
||||||
|
// **packages source from** it has not moved — its own source is somewhere else, at the commit it
|
||||||
|
// already records — so it is rebuilt and its record left alone. Writing this commit as its source
|
||||||
|
// would make it permanently behind a repository its manifest does not come from. `already` counts
|
||||||
|
// the modules built from this repository that are already built from this very commit.
|
||||||
|
func mergeCandidates(m link.SourceMoved, entries []inventory.Entry,
|
||||||
|
read map[string][]inventory.ReadRepository) (from, packaging []inventory.Entry, already int) {
|
||||||
|
for _, e := range entries {
|
||||||
|
switch {
|
||||||
|
case sourceIs(e.Source, m):
|
||||||
|
if e.Source.BuiltFrom == m.Commit {
|
||||||
|
already++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **A merge older than the last look at the source is history, not a move.** The forge
|
||||||
|
// announces what it finds merged, and an old merge surfacing late would otherwise move
|
||||||
|
// the recorded head backwards and rebuild everything built from that repository, once
|
||||||
|
// per old merge (2026-09-28).
|
||||||
|
if isHistory(m.MergedAt, e.Source.Seen) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
from = append(from, e)
|
||||||
|
case readsFrom(read[e.Manifest.Module], m):
|
||||||
|
packaging = append(packaging, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return from, packaging, already
|
||||||
|
}
|
||||||
|
|
||||||
|
// wouldMove is the modules built from the merged repository that acting on this merge would mark as
|
||||||
|
// moved and rebuild — SourceMoved's judgement, made without acting (novox/hq issue 266). Empty for a
|
||||||
|
// merge already acted on: acting marks each of them as looked at, so the merge then reads as history.
|
||||||
|
//
|
||||||
|
// **Only the modules built from it, never the ones that merely package source from it.** Acting
|
||||||
|
// records nothing about those, so a merge acted on would go on reading as unacted for them, and be
|
||||||
|
// acted on again on every look. A merge that moves both is caught by the first kind, and acting on it
|
||||||
|
// rebuilds the second as well.
|
||||||
|
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
|
||||||
|
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||||
|
from, _, _ := mergeCandidates(m, entries, read)
|
||||||
|
return whatTheMergeTouched(from, entries, m)
|
||||||
|
}
|
||||||
|
|
||||||
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
|
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
|
||||||
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
|
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
|
||||||
// An empty recorded ref is the repository's default branch, which is what a merge into the base
|
// An empty recorded ref is the repository's default branch, which is what a merge into the base
|
||||||
@@ -423,25 +538,45 @@ func lastLookAt(entries []inventory.Entry, m link.SourceMoved) time.Time {
|
|||||||
//
|
//
|
||||||
// A change inside *another* module's directory is that module's business and not this one's, even
|
// A change inside *another* module's directory is that module's business and not this one's, even
|
||||||
// when the mesh does not hold that module: `known` is every module this repository is known to hold,
|
// when the mesh does not hold that module: `known` is every module this repository is known to hold,
|
||||||
// whatever branch it was registered from. That is also the limit of this — a repository whose shared
|
// whatever branch it was registered from.
|
||||||
// code sits inside a directory the mesh has never seen a module in reads as shared, and everything
|
//
|
||||||
// is rebuilt. Rebuilding too much is the safe direction: the fault this whole path exists for is a
|
// **And a module the mesh has never seen is still a module** (novox/hq issue 252). A merge adding a
|
||||||
// mesh that believes it is current and is not (novox/hq 04-ISSUES/131).
|
// new module to the catalogue repository — `modules/newmod/module.json` and its files — read as a
|
||||||
|
// change to shared code, because `modules/newmod` was nobody's known directory, and every module
|
||||||
|
// built from the repository was rebuilt and rolled out for a module none of them is. So the
|
||||||
|
// directories that hold modules are known too: the parents of the known modules' directories
|
||||||
|
// (`modules`, never the root). A changed path `<parent>/<name>/…` belongs to the module at
|
||||||
|
// `<parent>/<name>` — held or not — and rebuilds nothing else, **provided it is shown to be a
|
||||||
|
// module**: its `module.json` is among the changed files (added, changed, or removed with it). A
|
||||||
|
// directory under the same parent whose manifest the merge did not touch may as well be a shared
|
||||||
|
// library (`modules/lib`), and that is still read as shared. Rebuilding too much remains the safe
|
||||||
|
// direction: the fault this whole path exists for is a mesh that believes it is current and is not
|
||||||
|
// (novox/hq 04-ISSUES/131). A file at the root, or directly in a parent, is shared as it always was.
|
||||||
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry {
|
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry {
|
||||||
// Nothing said about the files, or not all of them said: everything built from it is affected.
|
// Nothing said about the files, or not all of them said: everything built from it is affected.
|
||||||
if len(m.Paths) == 0 || m.PathsTruncated {
|
if len(m.Paths) == 0 || m.PathsTruncated {
|
||||||
return candidates
|
return candidates
|
||||||
}
|
}
|
||||||
var dirs []string
|
var dirs []string
|
||||||
|
parents := map[string]bool{}
|
||||||
for _, e := range known {
|
for _, e := range known {
|
||||||
if e.Source.Path != "" && sameRepository(e.Source.Repository, m) {
|
if e.Source.Path != "" && sameRepository(e.Source.Repository, m) {
|
||||||
dirs = append(dirs, e.Source.Path)
|
dir := strings.Trim(e.Source.Path, "/")
|
||||||
|
dirs = append(dirs, dir)
|
||||||
|
if parent := path.Dir(dir); parent != "." && parent != "/" {
|
||||||
|
parents[parent] = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
changed := map[string]bool{}
|
||||||
for _, p := range m.Paths {
|
for _, p := range m.Paths {
|
||||||
if !insideAny(p, dirs) {
|
changed[strings.TrimPrefix(p, "/")] = true
|
||||||
return candidates
|
}
|
||||||
|
for _, p := range m.Paths {
|
||||||
|
if insideAny(p, dirs) || inAModuleOfItsOwn(p, parents, changed) {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
return candidates
|
||||||
}
|
}
|
||||||
var out []inventory.Entry
|
var out []inventory.Entry
|
||||||
for _, e := range candidates {
|
for _, e := range candidates {
|
||||||
@@ -452,6 +587,28 @@ func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// inAModuleOfItsOwn is whether a changed file is inside a module directory the mesh does not know —
|
||||||
|
// `<parent>/<name>/…` under a directory known to hold modules, whose `module.json` the same merge
|
||||||
|
// changed (novox/hq issue 252). Such a file is that module's business and nobody else's.
|
||||||
|
func inAModuleOfItsOwn(p string, parents, changed map[string]bool) bool {
|
||||||
|
p = strings.TrimPrefix(p, "/")
|
||||||
|
for parent := range parents {
|
||||||
|
rest, under := strings.CutPrefix(p, parent+"/")
|
||||||
|
if !under {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name, _, inADirectory := strings.Cut(rest, "/")
|
||||||
|
if !inADirectory || name == "" {
|
||||||
|
// A file directly in the parent — `modules/README.md` — is about all of them.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if changed[parent+"/"+name+"/module.json"] {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// inside is whether a changed file is in a directory: that directory itself, or under it.
|
// inside is whether a changed file is in a directory: that directory itself, or under it.
|
||||||
func inside(path, dir string) bool {
|
func inside(path, dir string) bool {
|
||||||
dir = strings.Trim(dir, "/")
|
dir = strings.Trim(dir, "/")
|
||||||
|
|||||||
@@ -0,0 +1,549 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The watchdogs (novox/hq to-be 45 §3): every row of the signals table, run over what the serving
|
||||||
|
// controller knows, every half minute.
|
||||||
|
//
|
||||||
|
// **One loop, one gathering, every row.** The facts are gathered once a tick — the store's machines,
|
||||||
|
// plans and durations, the bus's queue and consumers, what this process heard — and each row's watch
|
||||||
|
// is a pure reading of them, which is what lets the test generated from the table suppress a signal by
|
||||||
|
// changing a fact. A part that cannot be gathered makes the rows that read it blind: each says so as
|
||||||
|
// a condition of its own (`probe.<row>.failed`) and keeps what it raised before, because a watchdog
|
||||||
|
// that cannot see must not read as one that sees nothing wrong (ADR 0227 rule 4).
|
||||||
|
//
|
||||||
|
// **The watchdogs are themselves watched.** Each tick is recorded; the self-check (doctor.go) fails
|
||||||
|
// its probe DW when the ticks stop, and the watchdogs raise S10 when the self-check stops — two loops
|
||||||
|
// watching each other, and mesh-watcher on a second machine watching the self-check's heartbeat for
|
||||||
|
// the case both stop with the process.
|
||||||
|
|
||||||
|
// watchEvery is how often the watchdogs run.
|
||||||
|
var watchEvery = 30 * time.Second
|
||||||
|
|
||||||
|
// signalFacts is what one tick of the watchdogs reads.
|
||||||
|
type signalFacts struct {
|
||||||
|
now time.Time
|
||||||
|
started time.Time
|
||||||
|
// host is the machine this controller runs on, as the mesh names it, for a condition about itself.
|
||||||
|
host string
|
||||||
|
|
||||||
|
machines []machineFacts
|
||||||
|
machinesErr error
|
||||||
|
// toolsHeardFrom is when this process began hearing node tools: one not heard since is silent
|
||||||
|
// since then at the most.
|
||||||
|
toolsHeardFrom time.Time
|
||||||
|
|
||||||
|
plans []planFacts
|
||||||
|
plansErr error
|
||||||
|
|
||||||
|
loop loopFacts
|
||||||
|
loopErr error
|
||||||
|
|
||||||
|
merges []missedMerge
|
||||||
|
mergesPassed time.Time
|
||||||
|
mergesErr error
|
||||||
|
|
||||||
|
asks []askFacts
|
||||||
|
asksErr error
|
||||||
|
|
||||||
|
calls []link.Call
|
||||||
|
|
||||||
|
standings []conditions.Condition
|
||||||
|
standingsErr error
|
||||||
|
|
||||||
|
advisories []link.Advisory
|
||||||
|
lostConsumers map[string]bool
|
||||||
|
advisoriesErr error
|
||||||
|
|
||||||
|
selfCheck selfCheckFacts
|
||||||
|
|
||||||
|
staleRefusals map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
type machineFacts struct {
|
||||||
|
name string
|
||||||
|
control bool
|
||||||
|
// lastHeard is the store's last word from it, any word; zero when never.
|
||||||
|
lastHeard time.Time
|
||||||
|
// every is the heartbeat interval it said; zero when it said none.
|
||||||
|
every time.Duration
|
||||||
|
power link.PowerState
|
||||||
|
// sentAt is when it was last sent a declaration; reportedCurrent whether it has reported that one.
|
||||||
|
sentAt time.Time
|
||||||
|
reportedCurrent bool
|
||||||
|
reportedAt time.Time
|
||||||
|
// lastApply is its newest measured apply.
|
||||||
|
lastApply time.Duration
|
||||||
|
// tools is whether node-tools is assigned there; toolsHeard and toolsEvery its heartbeat.
|
||||||
|
tools bool
|
||||||
|
toolsHeard time.Time
|
||||||
|
toolsEvery time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||||
|
func (m machineFacts) asleep() bool { return m.power.Away() }
|
||||||
|
|
||||||
|
type planFacts struct {
|
||||||
|
id, repository, commit string
|
||||||
|
tier, tiers int
|
||||||
|
entered time.Time
|
||||||
|
bound time.Duration
|
||||||
|
waiting string
|
||||||
|
paused bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type loopFacts struct {
|
||||||
|
took time.Time
|
||||||
|
pending uint64
|
||||||
|
where string
|
||||||
|
}
|
||||||
|
|
||||||
|
type askFacts struct {
|
||||||
|
id, seat, what, state, on string
|
||||||
|
since time.Time
|
||||||
|
bound time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
type selfCheckFacts struct {
|
||||||
|
last time.Time
|
||||||
|
every time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchdogs is the loop and what it needs.
|
||||||
|
type watchdogs struct {
|
||||||
|
open *stores
|
||||||
|
server *link.Server
|
||||||
|
js *broker.JetStream
|
||||||
|
keeper *conditions.Keeper
|
||||||
|
doctor *doctor
|
||||||
|
|
||||||
|
started time.Time
|
||||||
|
// acting says this controller is the one acting, not one standing by (link.Holding): a controller
|
||||||
|
// standing by hears no heartbeat and would call every machine silent.
|
||||||
|
acting func() bool
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
ticked time.Time
|
||||||
|
last *signalFacts
|
||||||
|
failed string
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastTick is when the watchdogs last finished a tick.
|
||||||
|
func (w *watchdogs) lastTick() time.Time {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.ticked
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastFacts is the facts of the newest tick, for `doctor signals`; nil before the first.
|
||||||
|
func (w *watchdogs) lastFacts() *signalFacts {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.last
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep runs the watchdogs until ctx ends.
|
||||||
|
func (w *watchdogs) keep(ctx context.Context) {
|
||||||
|
tick := time.NewTicker(watchEvery)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
w.tick(ctx)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tick is one run of every row.
|
||||||
|
func (w *watchdogs) tick(ctx context.Context) {
|
||||||
|
if w.acting != nil && !w.acting() {
|
||||||
|
// Standing by: nothing seen, nothing said, and the tick counted — this process's self-check
|
||||||
|
// is not the one that matters while another acts.
|
||||||
|
w.mu.Lock()
|
||||||
|
w.ticked = time.Now()
|
||||||
|
w.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
running, cancel := context.WithTimeout(ctx, watchEvery)
|
||||||
|
defer cancel()
|
||||||
|
w.see(running, w.gather(running))
|
||||||
|
}
|
||||||
|
|
||||||
|
// see runs every watched row over one gathering, keeps what each found, and records the tick.
|
||||||
|
func (w *watchdogs) see(running context.Context, f *signalFacts) {
|
||||||
|
var problems []string
|
||||||
|
var blind []conditions.Observation
|
||||||
|
for _, row := range watchedRows() {
|
||||||
|
if err := row.needs(f); err != nil {
|
||||||
|
blind = append(blind, blindRow(row, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := w.keeper.Reconcile(running, row.Row, row.watch(f)); err != nil {
|
||||||
|
problems = append(problems, row.Row+": "+err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The rows that could not see, said; the ones that see again, cleared.
|
||||||
|
if err := w.keeper.Reconcile(running, sourceWatchdogs, blind); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
// A provider no longer assigned where it ran: its standing is resolved by the assignment.
|
||||||
|
if f.standingsErr == nil && w.open != nil {
|
||||||
|
if err := unassignedProviders(running, w.open.inventory, w.keeper, f.standings); err != nil {
|
||||||
|
problems = append(problems, "S8: "+err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := w.keeper.EndSilences(running); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
failed := ""
|
||||||
|
if len(problems) > 0 {
|
||||||
|
failed = fmt.Sprintf("%v", problems)
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
said := w.failed
|
||||||
|
w.ticked, w.last, w.failed = time.Now(), f, failed
|
||||||
|
w.mu.Unlock()
|
||||||
|
if failed != said {
|
||||||
|
if failed != "" {
|
||||||
|
fmt.Printf("the watchdogs could not keep what they saw: %s\n", failed)
|
||||||
|
} else if said != "" {
|
||||||
|
fmt.Println("the watchdogs keep what they see again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceWatchdogs is what raises a blind row's condition.
|
||||||
|
const sourceWatchdogs = "watchdogs"
|
||||||
|
|
||||||
|
// blindRow is a row whose facts could not be gathered, as a condition of its own.
|
||||||
|
func blindRow(row signalRow, err error) conditions.Observation {
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeProbe, ID: row.Row, Kind: "probe-failed", Token: "failed",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the watchdog of %s (%s) cannot see: what it reads could not be read, so nothing "+
|
||||||
|
"it would raise can be — and nothing it raised before is cleared", row.Row, row.Signal),
|
||||||
|
Said: firstLine(err.Error())}
|
||||||
|
}
|
||||||
|
|
||||||
|
// gather reads every fact a tick needs. Each part's failure is kept beside it, never an empty part.
|
||||||
|
func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||||
|
now := time.Now()
|
||||||
|
f := &signalFacts{now: now, started: w.started, toolsHeardFrom: link.ToolsBeats.Started(), calls: link.Calls.Running(),
|
||||||
|
staleRefusals: link.StaleRefusals.Within(now.Add(-staleRefusalsWithin)), lostConsumers: map[string]bool{}}
|
||||||
|
if w.doctor != nil {
|
||||||
|
f.selfCheck = selfCheckFacts{last: w.doctor.lastRunEnded(), every: doctorEvery}
|
||||||
|
}
|
||||||
|
inv := w.open.inventory
|
||||||
|
f.host = controlHost(ctx, inv)
|
||||||
|
f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now)
|
||||||
|
f.plans, f.plansErr = gatherPlans(ctx, inv, now)
|
||||||
|
f.loop, f.loopErr = w.gatherLoop()
|
||||||
|
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
||||||
|
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
||||||
|
f.mergesErr = fmt.Errorf("the catch-up of merges has not passed since %s", f.mergesPassed.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||||
|
if open, err := w.keeper.Open(ctx); err != nil {
|
||||||
|
f.standingsErr = err
|
||||||
|
} else {
|
||||||
|
f.standings = providerStandings(open)
|
||||||
|
}
|
||||||
|
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||||
|
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// controlHost is the machine running the controller, as the mesh names it: the one the controller
|
||||||
|
// module is assigned to, or this process's host name where that is not one machine.
|
||||||
|
func controlHost(ctx context.Context, inv *inventory.Inventory) string {
|
||||||
|
if on, err := inv.Running(ctx, "mesh-controller"); err == nil && len(on) == 1 {
|
||||||
|
return on[0]
|
||||||
|
}
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]machineFacts, error) {
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the machines cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("what each machine last reported cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
reported := map[string]inventory.Reported{}
|
||||||
|
for _, r := range reports {
|
||||||
|
reported[r.Node] = r
|
||||||
|
}
|
||||||
|
control, err := inv.Running(ctx, "mesh-controller")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("where the controller runs cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
tooled, err := inv.Running(ctx, broker.RuntimeModule)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("where the node tools run cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured applies cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
lastApply := map[string]time.Duration{}
|
||||||
|
for _, d := range applies { // oldest first: the last one read is the newest
|
||||||
|
lastApply[d.Node] = d.Took
|
||||||
|
}
|
||||||
|
var out []machineFacts
|
||||||
|
anyPast := false
|
||||||
|
for _, n := range nodes {
|
||||||
|
m := machineFacts{name: n.Name, control: slices.Contains(control, n.Name), lastHeard: n.LastSeen,
|
||||||
|
lastApply: lastApply[n.Name], tools: slices.Contains(tooled, n.Name)}
|
||||||
|
if beat, ok := link.HostBeats.Of(n.Name); ok {
|
||||||
|
m.every = beat.Every
|
||||||
|
}
|
||||||
|
if beat, ok := link.ToolsBeats.Of(n.Name); ok {
|
||||||
|
m.toolsHeard, m.toolsEvery = beat.At, beat.Every
|
||||||
|
}
|
||||||
|
if r, ok := reported[n.Name]; ok {
|
||||||
|
if r.Sent != nil {
|
||||||
|
m.sentAt = *r.Sent
|
||||||
|
}
|
||||||
|
if r.At != nil {
|
||||||
|
m.reportedAt = *r.At
|
||||||
|
}
|
||||||
|
m.reportedCurrent = r.Current
|
||||||
|
}
|
||||||
|
if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||||
|
anyPast = true
|
||||||
|
}
|
||||||
|
if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
|
||||||
|
anyPast = true
|
||||||
|
}
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
// What a machine past its bound last said of its power, read only then: a machine that said it
|
||||||
|
// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
|
||||||
|
// which is the louder mistake and the right one to make.
|
||||||
|
if anyPast && w.server != nil {
|
||||||
|
states, err := w.server.PowerStates(ctx, now.Add(-7*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("what machines said of their power cannot be read, so a sleeping one is called silent: %v\n", err)
|
||||||
|
}
|
||||||
|
for i := range out {
|
||||||
|
out[i].power = states[out[i].name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherPlans is every open plan, its tier's bound from what was measured, and what it waits on.
|
||||||
|
func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]planFacts, error) {
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the open plans cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if len(plans) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
measured := map[string][]time.Duration{}
|
||||||
|
for _, d := range tiers {
|
||||||
|
measured[d.Subject] = append(measured[d.Subject], d.Took)
|
||||||
|
}
|
||||||
|
pause := buildSeatPause(ctx, inv, plans)
|
||||||
|
var out []planFacts
|
||||||
|
for _, p := range plans {
|
||||||
|
_, paused := pausedWaiting(p, pause, now)
|
||||||
|
out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
|
||||||
|
tiers: len(p.Tiers), entered: p.TierEntered, bound: max(tierAtLeast, 3*p90(measured[p.Repository])),
|
||||||
|
waiting: planLineWith(p, now, pause), paused: paused})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// p90 is the ninetieth percentile of measurements; zero for none.
|
||||||
|
func p90(took []time.Duration) time.Duration {
|
||||||
|
if len(took) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
sorted := append([]time.Duration(nil), took...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
|
||||||
|
return sorted[int(0.9*float64(len(sorted)-1))]
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherLoop is when the event loop last took a message and what its consumers hold.
|
||||||
|
func (w *watchdogs) gatherLoop() (loopFacts, error) {
|
||||||
|
f := loopFacts{took: link.Loop.Last()}
|
||||||
|
if w.js == nil {
|
||||||
|
return f, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
var where []string
|
||||||
|
for _, c := range broker.MeshConsumers() {
|
||||||
|
info, err := w.js.Context().ConsumerInfo(c.Stream, c.Name)
|
||||||
|
if err != nil {
|
||||||
|
return f, fmt.Errorf("the controller's consumer on %s cannot be read: %w", c.Stream, err)
|
||||||
|
}
|
||||||
|
if held := info.NumPending + uint64(info.NumAckPending); held > 0 {
|
||||||
|
f.pending += held
|
||||||
|
where = append(where, fmt.Sprintf("%d on %s", held, c.Stream))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.where = fmt.Sprint(where)
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherAsks is every ask in the build seat's queue that is in flight or dead, with its bound.
|
||||||
|
func (w *watchdogs) gatherAsks(ctx context.Context, inv *inventory.Inventory) ([]askFacts, error) {
|
||||||
|
if w.js == nil {
|
||||||
|
return nil, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
seat := buildSeatAmong(entries)
|
||||||
|
q, err := link.ReadQueue(ctx, w.js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
builds, err := inv.Durations(ctx, inventory.DurationBuild, time.Now().Add(-14*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured builds cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var took []time.Duration
|
||||||
|
for _, d := range builds {
|
||||||
|
took = append(took, d.Took)
|
||||||
|
}
|
||||||
|
bound := askDefault
|
||||||
|
if len(took) > 0 {
|
||||||
|
bound = max(askAtLeast, 3*p90(took))
|
||||||
|
}
|
||||||
|
var out []askFacts
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
if a.State != link.AskInFlight && a.State != link.AskDead {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
since := a.Started
|
||||||
|
if since.IsZero() {
|
||||||
|
since = a.AskedAt
|
||||||
|
}
|
||||||
|
what := a.Repository
|
||||||
|
if a.Path != "" {
|
||||||
|
what += " " + a.Path
|
||||||
|
}
|
||||||
|
out = append(out, askFacts{id: a.ID, seat: seat, what: what, state: a.State, on: a.On, since: since, bound: bound})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lostConsumers is, of the consumers the bus said were deleted, each that is still missing and that
|
||||||
|
// the mesh expects: a consumer removed because its module was unassigned is not lost.
|
||||||
|
func (w *watchdogs) lostConsumers(ctx context.Context, heard []link.Advisory) (map[string]bool, error) {
|
||||||
|
out := map[string]bool{}
|
||||||
|
var deleted []link.Advisory
|
||||||
|
for _, a := range heard {
|
||||||
|
if a.Kind == link.AdvisoryConsumerLost {
|
||||||
|
deleted = append(deleted, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(deleted) == 0 {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if w.js == nil {
|
||||||
|
return nil, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
_, expected, err := expectedBusObjects(ctx, w.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
want := map[string]bool{}
|
||||||
|
for _, c := range expected {
|
||||||
|
want[c.Stream+"."+c.Name] = true
|
||||||
|
}
|
||||||
|
for _, a := range deleted {
|
||||||
|
_, err := w.js.Context().ConsumerInfo(a.Stream, a.Consumer)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||||
|
out[a.Stream+"."+a.Consumer] = want[a.Stream+"."+a.Consumer]
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("whether %s exists cannot be read: %w", link.ConsumerInWords(a.Stream, a.Consumer), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// later is the later of two moments.
|
||||||
|
func later(a, b time.Time) time.Time {
|
||||||
|
if a.After(b) {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchTheMesh opens the condition store and starts the watchdogs, the bus's advisories and the
|
||||||
|
// self-check, for the serving controller; the returned function stops them. Nil when the store could
|
||||||
|
// not be opened, which is said.
|
||||||
|
func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
|
||||||
|
keeper, err := keeperOn(ctx, bus.Conn)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
|
||||||
|
"status says the conditions cannot be read: %v\n", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
conditionsFrom = keeper
|
||||||
|
logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||||
|
stopHearing, err := server.HearAdvisories(logf)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("what the bus says about itself cannot be heard (S9 is blind): %v\n", err)
|
||||||
|
stopHearing = func() {}
|
||||||
|
}
|
||||||
|
host := controlHost(ctx, open.inventory)
|
||||||
|
w := &watchdogs{open: open, server: server, js: server.JetStream(), keeper: keeper, started: time.Now(),
|
||||||
|
acting: link.Holding}
|
||||||
|
d := &doctor{open: open, js: server.JetStream(), keeper: keeper, teller: bus, watchdogs: w, host: host}
|
||||||
|
w.doctor = d
|
||||||
|
doctorFrom = d
|
||||||
|
watching, stop := context.WithCancel(ctx)
|
||||||
|
go w.keep(watching)
|
||||||
|
go d.keep(watching)
|
||||||
|
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||||
|
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||||
|
broker.ConditionsBucket, conditions.Seat)
|
||||||
|
return func() {
|
||||||
|
stop()
|
||||||
|
stopHearing()
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
keeper.Close(flushing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runnableProbes are the probes the registry runs.
|
||||||
|
func runnableProbes() []probe {
|
||||||
|
var out []probe
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
if p.run != nil {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -5,7 +5,9 @@ go 1.26.0
|
|||||||
require (
|
require (
|
||||||
github.com/jackc/pgx/v5 v5.10.0
|
github.com/jackc/pgx/v5 v5.10.0
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
|
github.com/novox/mesh-host v0.0.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
|
golang.org/x/net v0.58.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -15,8 +17,15 @@ require (
|
|||||||
github.com/klauspost/compress v1.20.0 // indirect
|
github.com/klauspost/compress v1.20.0 // indirect
|
||||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
github.com/nats-io/nuid v1.0.1 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
golang.org/x/net v0.58.0 // indirect
|
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/sys v0.48.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The node-engine's own validator (mesh-host/validate, novox/hq to-be 45 D1): one validator, the host's.
|
||||||
|
// The host's module path names no forge a build can fetch from, so the module is read from the one
|
||||||
|
// that holds it, at the host's commit — **once, by whoever moves the pin, into vendor/**, which is
|
||||||
|
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||||
|
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||||
|
// `go mod vendor` fails loudly, at once, everywhere.
|
||||||
|
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2 h1:b4+F4tTfrPkuJTST+rkwIHpEb4mkVJR9158hr6nnc4g=
|
||||||
|
git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
|||||||
@@ -0,0 +1,345 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||||
|
//
|
||||||
|
// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
|
||||||
|
// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
|
||||||
|
// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
|
||||||
|
// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
|
||||||
|
// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
|
||||||
|
// bare blob is unreferenced, so the first real collection would have deleted every archive the
|
||||||
|
// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
|
||||||
|
// collector runs `--dry-run` until this is true.
|
||||||
|
//
|
||||||
|
// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
|
||||||
|
// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
|
||||||
|
// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
|
||||||
|
// holder first, which is what lets the bytes go at the next collection.
|
||||||
|
//
|
||||||
|
// **Nothing a machine reads changes.** The recorded reference stays
|
||||||
|
// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
|
||||||
|
// before. The holder is the store's bookkeeping, not a second way to reach anything.
|
||||||
|
//
|
||||||
|
// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
|
||||||
|
// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
|
||||||
|
// compute which manifest holds any archive from the reference it already has plus one HEAD for the
|
||||||
|
// size. No schema change, no second record that could disagree with the store.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// mediaManifest is the type a holder is put and asked for as.
|
||||||
|
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
|
||||||
|
// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
|
||||||
|
// whose only purpose is to name its layer.
|
||||||
|
mediaEmpty = "application/vnd.oci.empty.v1+json"
|
||||||
|
// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
|
||||||
|
emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
|
||||||
|
// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
|
||||||
|
// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
|
||||||
|
// not read from anywhere, because the holder must be recomputable from the reference alone.
|
||||||
|
mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
|
||||||
|
)
|
||||||
|
|
||||||
|
// emptyConfig is the content emptyDigest names.
|
||||||
|
var emptyConfig = []byte("{}")
|
||||||
|
|
||||||
|
// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
|
||||||
|
// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
|
||||||
|
// (measured 2026-09-28; internal/builder/registry.go says how that was found).
|
||||||
|
var manifestAccept = []string{
|
||||||
|
mediaManifest,
|
||||||
|
"application/vnd.docker.distribution.manifest.v2+json",
|
||||||
|
}
|
||||||
|
|
||||||
|
type descriptor struct {
|
||||||
|
MediaType string `json:"mediaType"`
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type holderManifest struct {
|
||||||
|
SchemaVersion int `json:"schemaVersion"`
|
||||||
|
MediaType string `json:"mediaType"`
|
||||||
|
Config descriptor `json:"config"`
|
||||||
|
Layers []descriptor `json:"layers"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holder is the manifest that holds an archive in the store, and its digest.
|
||||||
|
//
|
||||||
|
// A pure function of the archive's digest and size: the same two in give the same bytes out,
|
||||||
|
// always, because `encoding/json` writes a struct's fields in their declared order and there is
|
||||||
|
// nothing here that varies by when or where it was composed.
|
||||||
|
func Holder(digest string, size int64) (body []byte, holder string) {
|
||||||
|
body, err := json.Marshal(holderManifest{
|
||||||
|
SchemaVersion: 2,
|
||||||
|
MediaType: mediaManifest,
|
||||||
|
Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
|
||||||
|
Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
// Marshalling a struct of strings and integers cannot fail.
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
return body, "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
|
||||||
|
//
|
||||||
|
// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
|
||||||
|
// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
|
||||||
|
// is what lets the sweep run it over every kept archive on every build and so backfill the bare
|
||||||
|
// blobs published before holders existed (novox/hq issue 253).
|
||||||
|
//
|
||||||
|
// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
|
||||||
|
// fact the caller reports rather than one this invents a remedy for.
|
||||||
|
func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
|
||||||
|
repository, digest, archive, err := s.archive(reference)
|
||||||
|
if err != nil || !archive {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
size, err := s.blobSize(ctx, repository, digest)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return s.HoldBlob(ctx, repository, digest, size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
|
||||||
|
// question an operator needs answered with "none unheld" before the collector is let loose.
|
||||||
|
//
|
||||||
|
// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
|
||||||
|
func (s Store) Held(ctx context.Context, reference string) (bool, error) {
|
||||||
|
repository, digest, archive, err := s.archive(reference)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if !archive {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
size, err := s.blobSize(ctx, repository, digest)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
_, holder := Holder(digest, size)
|
||||||
|
return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
|
||||||
|
// there already. Answers whether it wrote one.
|
||||||
|
//
|
||||||
|
// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
|
||||||
|
// the store reports. Both arrive at the same holder, which is the point of composing it.
|
||||||
|
func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
|
||||||
|
if s.Address == "" {
|
||||||
|
return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
|
||||||
|
}
|
||||||
|
body, holder := Holder(digest, size)
|
||||||
|
there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if there {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// The config must be in the repository before a manifest naming it is accepted: a registry
|
||||||
|
// refuses a manifest whose blobs it cannot find there, which is the property that makes a
|
||||||
|
// holder mean something.
|
||||||
|
if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// **By digest, never by tag.** A tag would be one more name to move and one more thing the
|
||||||
|
// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
|
||||||
|
// it pins by digest, and an untagged manifest is kept by plain collection.
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
|
||||||
|
s.url(repository, "manifests", holder), bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
request.Header.Set("Content-Type", mediaManifest)
|
||||||
|
response, err := s.client().Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusCreated {
|
||||||
|
said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||||
|
return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
|
||||||
|
repository, digest, response.Status, strings.TrimSpace(string(said)))
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
|
||||||
|
//
|
||||||
|
// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
|
||||||
|
// collector would keep its bytes for ever on the strength of it — the sweep would record the
|
||||||
|
// archive collected while the disk said otherwise. Deleting the holder first and failing before
|
||||||
|
// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
|
||||||
|
//
|
||||||
|
// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
|
||||||
|
// and without the blob there is nothing left for a holder to keep. A store that never had a holder
|
||||||
|
// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
|
||||||
|
// that is the outcome wanted.
|
||||||
|
func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
|
||||||
|
size, err := s.blobSize(ctx, repository, digest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, holder := Holder(digest, size)
|
||||||
|
err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
|
||||||
|
if err == Gone {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// archive reads a recorded reference into its repository and digest, and whether it is an archive
|
||||||
|
// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
|
||||||
|
func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
|
||||||
|
path, kept := catalogue.InArtifactStore(reference)
|
||||||
|
if !kept {
|
||||||
|
return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||||
|
}
|
||||||
|
if s.Address == "" {
|
||||||
|
return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||||
|
}
|
||||||
|
repository, kind, digest, err := split(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", false, err
|
||||||
|
}
|
||||||
|
return repository, digest, kind == "blobs", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// blobSize is how large the store says a blob is; Gone when it does not have it.
|
||||||
|
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
response, err := s.client().Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
switch response.StatusCode {
|
||||||
|
case http.StatusOK:
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return 0, Gone
|
||||||
|
default:
|
||||||
|
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||||
|
}
|
||||||
|
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
|
||||||
|
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
|
||||||
|
// it, and the header is what the registry itself wrote.
|
||||||
|
if length := response.Header.Get("Content-Length"); length != "" {
|
||||||
|
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if response.ContentLength >= 0 {
|
||||||
|
return response.ContentLength, nil
|
||||||
|
}
|
||||||
|
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
|
||||||
|
// naming the digest — the registry's own two steps, the same the builder takes for an archive.
|
||||||
|
func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
|
||||||
|
if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
|
||||||
|
return err
|
||||||
|
} else if there {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
begun, err := s.client().Do(start)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
|
||||||
|
}
|
||||||
|
begun.Body.Close()
|
||||||
|
if begun.StatusCode != http.StatusAccepted {
|
||||||
|
return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
|
||||||
|
}
|
||||||
|
where := begun.Header.Get("Location")
|
||||||
|
if where == "" {
|
||||||
|
return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(where, "/") {
|
||||||
|
where = "http://" + s.Address + where
|
||||||
|
}
|
||||||
|
separator := "?"
|
||||||
|
if strings.Contains(where, "?") {
|
||||||
|
separator = "&"
|
||||||
|
}
|
||||||
|
put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
put.Header.Set("Content-Type", "application/octet-stream")
|
||||||
|
done, err := s.client().Do(put)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer done.Body.Close()
|
||||||
|
if done.StatusCode != http.StatusCreated {
|
||||||
|
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||||
|
return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// has is whether the store answers 200 for a HEAD at that URL.
|
||||||
|
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
for _, media := range accept {
|
||||||
|
request.Header.Add("Accept", media)
|
||||||
|
}
|
||||||
|
response, err := s.client().Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
switch response.StatusCode {
|
||||||
|
case http.StatusOK:
|
||||||
|
return true, nil
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return false, nil
|
||||||
|
default:
|
||||||
|
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Store) url(repository, kind, digest string) string {
|
||||||
|
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Store) client() *http.Client {
|
||||||
|
if s.HTTP != nil {
|
||||||
|
return s.HTTP
|
||||||
|
}
|
||||||
|
return &http.Client{Timeout: 30 * time.Second}
|
||||||
|
}
|
||||||
@@ -0,0 +1,268 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||||
|
//
|
||||||
|
// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
|
||||||
|
// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
|
||||||
|
// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
|
||||||
|
// type. What is asserted is this side's decisions; the live test below asserts the registry's.
|
||||||
|
|
||||||
|
type memRegistry struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
blobs map[string][]byte // repository + "@" + digest
|
||||||
|
manifests map[string][]byte // repository + "@" + digest
|
||||||
|
writes []string // every PUT and DELETE, as "METHOD path"
|
||||||
|
}
|
||||||
|
|
||||||
|
func digestOf(body []byte) string {
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
return "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memRegistry) serve(t *testing.T) Store {
|
||||||
|
t.Helper()
|
||||||
|
m.blobs = map[string][]byte{}
|
||||||
|
m.manifests = map[string][]byte{}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
path := strings.TrimPrefix(r.URL.Path, "/v2/")
|
||||||
|
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
|
||||||
|
m.writes = append(m.writes, r.Method+" "+r.URL.Path)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
|
||||||
|
repository := strings.TrimSuffix(path, "/blobs/uploads/")
|
||||||
|
w.Header().Set("Location", "/upload/"+repository+"?state=x")
|
||||||
|
w.WriteHeader(http.StatusAccepted)
|
||||||
|
case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
|
||||||
|
repository := strings.TrimPrefix(r.URL.Path, "/upload/")
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
digest := r.URL.Query().Get("digest")
|
||||||
|
if digest != digestOf(body) {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.blobs[repository+"@"+digest] = body
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
case strings.Contains(path, "/blobs/"):
|
||||||
|
repository, digest, _ := strings.Cut(path, "/blobs/")
|
||||||
|
key := repository + "@" + digest
|
||||||
|
body, ok := m.blobs[key]
|
||||||
|
if !ok {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodHead:
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
case http.MethodDelete:
|
||||||
|
delete(m.blobs, key)
|
||||||
|
w.WriteHeader(http.StatusAccepted)
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||||
|
}
|
||||||
|
case strings.Contains(path, "/manifests/"):
|
||||||
|
repository, digest, _ := strings.Cut(path, "/manifests/")
|
||||||
|
key := repository + "@" + digest
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodHead:
|
||||||
|
if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
case http.MethodPut:
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
if digest != digestOf(body) {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var named holderManifest
|
||||||
|
if err := json.Unmarshal(body, &named); err != nil {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, d := range append([]descriptor{named.Config}, named.Layers...) {
|
||||||
|
if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.manifests[key] = body
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
case http.MethodDelete:
|
||||||
|
if _, ok := m.manifests[key]; !ok {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
delete(m.manifests, key)
|
||||||
|
w.WriteHeader(http.StatusAccepted)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
|
||||||
|
func (m *memRegistry) bare(repository string, body []byte) string {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
digest := digestOf(body)
|
||||||
|
m.blobs[repository+"@"+digest] = body
|
||||||
|
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
|
||||||
|
// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
|
||||||
|
// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
|
||||||
|
// never be mistaken for one of a different blob.
|
||||||
|
digest := "sha256:" + strings.Repeat("a", 64)
|
||||||
|
one, first := Holder(digest, 42)
|
||||||
|
two, second := Holder(digest, 42)
|
||||||
|
if !bytes.Equal(one, two) || first != second {
|
||||||
|
t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
|
||||||
|
}
|
||||||
|
if _, other := Holder(digest, 43); other == first {
|
||||||
|
t.Fatal("a different size composed the same holder")
|
||||||
|
}
|
||||||
|
want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
|
||||||
|
`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
|
||||||
|
`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
|
||||||
|
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
|
||||||
|
if string(one) != want {
|
||||||
|
t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
|
||||||
|
}
|
||||||
|
if digestOf(emptyConfig) != emptyDigest {
|
||||||
|
t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
|
||||||
|
// The archives published before this have no holder. Hold, run over every kept archive on
|
||||||
|
// every sweep, writes one the first time and nothing after.
|
||||||
|
m := &memRegistry{}
|
||||||
|
store := m.serve(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
body := []byte("a theme")
|
||||||
|
reference := m.bare("shell/config", body)
|
||||||
|
|
||||||
|
if held, err := store.Held(ctx, reference); err != nil || held {
|
||||||
|
t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
|
||||||
|
}
|
||||||
|
wrote, err := store.Hold(ctx, reference)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !wrote {
|
||||||
|
t.Fatal("holding a bare archive wrote nothing")
|
||||||
|
}
|
||||||
|
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||||
|
if _, ok := m.manifests["shell/config@"+holder]; !ok {
|
||||||
|
t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
|
||||||
|
}
|
||||||
|
if held, err := store.Held(ctx, reference); err != nil || !held {
|
||||||
|
t.Fatalf("after holding, held=%v (%v)", held, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
writes := len(m.writes)
|
||||||
|
wrote, err = store.Hold(ctx, reference)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if wrote || len(m.writes) != writes {
|
||||||
|
t.Fatalf("holding again wrote %v", m.writes[writes:])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
|
||||||
|
m := &memRegistry{}
|
||||||
|
store := m.serve(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// An image is its own manifest: nothing is asked.
|
||||||
|
wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
|
||||||
|
if err != nil || wrote || len(m.writes) != 0 {
|
||||||
|
t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
|
||||||
|
}
|
||||||
|
// An archive the store does not have is a fact to report, not something to invent a holder for.
|
||||||
|
_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
|
||||||
|
if !errors.Is(err, Gone) {
|
||||||
|
t.Fatalf("holding a missing archive answered %v, want Gone", err)
|
||||||
|
}
|
||||||
|
// And a reference that is not the mesh's is refused as such.
|
||||||
|
if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
|
||||||
|
t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
|
||||||
|
// A holder left behind would keep the bytes through every collection while the record said
|
||||||
|
// collected; the link deleted first and the holder failing after would be that exactly.
|
||||||
|
m := &memRegistry{}
|
||||||
|
store := m.serve(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
body := []byte("an old theme")
|
||||||
|
reference := m.bare("shell/config", body)
|
||||||
|
if _, err := store.Hold(ctx, reference); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m.writes = nil
|
||||||
|
|
||||||
|
if err := store.LetGo(ctx, reference); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||||
|
want := []string{
|
||||||
|
"DELETE /v2/shell/config/manifests/" + holder,
|
||||||
|
"DELETE /v2/shell/config/blobs/" + digestOf(body),
|
||||||
|
}
|
||||||
|
if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
|
||||||
|
t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
|
||||||
|
}
|
||||||
|
if len(m.manifests) != 0 {
|
||||||
|
t.Fatalf("a holder survived: %v", m.manifests)
|
||||||
|
}
|
||||||
|
// Asked again, the archive is already gone, which is the outcome wanted.
|
||||||
|
if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
|
||||||
|
t.Fatalf("letting go twice answered %v, want Gone", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
|
||||||
|
// An archive published before holders and let go of before any sweep held it: the holder's
|
||||||
|
// delete answers 404, which is the outcome wanted, and the blob still goes.
|
||||||
|
m := &memRegistry{}
|
||||||
|
store := m.serve(t)
|
||||||
|
reference := m.bare("shell/config", []byte("never held"))
|
||||||
|
if err := store.LetGo(context.Background(), reference); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(m.blobs) != 0 {
|
||||||
|
t.Fatalf("the blob survived: %v", m.blobs)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The registry's own collector keeps a held archive and takes a bare one (novox/hq issue 253,
|
||||||
|
// ADR 0189).
|
||||||
|
//
|
||||||
|
// Everything else here is asserted against a fake, which can only say what this side asks. This is
|
||||||
|
// the one question a fake cannot answer — what `registry garbage-collect` actually does with what
|
||||||
|
// this side wrote — and it is the whole of whether the store's nightly step may stop being a dry
|
||||||
|
// run. Against the very image the mesh's store runs:
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name mesh-controller-registry -p 15000:5000 \
|
||||||
|
// -e REGISTRY_STORAGE_DELETE_ENABLED=true registry:2.8.3
|
||||||
|
// MESH_TEST_REGISTRY=127.0.0.1:15000 MESH_TEST_REGISTRY_CONTAINER=mesh-controller-registry \
|
||||||
|
// go test -run Live ./internal/artifacts/
|
||||||
|
// docker stop mesh-controller-registry
|
||||||
|
//
|
||||||
|
// Skipped without both variables: it needs a registry it may write to and collect, and a container
|
||||||
|
// to run the collector in.
|
||||||
|
func TestLiveTheRegistrysCollectorKeepsWhatIsHeldAndTakesWhatIsNot(t *testing.T) {
|
||||||
|
address := os.Getenv("MESH_TEST_REGISTRY")
|
||||||
|
container := os.Getenv("MESH_TEST_REGISTRY_CONTAINER")
|
||||||
|
if address == "" || container == "" {
|
||||||
|
t.Skip("no MESH_TEST_REGISTRY / MESH_TEST_REGISTRY_CONTAINER; see this test's comment for the registry to raise")
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
store := Store{Address: address}
|
||||||
|
run := time.Now().UnixNano()
|
||||||
|
|
||||||
|
// Four archives in four repositories, each a different story. Distinct bytes per run, so a
|
||||||
|
// registry reused across runs cannot answer for an earlier one.
|
||||||
|
put := func(name string) (repository, digest string, body []byte) {
|
||||||
|
repository = fmt.Sprintf("live-%d/%s", run, name)
|
||||||
|
body = []byte(fmt.Sprintf("%s archive of run %d", name, run))
|
||||||
|
digest = digestOf(body)
|
||||||
|
if err := store.putBlob(ctx, repository, digest, body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return repository, digest, body
|
||||||
|
}
|
||||||
|
reference := func(repository, digest string) string {
|
||||||
|
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||||
|
}
|
||||||
|
|
||||||
|
// Published held — what PublishArchive now does.
|
||||||
|
heldRepo, heldDigest, heldBody := put("held")
|
||||||
|
if _, err := store.HoldBlob(ctx, heldRepo, heldDigest, int64(len(heldBody))); err != nil {
|
||||||
|
t.Fatalf("the registry refused a holder: %v", err)
|
||||||
|
}
|
||||||
|
// Published bare, as before, and never held: what the collector must take.
|
||||||
|
_, bareDigest, _ := put("bare")
|
||||||
|
// Published bare and then held by the sweep: the backfill.
|
||||||
|
backRepo, backDigest, backBody := put("backfilled")
|
||||||
|
if wrote, err := store.Hold(ctx, reference(backRepo, backDigest)); err != nil || !wrote {
|
||||||
|
t.Fatalf("backfilling wrote=%v: %v", wrote, err)
|
||||||
|
}
|
||||||
|
if held, err := store.Held(ctx, reference(backRepo, backDigest)); err != nil || !held {
|
||||||
|
t.Fatalf("after backfilling, held=%v: %v", held, err)
|
||||||
|
}
|
||||||
|
// Held, and then let go of by the sweep: holder first, then the link.
|
||||||
|
goneRepo, goneDigest, _ := put("let-go")
|
||||||
|
if _, err := store.Hold(ctx, reference(goneRepo, goneDigest)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.LetGo(ctx, reference(goneRepo, goneDigest)); err != nil {
|
||||||
|
t.Fatalf("letting go of a held archive: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
collected, err := exec.CommandContext(ctx, "docker", "exec", container,
|
||||||
|
"registry", "garbage-collect", "/etc/docker/registry/config.yml").CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the collector failed: %v\n%s", err, collected)
|
||||||
|
}
|
||||||
|
t.Logf("the collector said:\n%s", lastLines(string(collected), 12))
|
||||||
|
|
||||||
|
// What is asserted is the bytes on the store's disk, not what the running server answers: the
|
||||||
|
// server caches blob descriptors in memory and can answer for a blob the collector removed.
|
||||||
|
onDisk := func(digest string) bool {
|
||||||
|
hex := strings.TrimPrefix(digest, "sha256:")
|
||||||
|
path := "/var/lib/registry/docker/registry/v2/blobs/sha256/" + hex[:2] + "/" + hex + "/data"
|
||||||
|
return exec.CommandContext(ctx, "docker", "exec", container, "test", "-f", path).Run() == nil
|
||||||
|
}
|
||||||
|
if !onDisk(heldDigest) {
|
||||||
|
t.Error("the collector took an archive published held")
|
||||||
|
}
|
||||||
|
if !onDisk(backDigest) {
|
||||||
|
t.Error("the collector took an archive the sweep backfilled a holder for")
|
||||||
|
}
|
||||||
|
if onDisk(bareDigest) {
|
||||||
|
t.Error("the collector kept a bare archive — then the holders prove nothing, and this test is wrong")
|
||||||
|
}
|
||||||
|
if onDisk(goneDigest) {
|
||||||
|
t.Error("the collector kept an archive the sweep let go of: its holder outlived its link")
|
||||||
|
}
|
||||||
|
// And what survived is still fetched exactly as machines fetch it: the blob, by digest.
|
||||||
|
for repository, want := range map[string][]byte{heldRepo: heldBody, backRepo: backBody} {
|
||||||
|
digest := digestOf(want)
|
||||||
|
response, err := http.Get(catalogue.Routed(reference(repository, digest), address))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _ := io.ReadAll(response.Body)
|
||||||
|
response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusOK || !bytes.Equal(got, want) {
|
||||||
|
t.Errorf("%s answered %s with %q after collection", repository, response.Status, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func lastLines(s string, n int) string {
|
||||||
|
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||||
|
if len(lines) > n {
|
||||||
|
lines = lines[len(lines)-n:]
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
// Package artifacts speaks to the mesh's artifact store over its own door.
|
||||||
|
//
|
||||||
|
// Only what the mesh needs that nothing else does: letting go of something it put there
|
||||||
|
// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
|
||||||
|
// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
|
||||||
|
// is every machine's, through its runtime. This is the one operation that belongs to the thing
|
||||||
|
// holding the records, because it is the only one that is a decision rather than a transfer.
|
||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Store is the artifact store at an address, as this machine reaches it.
|
||||||
|
type Store struct {
|
||||||
|
// Address is `host:port` — the store as the caller reaches it now, composed and never
|
||||||
|
// recorded (novox/hq 04-ISSUES/102).
|
||||||
|
Address string
|
||||||
|
// HTTP is the client used; nil is a client with a modest timeout.
|
||||||
|
HTTP *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
|
||||||
|
var Gone = errors.New("the store does not hold it")
|
||||||
|
|
||||||
|
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing
|
||||||
|
// the store holds by digest.
|
||||||
|
//
|
||||||
|
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite
|
||||||
|
// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an
|
||||||
|
// error" is how a cautious loop became one that did nothing while reporting the right number.
|
||||||
|
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||||
|
|
||||||
|
// LetGo asks the store to drop one artifact the mesh recorded making.
|
||||||
|
//
|
||||||
|
// Takes a reference as the mesh records it — `artifact-store://<module>/<artifact>@sha256:…` for
|
||||||
|
// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
|
||||||
|
// and composes the address here at the moment of use.
|
||||||
|
//
|
||||||
|
// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
|
||||||
|
// issue 253); an image in one.
|
||||||
|
//
|
||||||
|
// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
|
||||||
|
// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
|
||||||
|
// which of the two happened.
|
||||||
|
func (s Store) LetGo(ctx context.Context, reference string) error {
|
||||||
|
// **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its
|
||||||
|
// own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…`
|
||||||
|
// as the mesh's too — it cannot tell one registry host from another — so normalising belongs
|
||||||
|
// where the provenance is known, which is the sweep reading its own build records, not here
|
||||||
|
// where the only job is to refuse anything that is not plainly ours.
|
||||||
|
path, kept := catalogue.InArtifactStore(reference)
|
||||||
|
if !kept {
|
||||||
|
// Nothing the mesh put in its own store. Refused rather than attempted: composing a
|
||||||
|
// delete for a reference of unknown shape is how a sweep reaches something that is not
|
||||||
|
// the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on.
|
||||||
|
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||||
|
}
|
||||||
|
if s.Address == "" {
|
||||||
|
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||||
|
}
|
||||||
|
repository, kind, digest, err := split(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if kind == "blobs" {
|
||||||
|
// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
|
||||||
|
// naming the blob would keep its bytes through every collection while the record said
|
||||||
|
// collected. Gone here means the store has no such blob, so there is nothing to let go.
|
||||||
|
if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s.remove(ctx, s.url(repository, kind, digest), reference)
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove asks the store to delete what is at url. Gone when it has no such thing.
|
||||||
|
func (s Store) remove(ctx context.Context, url, what string) error {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
response, err := s.client().Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
switch response.StatusCode {
|
||||||
|
case http.StatusAccepted, http.StatusOK, http.StatusNoContent:
|
||||||
|
return nil
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return Gone
|
||||||
|
case http.StatusMethodNotAllowed:
|
||||||
|
// The registry was started without deletion enabled. Said plainly, because the remedy is
|
||||||
|
// a setting on the store's module and not anything about this artifact.
|
||||||
|
return fmt.Errorf(
|
||||||
|
"the artifact store refuses deletion: its server was started without it enabled "+
|
||||||
|
"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
|
||||||
|
"module is applied again (novox/hq ADR 0189). Asking about %s", what)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// split reads a recorded path into the repository, which endpoint names the thing, and the digest.
|
||||||
|
//
|
||||||
|
// Two shapes, which are the two the mesh records: `<repository>@sha256:<hex>` is a manifest, and
|
||||||
|
// `<repository>/blobs/sha256:<hex>` is a blob.
|
||||||
|
func split(path string) (repository, kind, digest string, err error) {
|
||||||
|
if before, after, ok := strings.Cut(path, "@sha256:"); ok {
|
||||||
|
return before, "manifests", "sha256:" + after, nil
|
||||||
|
}
|
||||||
|
if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok {
|
||||||
|
return before, "blobs", "sha256:" + after, nil
|
||||||
|
}
|
||||||
|
return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path)
|
||||||
|
}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||||
|
//
|
||||||
|
// A fake store records what it was asked to delete, so what is asserted is the mesh's decision
|
||||||
|
// and the shape of the request — not the registry's behaviour, which is the registry's to test.
|
||||||
|
|
||||||
|
func fakeStore(t *testing.T, answer int) (Store, *[]string) {
|
||||||
|
t.Helper()
|
||||||
|
var asked []string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
|
||||||
|
// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
|
||||||
|
// that does not have the thing does not have its blob either.
|
||||||
|
if answer == http.StatusNotFound {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Length", "7")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.Method != http.MethodDelete {
|
||||||
|
t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
|
||||||
|
}
|
||||||
|
asked = append(asked, r.URL.Path)
|
||||||
|
w.WriteHeader(answer)
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
|
||||||
|
// The two shapes the mesh records: a manifest by digest, and a blob by digest. They are
|
||||||
|
// different endpoints, and asking at the wrong one answers 404 — which this would then
|
||||||
|
// record as collected, leaving the bytes on disk for ever while the record says otherwise.
|
||||||
|
store, asked := fakeStore(t, http.StatusAccepted)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123"
|
||||||
|
archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456"
|
||||||
|
if err := store.LetGo(ctx, image); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.LetGo(ctx, archive); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The archive's holder goes first, then the archive (novox/hq issue 253).
|
||||||
|
_, holder := Holder("sha256:def456", 7)
|
||||||
|
want := []string{
|
||||||
|
"/v2/web/app/manifests/sha256:abc123",
|
||||||
|
"/v2/web/config/manifests/" + holder,
|
||||||
|
"/v2/web/config/blobs/sha256:def456",
|
||||||
|
}
|
||||||
|
if strings.Join(*asked, " ") != strings.Join(want, " ") {
|
||||||
|
t.Fatalf("the store was asked %v; want %v", *asked, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) {
|
||||||
|
// The outcome wanted, already true. Told apart from success only so the sweep can say which
|
||||||
|
// happened; both are recorded, because retrying for ever is the thing to avoid.
|
||||||
|
store, _ := fakeStore(t, http.StatusNotFound)
|
||||||
|
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||||
|
if !errors.Is(err, Gone) {
|
||||||
|
t.Fatalf("a store that does not hold it answered %v, want Gone", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) {
|
||||||
|
// The registry answers 405 when it was started without deletion enabled. The remedy is a
|
||||||
|
// setting on the store's module, and saying "405" would send somebody to the wrong place.
|
||||||
|
store, _ := fakeStore(t, http.StatusMethodNotAllowed)
|
||||||
|
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a store that refuses deletion was read as success")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") {
|
||||||
|
t.Fatalf("the refusal does not name the remedy: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) {
|
||||||
|
// The whole safety of the sweep is that it names only what the mesh recorded putting there.
|
||||||
|
// A reference of another shape — a vendor's image, a package version — is refused rather
|
||||||
|
// than composed into a delete somewhere that is not the mesh's store.
|
||||||
|
store, asked := fakeStore(t, http.StatusAccepted)
|
||||||
|
for _, reference := range []string{
|
||||||
|
"docker.io/library/registry@sha256:abc123",
|
||||||
|
"registry@sha256:abc123",
|
||||||
|
"1.4.2",
|
||||||
|
} {
|
||||||
|
if err := store.LetGo(context.Background(), reference); err == nil {
|
||||||
|
t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(*asked) != 0 {
|
||||||
|
t.Fatalf("the store was asked about %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the
|
||||||
|
// record and not about the store (novox/hq issue 226).
|
||||||
|
//
|
||||||
|
// The sweep skips one and abandons itself for the other, so they cannot be the same error. The
|
||||||
|
// first live run met a reference recorded with the store's old address, read the refusal as "the
|
||||||
|
// store refuses everything", and collected none of the 1681 it had found.
|
||||||
|
func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) {
|
||||||
|
store, asked := fakeStore(t, http.StatusAccepted)
|
||||||
|
for _, reference := range []string{
|
||||||
|
"docker.io/library/registry@sha256:abc123",
|
||||||
|
"127.0.0.1:5100/mesh-tools/build@sha256:abc123",
|
||||||
|
"1.4.2",
|
||||||
|
} {
|
||||||
|
err := store.LetGo(context.Background(), reference)
|
||||||
|
if !errors.Is(err, ErrNotOurs) {
|
||||||
|
t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(*asked) != 0 {
|
||||||
|
t.Fatalf("the store was asked about %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A seat's cancelled set (novox/hq ADR 0219).
|
||||||
|
//
|
||||||
|
// **Cancelling an ask is deleting its message from the seat's work queue** — and that alone has a
|
||||||
|
// race no ordering on one side closes: a holder may fetch the ask in the moment between the
|
||||||
|
// controller reading the queue and deleting the message, and build what a person cancelled. So the
|
||||||
|
// controller first writes the ask's id into the seat's cancelled set, and a holder looks its ask up
|
||||||
|
// there after taking it and before building: listed, it terminates the ask and announces it failed
|
||||||
|
// rather than starting it.
|
||||||
|
//
|
||||||
|
// **A key-value bucket, because that is the shape the bus already has for "a small set the
|
||||||
|
// controller writes and many read cheaply"** (ADR 0201's state buckets): one direct read by key per
|
||||||
|
// ask taken — no consumer, no subscription a holder must keep, nothing that grows a holder's grants
|
||||||
|
// beyond one read subject. Kept beside the seat's own stream and worker and named like them, so the
|
||||||
|
// three objects of one work queue read as one family; asserted by the controller with the queue,
|
||||||
|
// and aged out with it — an id cancelled a week ago names an ask the queue no longer holds.
|
||||||
|
|
||||||
|
// CancelledSetName is the bucket holding a seat's cancelled asks.
|
||||||
|
func CancelledSetName(seat string) string { return "SEAT_" + upperSnake(seat) + "_cancelled" }
|
||||||
|
|
||||||
|
// hasCancelledSet is whether a seat's queue can be cancelled from: the work queues the controller
|
||||||
|
// asks, whose asks it alone shows and changes. A module's own seat's queue is that module's affair.
|
||||||
|
func hasCancelledSet(seat string) bool {
|
||||||
|
for _, s := range seatsTheControllerAsks {
|
||||||
|
if s == seat {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsCancelledSet says a bucket is a seat's cancelled set rather than a module's state — the mesh's
|
||||||
|
// own, and never one to report as state nothing declares.
|
||||||
|
func IsCancelledSet(bucket string) bool {
|
||||||
|
return strings.HasPrefix(bucket, "SEAT_") && strings.HasSuffix(bucket, "_cancelled")
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancelledSetAge is how long a cancelled id is kept: as long as the seat's queue keeps an ask.
|
||||||
|
const cancelledSetAge = 7 * 24 * time.Hour
|
||||||
|
|
||||||
|
// A CancelledSetAsserter is what raising the cancelled sets needs of a connection.
|
||||||
|
type CancelledSetAsserter interface {
|
||||||
|
EnsureCancelledSet(seat string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// RaiseCancelledSets asserts the cancelled set of every work queue the controller asks.
|
||||||
|
func RaiseCancelledSets(a CancelledSetAsserter, seats []DeclaredSeat) error {
|
||||||
|
for _, s := range seats {
|
||||||
|
if len(s.Accepts) == 0 || !hasCancelledSet(s.Name) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := a.EnsureCancelledSet(s.Name); err != nil {
|
||||||
|
return fmt.Errorf("asserting the cancelled set of %s: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureCancelledSet creates a seat's cancelled set if absent and brings its options to match.
|
||||||
|
// Direct reads on, which is how a holder looks an id up with one request.
|
||||||
|
func (j *JetStream) EnsureCancelledSet(seat string) error {
|
||||||
|
js, err := jetstream.New(j.conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: CancelledSetName(seat),
|
||||||
|
Description: fmt.Sprintf("the asks of the %s seat cancelled by hand (novox/hq ADR 0219): written "+
|
||||||
|
"by the controller before it deletes an ask from the queue, read by a holder on taking one, "+
|
||||||
|
"so an ask fetched in that moment is ended rather than built", seat),
|
||||||
|
History: 1,
|
||||||
|
TTL: cancelledSetAge,
|
||||||
|
MaxValueSize: 4 * 1024,
|
||||||
|
MaxBytes: 4 * 1024 * 1024,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", CancelledSetName(seat), err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// A build agent reads its seat's cancelled set by key and nothing more, answers its verbs on its own
|
||||||
|
// machine's subjects, and says whether it is paused under its own machine's name; the controller may
|
||||||
|
// ask any machine's holder its verbs (novox/hq ADR 0219).
|
||||||
|
func TestTheBuildQueueIsControlledWithTheGrantsItNeedsAndNoMore(t *testing.T) {
|
||||||
|
seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"},
|
||||||
|
Emits: []string{"started", "built", "log.*", "paused.*"},
|
||||||
|
Serves: []string{"current", "kill", "pause", "resume"}}
|
||||||
|
holder, err := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "build-agent",
|
||||||
|
Holds: []Seat{seat}, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
has(t, holder.Publish, "$JS.API.DIRECT.GET.KV_SEAT_NODE_BUILD_AGENT_cancelled.$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
|
||||||
|
hasNot(t, holder.Publish, "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
|
||||||
|
has(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.ace")
|
||||||
|
hasNot(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.*")
|
||||||
|
has(t, holder.Publish, "mesh.seat.node-build-agent.event.log.*")
|
||||||
|
has(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.ace")
|
||||||
|
hasNot(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.g14")
|
||||||
|
|
||||||
|
// A module's own seat's queue is its own affair: no cancelled set, no grant for one.
|
||||||
|
other, _ := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "telegram",
|
||||||
|
Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, PasswordHash: "x"})
|
||||||
|
hasNot(t, other.Publish, "$JS.API.DIRECT.GET.KV_SEAT_TELEGRAM_SENDER_cancelled.$KV.SEAT_TELEGRAM_SENDER_cancelled.>")
|
||||||
|
|
||||||
|
controller, _ := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
has(t, controller.Publish, "mesh.seat.node-build-agent.tool.>")
|
||||||
|
|
||||||
|
if CancelledSetName("node-build-agent") != "SEAT_NODE_BUILD_AGENT_cancelled" ||
|
||||||
|
!IsCancelledSet("SEAT_NODE_BUILD_AGENT_cancelled") || IsCancelledSet("build-agent_cancelled") {
|
||||||
|
t.Error("the cancelled set is not named as its seat's family")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only the work queues the controller asks get a cancelled set.
|
||||||
|
func TestOnlyTheControllersQueuesHaveACancelledSet(t *testing.T) {
|
||||||
|
var asserted []string
|
||||||
|
a := asserterFunc(func(seat string) error { asserted = append(asserted, seat); return nil })
|
||||||
|
if err := RaiseCancelledSets(a, []DeclaredSeat{
|
||||||
|
{Name: "node-build-agent", Accepts: []string{"build"}},
|
||||||
|
{Name: "telegram-sender", Accepts: []string{"send"}},
|
||||||
|
{Name: "mesh-controller"},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(asserted) != 1 || asserted[0] != "node-build-agent" {
|
||||||
|
t.Fatalf("asserted %v", asserted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type asserterFunc func(string) error
|
||||||
|
|
||||||
|
func (f asserterFunc) EnsureCancelledSet(seat string) error { return f(seat) }
|
||||||
|
|
||||||
|
// A seat's cancelled set is never reported as state nothing declares.
|
||||||
|
func TestACancelledSetIsNotUndeclaredState(t *testing.T) {
|
||||||
|
undeclared, err := RaiseBuckets(fakeBuckets{names: []string{"SEAT_NODE_BUILD_AGENT_cancelled", "gone_state"}}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(undeclared) != 1 || undeclared[0] != "gone_state" {
|
||||||
|
t.Fatalf("undeclared %v", undeclared)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeBuckets struct{ names []string }
|
||||||
|
|
||||||
|
func (f fakeBuckets) EnsureBucket(Bucket) error { return nil }
|
||||||
|
func (f fakeBuckets) BucketNames() ([]string, error) { return f.names, nil }
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A consumer that reacts to announcements, made on a stream that keeps a week of them, starts from now:
|
||||||
|
// made from the start it replays every merge and build of that week (novox/hq issue 248). And a reset
|
||||||
|
// re-makes a stuck one from now, its configuration otherwise kept, and refuses a work queue.
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAConsumerMadeFromNow
|
||||||
|
func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
js, err := Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
const stream = "HISTORY_TEST"
|
||||||
|
_ = js.js.DeleteStream(stream)
|
||||||
|
if _, err := js.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{"history.>"}, Storage: nats.MemoryStorage}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = js.js.DeleteStream(stream) }()
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
if _, err := js.js.Publish("history.merged", []byte(fmt.Sprint(i))); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fresh := Consumer{Name: "fresh", Stream: stream, Filters: []string{"history.merged"}, Push: true,
|
||||||
|
AckWaitSeconds: 30, MaxDeliver: 5, MaxAckPending: 1, FromNow: true}
|
||||||
|
if err := js.EnsureConsumer(fresh); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, _ := js.js.ConsumerInfo(stream, "fresh")
|
||||||
|
if info.NumPending != 0 || info.Config.DeliverPolicy != nats.DeliverNewPolicy {
|
||||||
|
t.Fatalf("a consumer made from now holds %d of the stream's past (policy %v)", info.NumPending, info.Config.DeliverPolicy)
|
||||||
|
}
|
||||||
|
_, _ = js.js.Publish("history.merged", []byte("new"))
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
if info, _ = js.js.ConsumerInfo(stream, "fresh"); info.NumPending != 1 {
|
||||||
|
t.Fatalf("a new announcement is not pending: %d", info.NumPending)
|
||||||
|
}
|
||||||
|
// Asserted again, it keeps where it is.
|
||||||
|
if err := js.EnsureConsumer(fresh); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One made from the start, as the server's default makes it, and stuck behind its history.
|
||||||
|
old := fresh
|
||||||
|
old.Name, old.FromNow = "old", false
|
||||||
|
if err := js.EnsureConsumer(old); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info, _ = js.js.ConsumerInfo(stream, "old"); info.NumPending != 6 {
|
||||||
|
t.Fatalf("the default should replay all six: %d", info.NumPending)
|
||||||
|
}
|
||||||
|
before, after, err := js.ResetConsumer(stream, "old")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, _ = js.js.ConsumerInfo(stream, "old")
|
||||||
|
if before.Pending != 6 || after.Pending != 0 || info.Config.MaxAckPending != 1 || info.Config.MaxDeliver != 5 ||
|
||||||
|
info.Config.AckWait != 30*time.Second || info.Config.DeliverSubject == "" {
|
||||||
|
t.Fatalf("before %+v after %+v config %+v", before, after, info.Config)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Never a work queue: what is pending there is work.
|
||||||
|
const queue = "QUEUE_TEST"
|
||||||
|
_ = js.js.DeleteStream(queue)
|
||||||
|
if _, err := js.js.AddStream(&nats.StreamConfig{Name: queue, Subjects: []string{"queue.>"}, Retention: nats.WorkQueuePolicy, Storage: nats.MemoryStorage}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = js.js.DeleteStream(queue) }()
|
||||||
|
if _, err := js.js.AddConsumer(queue, &nats.ConsumerConfig{Durable: "w", AckPolicy: nats.AckExplicitPolicy}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := js.ResetConsumer(queue, "w"); err == nil {
|
||||||
|
t.Fatal("a work queue's consumer was reset")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The controller's own key-value buckets (novox/hq to-be 45 §1, §6, §7).
|
||||||
|
//
|
||||||
|
// **What the controller must remember across its own restart, it keeps on the bus.** A call's
|
||||||
|
// outcome lived in the memory of the process that served it (novox/hq issue 265), so a controller
|
||||||
|
// replaced while a push ran answered "no such call" for the one thing its caller had been told to
|
||||||
|
// ask about. The bus already outlives the controller and is the shape ADR 0201 gives a module's
|
||||||
|
// current state: one value per key, written by one owner, read by anybody granted it. These are the
|
||||||
|
// controller's, written by it alone — the writers table of to-be 45 §1 — and asserted on every start
|
||||||
|
// like the streams, so a bus raised from nothing has them before the first call is served.
|
||||||
|
|
||||||
|
// CallsBucket keeps every call of the mesh's own verbs and what came of it; HandActsBucket every act
|
||||||
|
// a person did by hand, with why; ConditionsBucket every condition open now (to-be 45 §2), one key
|
||||||
|
// each, and ConditionHistoryBucket every transition of one — raised, changed, silenced, cleared —
|
||||||
|
// for ninety days.
|
||||||
|
//
|
||||||
|
// **The history is a bucket of its own** because its keys expire and an open condition's must not:
|
||||||
|
// a bucket has one age for every key, and a condition open longer than the history is kept would
|
||||||
|
// otherwise vanish from the store while still true.
|
||||||
|
var (
|
||||||
|
CallsBucket = BucketName(ControllerSeat, "calls")
|
||||||
|
HandActsBucket = BucketName(ControllerSeat, "hand-acts")
|
||||||
|
ConditionsBucket = BucketName(ControllerSeat, "conditions")
|
||||||
|
ConditionHistoryBucket = BucketName(ControllerSeat, "condition-history")
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
|
||||||
|
// A call is two keys — its record, and its answer apart so a listing does not read every answer —
|
||||||
|
// so the stream holds twice as many messages as it keeps calls.
|
||||||
|
const (
|
||||||
|
KeptCallsDurably = 1000
|
||||||
|
CallsKeptFor = 14 * 24 * time.Hour
|
||||||
|
// CallAnswerBytes is the most of one answer kept: a whole declaration is far smaller, and an
|
||||||
|
// answer larger is cut and says so.
|
||||||
|
CallAnswerBytes = 64 << 10
|
||||||
|
// HandActsKeptFor is as long as a condition's history (to-be 45 §2): an act by hand is read
|
||||||
|
// back beside what it addressed.
|
||||||
|
HandActsKeptFor = 90 * 24 * time.Hour
|
||||||
|
// ConditionHistoryKeptFor is how long a condition's transitions are kept (to-be 45 §2).
|
||||||
|
ConditionHistoryKeptFor = 90 * 24 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
|
||||||
|
func IsControllerBucket(bucket string) bool {
|
||||||
|
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||||
|
bucket == ConditionHistoryBucket
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||||
|
func ControllerBuckets() []string {
|
||||||
|
return []string{CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||||
|
type ControllerBucketsAsserter interface {
|
||||||
|
EnsureControllerBuckets() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureControllerBuckets creates the controller's buckets if absent and brings their options to
|
||||||
|
// match. An update, never a delete: what they hold is the record of what the mesh was asked.
|
||||||
|
func (j *JetStream) EnsureControllerBuckets() error {
|
||||||
|
js, err := jetstream.New(j.conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: CallsBucket,
|
||||||
|
Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +
|
||||||
|
"265): written by the controller alone, read through `calls`; the last thousand, or fourteen days",
|
||||||
|
History: 1,
|
||||||
|
TTL: CallsKeptFor,
|
||||||
|
MaxValueSize: CallAnswerBytes + 4<<10,
|
||||||
|
MaxBytes: 2 * KeptCallsDurably * (CallAnswerBytes + 4<<10),
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", CallsBucket, err)
|
||||||
|
}
|
||||||
|
// **The count, on the stream under the bucket.** A bucket has an age and a size and no count;
|
||||||
|
// the stream it is made of does, and with one value per key the oldest message is the oldest
|
||||||
|
// call. Asserted after the bucket, every time, because asserting the bucket writes the stream's
|
||||||
|
// configuration whole and puts the count back to none.
|
||||||
|
stream, err := js.Stream(ctx, "KV_"+CallsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading the stream under %s: %w", CallsBucket, err)
|
||||||
|
}
|
||||||
|
cfg := stream.CachedInfo().Config
|
||||||
|
if cfg.MaxMsgs != 2*KeptCallsDurably {
|
||||||
|
cfg.MaxMsgs = 2 * KeptCallsDurably
|
||||||
|
cfg.Discard = jetstream.DiscardOld
|
||||||
|
if _, err := js.UpdateStream(ctx, cfg); err != nil {
|
||||||
|
return fmt.Errorf("bounding %s to the last %d calls: %w", CallsBucket, KeptCallsDurably, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: HandActsBucket,
|
||||||
|
Description: "every act a person did by hand, with why (novox/hq to-be 45 §7): written by the " +
|
||||||
|
"controller's repairing verbs and `hand-act record`, read through `hand-acts`",
|
||||||
|
History: 1,
|
||||||
|
TTL: HandActsKeptFor,
|
||||||
|
MaxValueSize: 16 << 10,
|
||||||
|
MaxBytes: 64 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", HandActsBucket, err)
|
||||||
|
}
|
||||||
|
// **No age on the open conditions.** A condition is removed when observation clears it and at no
|
||||||
|
// other moment: one that expired would be a fault the store forgot while it was still true.
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: ConditionsBucket,
|
||||||
|
Description: "every condition open now, one key each (novox/hq to-be 45 §2): written by the " +
|
||||||
|
"controller alone, raised and cleared by observation, read through `conditions`",
|
||||||
|
History: 1,
|
||||||
|
MaxValueSize: 64 << 10,
|
||||||
|
MaxBytes: 64 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", ConditionsBucket, err)
|
||||||
|
}
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: ConditionHistoryBucket,
|
||||||
|
Description: "every transition of a condition — raised, changed, silenced, cleared — kept ninety " +
|
||||||
|
"days (novox/hq to-be 45 §2): written by the controller alone, read through `conditions history`",
|
||||||
|
History: 1,
|
||||||
|
TTL: ConditionHistoryKeptFor,
|
||||||
|
MaxValueSize: 64 << 10,
|
||||||
|
MaxBytes: 256 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", ConditionHistoryBucket, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a
|
||||||
|
// key is a publish to the bucket's own subject, which the management interface's grant does not
|
||||||
|
// cover: the cancelled sets' writes timed out for want of this, and the controller's own buckets
|
||||||
|
// would have.
|
||||||
|
func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
|
||||||
|
p, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>"}
|
||||||
|
for _, seat := range seatsTheControllerAsks {
|
||||||
|
if hasCancelledSet(seat) {
|
||||||
|
want = append(want, "$KV."+CancelledSetName(seat)+".>")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, subject := range want {
|
||||||
|
if !slices.Contains(p.Publish, subject) {
|
||||||
|
t.Errorf("the controller may not publish %s, so it cannot write that bucket", subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.Contains(p.Publish, "$KV.>") {
|
||||||
|
t.Error("the controller may write any bucket, a module's state included")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A machine's node tools may say they are there, as that machine and no other** (novox/hq to-be 45
|
||||||
|
// S11), and the controller may hear the bus's advisories and ask who answers — read-only, named.
|
||||||
|
func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) {
|
||||||
|
tools, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(tools.Publish, "mesh.control.anchor.tools-alive") {
|
||||||
|
t.Error("the node tools may not say they are there")
|
||||||
|
}
|
||||||
|
for _, s := range tools.Publish {
|
||||||
|
if strings.Contains(s, "tools-alive") && s != "mesh.control.anchor.tools-alive" {
|
||||||
|
t.Errorf("the node tools may say %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range BusAdvisories {
|
||||||
|
if !slices.Contains(controller.Subscribe, s) {
|
||||||
|
t.Errorf("the controller may not hear %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !slices.Contains(controller.Publish, "$SRV.INFO") || slices.Contains(controller.Subscribe, "$JS.EVENT.>") {
|
||||||
|
t.Error("the controller may not ask who answers, or hears every API call")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -47,7 +47,13 @@ type Consumer struct {
|
|||||||
// and a merge that came back rebuilt what it had just built, five times over on 2026-09-30.
|
// and a merge that came back rebuilt what it had just built, five times over on 2026-09-30.
|
||||||
// With one outstanding, the server holds the rest, and the heartbeat is keeping the message.
|
// With one outstanding, the server holds the rest, and the heartbeat is keeping the message.
|
||||||
MaxAckPending int
|
MaxAckPending int
|
||||||
Why string
|
// FromNow makes a consumer that does not exist yet start at the stream's end rather than its
|
||||||
|
// beginning (novox/hq issue 248). For a consumer that reacts to announcements — a merge, a build's
|
||||||
|
// outcome — on a stream that keeps a week of them: the server's default, everything the stream
|
||||||
|
// holds, replays every merge and every build of that week as if it had just happened. A consumer
|
||||||
|
// that exists keeps where it is, whatever this says; only its making is decided here.
|
||||||
|
FromNow bool
|
||||||
|
Why string
|
||||||
}
|
}
|
||||||
|
|
||||||
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
|
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
|
||||||
@@ -144,6 +150,44 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
|||||||
}, true
|
}, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ModuleConsumer is one module's durable consumer, with the module and node it is for.
|
||||||
|
type ModuleConsumer struct {
|
||||||
|
Node, Module string
|
||||||
|
Consumer Consumer
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumersOf is every module's durable consumer the composed users imply: each module user's, and
|
||||||
|
// each module a runtime carries — a carried module with no account of its own is no user (novox/hq
|
||||||
|
// issue 195), and its consumer is still the controller's to make, since the runtime reads it on the
|
||||||
|
// module's behalf (ADR 0198). One per module and node, whichever of the two named it first.
|
||||||
|
func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||||
|
var out []ModuleConsumer
|
||||||
|
seen := map[string]bool{}
|
||||||
|
add := func(p Principal) {
|
||||||
|
c, needed := ConsumerFor(p)
|
||||||
|
if !needed || seen[p.Node+"/"+p.Module] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[p.Node+"/"+p.Module] = true
|
||||||
|
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: c})
|
||||||
|
}
|
||||||
|
for _, p := range users {
|
||||||
|
if p.Kind == KindModule {
|
||||||
|
add(p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, p := range users {
|
||||||
|
if p.Kind != KindNodeTools {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, d := range p.Carries {
|
||||||
|
add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
|
||||||
|
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// HolderConsumerFor is the worker a seat's holders share on that seat's work queue.
|
// HolderConsumerFor is the worker a seat's holders share on that seat's work queue.
|
||||||
//
|
//
|
||||||
// **One worker for every holder, and each holder pulls one ask when it is idle** (novox/hq ADR
|
// **One worker for every holder, and each holder pulls one ask when it is idle** (novox/hq ADR
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package broker
|
package broker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"crypto/x509"
|
"crypto/x509"
|
||||||
@@ -12,6 +13,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The JetStream side of the controller: the one place the mesh's streams and consumers are
|
// The JetStream side of the controller: the one place the mesh's streams and consumers are
|
||||||
@@ -84,6 +86,13 @@ func pinnedTo(path string) (*tls.Config, error) {
|
|||||||
return PinnedToFingerprint(want), nil
|
return PinnedToFingerprint(want), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OnConn is the JetStream handle over a connection the caller already holds — the control plane's
|
||||||
|
// link — for asserting what the bus holds without dialling a second time.
|
||||||
|
func OnConn(conn *nats.Conn) *JetStream {
|
||||||
|
js, _ := conn.JetStream()
|
||||||
|
return &JetStream{conn: conn, js: js}
|
||||||
|
}
|
||||||
|
|
||||||
// DialPinned is Dial with the server's certificate pinned by a fingerprint the caller already holds
|
// DialPinned is Dial with the server's certificate pinned by a fingerprint the caller already holds
|
||||||
// — a module or a build machine that was handed one beside its credential, and has no file.
|
// — a module or a build machine that was handed one beside its credential, and has no file.
|
||||||
func DialPinned(url, fingerprint string, opts ...nats.Option) (*JetStream, error) {
|
func DialPinned(url, fingerprint string, opts ...nats.Option) (*JetStream, error) {
|
||||||
@@ -299,6 +308,9 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
case errors.Is(err, nats.ErrConsumerNotFound):
|
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||||
|
if c.FromNow {
|
||||||
|
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||||
|
}
|
||||||
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||||
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||||
}
|
}
|
||||||
@@ -308,6 +320,51 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ConsumerState is what a reset says about a consumer, before and after.
|
||||||
|
type ConsumerState struct {
|
||||||
|
DeliverPolicy string
|
||||||
|
Delivered uint64
|
||||||
|
AckFloor uint64
|
||||||
|
Pending uint64
|
||||||
|
AckPending int
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateOf(info *nats.ConsumerInfo) ConsumerState {
|
||||||
|
policy, _ := info.Config.DeliverPolicy.MarshalJSON()
|
||||||
|
return ConsumerState{DeliverPolicy: strings.Trim(string(policy), `"`), Delivered: info.Delivered.Stream,
|
||||||
|
AckFloor: info.AckFloor.Stream, Pending: info.NumPending, AckPending: info.NumAckPending}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResetConsumer re-makes a consumer to start from now, its configuration otherwise unchanged (novox/hq
|
||||||
|
// issue 248): what it had not yet delivered or acknowledged is dropped, which is the point — on a stream
|
||||||
|
// that keeps history, a consumer replaying a week of announcements does nothing anyone wants. Refused on
|
||||||
|
// a work queue, where what is pending is work nobody else will do.
|
||||||
|
func (j *JetStream) ResetConsumer(stream, name string) (before, after ConsumerState, err error) {
|
||||||
|
info, err := j.js.StreamInfo(stream)
|
||||||
|
if err != nil {
|
||||||
|
return before, after, fmt.Errorf("asking about stream %s: %w", stream, err)
|
||||||
|
}
|
||||||
|
if info.Config.Retention == nats.WorkQueuePolicy {
|
||||||
|
return before, after, fmt.Errorf("%s is a work queue: what its consumer has pending is work, and a reset would drop it", stream)
|
||||||
|
}
|
||||||
|
have, err := j.js.ConsumerInfo(stream, name)
|
||||||
|
if err != nil {
|
||||||
|
return before, after, fmt.Errorf("asking about consumer %s on %s: %w", name, stream, err)
|
||||||
|
}
|
||||||
|
before = stateOf(have)
|
||||||
|
want := have.Config
|
||||||
|
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||||
|
want.OptStartSeq, want.OptStartTime = 0, nil
|
||||||
|
if err := j.js.DeleteConsumer(stream, name); err != nil {
|
||||||
|
return before, after, fmt.Errorf("removing consumer %s on %s: %w", name, stream, err)
|
||||||
|
}
|
||||||
|
made, err := j.js.AddConsumer(stream, &want)
|
||||||
|
if err != nil {
|
||||||
|
return before, after, fmt.Errorf("re-making consumer %s on %s — it is gone until the controller asserts it at its next start: %w", name, stream, err)
|
||||||
|
}
|
||||||
|
return before, stateOf(made), nil
|
||||||
|
}
|
||||||
|
|
||||||
func retentionOf(r Retention) nats.RetentionPolicy {
|
func retentionOf(r Retention) nats.RetentionPolicy {
|
||||||
switch r {
|
switch r {
|
||||||
case RetentionWorkQueue:
|
case RetentionWorkQueue:
|
||||||
@@ -316,3 +373,50 @@ func retentionOf(r Retention) nats.RetentionPolicy {
|
|||||||
return nats.LimitsPolicy
|
return nats.LimitsPolicy
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EnsureBucket creates a module's bucket if it is absent and brings its options to match if it is
|
||||||
|
// present (novox/hq ADR 0201).
|
||||||
|
//
|
||||||
|
// **An update, never a delete and recreate**, for the reason a stream is updated: recreating
|
||||||
|
// discards what the bucket holds, and what a module's state holds is data. The mesh's caps are
|
||||||
|
// asserted with the owner's options, so a bucket made by hand converges to them.
|
||||||
|
func (j *JetStream) EnsureBucket(b Bucket) error {
|
||||||
|
history := b.History
|
||||||
|
if history == 0 {
|
||||||
|
history = 1
|
||||||
|
}
|
||||||
|
js, err := jetstream.New(j.conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: b.Bucket(),
|
||||||
|
Description: b.Why(),
|
||||||
|
History: uint8(history),
|
||||||
|
TTL: time.Duration(b.TTLSeconds) * time.Second,
|
||||||
|
MaxValueSize: StateMaxValueBytes,
|
||||||
|
MaxBytes: StateMaxBytes,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", b.Bucket(), err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BucketNames is every key-value bucket on the server, the mesh's and anybody else's.
|
||||||
|
func (j *JetStream) BucketNames() ([]string, error) {
|
||||||
|
js, err := jetstream.New(j.conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
lister := js.KeyValueStoreNames(ctx)
|
||||||
|
var out []string
|
||||||
|
for name := range lister.Name() {
|
||||||
|
out = append(out, name)
|
||||||
|
}
|
||||||
|
return out, lister.Error()
|
||||||
|
}
|
||||||
|
|||||||
@@ -45,6 +45,11 @@ type Membership struct {
|
|||||||
// module that must tell the mesh from the world, the route proxy serving an internal name, reads
|
// module that must tell the mesh from the world, the route proxy serving an internal name, reads
|
||||||
// it here rather than keeping a definition of its own.
|
// it here rather than keeping a definition of its own.
|
||||||
Mesh []string `json:"mesh,omitempty"`
|
Mesh []string `json:"mesh,omitempty"`
|
||||||
|
// State is every bucket this module's code may reach, by the name it uses for each, and whether
|
||||||
|
// it may write it (novox/hq ADR 0201): the runtime answers a bundle's state verbs from this list
|
||||||
|
// and refuses, with the reason, what is not on it — the bus enforces only the union over every
|
||||||
|
// module on the machine.
|
||||||
|
State []StateIssued `json:"state,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Served is one address a tool is answered on.
|
// Served is one address a tool is answered on.
|
||||||
@@ -103,6 +108,7 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
|
|||||||
m.Seats = append(m.Seats, SeatServed{Seat: s.Name, Verb: verb, Subject: seatToolSubject(s, verb, node)})
|
m.Seats = append(m.Seats, SeatServed{Seat: s.Name, Verb: verb, Subject: seatToolSubject(s, verb, node)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
m.State = stateIssuedFor(d)
|
||||||
if len(d.Invokes) > 0 {
|
if len(d.Invokes) > 0 {
|
||||||
m.Reaches = map[string][]string{}
|
m.Reaches = map[string][]string{}
|
||||||
for _, t := range d.Invokes {
|
for _, t := range d.Invokes {
|
||||||
|
|||||||
+91
-3
@@ -18,6 +18,7 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A Kind is what a principal is, which decides the shape of its authority rather than its
|
// A Kind is what a principal is, which decides the shape of its authority rather than its
|
||||||
@@ -103,6 +104,12 @@ type Principal struct {
|
|||||||
// permission and nothing beside it.
|
// permission and nothing beside it.
|
||||||
Invokes []string
|
Invokes []string
|
||||||
|
|
||||||
|
// State is the local names of the state this principal's module keeps, and Reads the state of
|
||||||
|
// others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's
|
||||||
|
// instances and read by whoever declares it.
|
||||||
|
State []string
|
||||||
|
Reads []string
|
||||||
|
|
||||||
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
||||||
// and never appears here: this file is written to a node's disk and read by a server, and a
|
// and never appears here: this file is written to a node's disk and read by a server, and a
|
||||||
// secret that can be read from a configuration file is a secret with a wider blast radius
|
// secret that can be read from a configuration file is a secret with a wider blast radius
|
||||||
@@ -118,6 +125,20 @@ type Principal struct {
|
|||||||
// goes with the retired seat row.
|
// goes with the retired seat row.
|
||||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||||
|
|
||||||
|
// SeatVerb is one verb of one seat, on every machine holding it.
|
||||||
|
type SeatVerb struct{ Seat, Verb string }
|
||||||
|
|
||||||
|
// VerbsTheSelfCheckAsks are the seat verbs the controller's self-check and watchdogs call (novox/hq
|
||||||
|
// to-be 45 §4): D8 reads every machine's ban list. **Named one by one, and the test that holds them
|
||||||
|
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
|
||||||
|
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
|
||||||
|
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
|
||||||
|
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}
|
||||||
|
|
||||||
|
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||||
|
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
||||||
|
var perMachineEvents = map[string]bool{"paused.*": true}
|
||||||
|
|
||||||
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
|
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
|
||||||
// controller may answer an enrolment is derived from the same constant the user is named from.
|
// controller may answer an enrolment is derived from the same constant the user is named from.
|
||||||
const enrolmentPrefix = "enrol"
|
const enrolmentPrefix = "enrol"
|
||||||
@@ -179,6 +200,13 @@ type Permissions struct {
|
|||||||
AllowResponses bool
|
AllowResponses bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResponseTTL is how long the bus lets a principal answer a request it received. Its one answer has
|
||||||
|
// to come inside this, and a seat's holder answers within link.AnswerWithin — inside it by design.
|
||||||
|
// **A broker reloading its user list forgets every answer it was about to permit**, whatever this
|
||||||
|
// says (novox/hq issue 265): a call that is still running when the list reloads has its answer
|
||||||
|
// refused, which is why a holder answers before it does what can reload it.
|
||||||
|
const ResponseTTL = time.Minute
|
||||||
|
|
||||||
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
|
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
|
||||||
// a permission that cannot be derived from a declaration is a permission nobody can explain.
|
// a permission that cannot be derived from a declaration is a permission nobody can explain.
|
||||||
func PermissionsFor(p Principal) (Permissions, error) {
|
func PermissionsFor(p Principal) (Permissions, error) {
|
||||||
@@ -215,6 +243,16 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// the role, and whichever machine holding it is idle takes it.
|
// the role, and whichever machine holding it is idle takes it.
|
||||||
for _, seat := range seatsTheControllerAsks {
|
for _, seat := range seatsTheControllerAsks {
|
||||||
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
||||||
|
// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
|
||||||
|
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||||
|
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||||
|
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
||||||
|
// **And its cancelled set** (novox/hq ADR 0219, issue 269): a cancel writes the ask's id
|
||||||
|
// there before it deletes the ask, and a write is a publish to the bucket's subject, which
|
||||||
|
// `$JS.API.>` does not cover — so every cancel timed out, refused by this list.
|
||||||
|
if hasCancelledSet(seat) {
|
||||||
|
pub = append(pub, "$KV."+CancelledSetName(seat)+".>")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||||
// facts under the seat it holds, because a role's events belong to the role and keep their
|
// facts under the seat it holds, because a role's events belong to the role and keep their
|
||||||
@@ -238,11 +276,19 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
||||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||||
sub = append(sub, announcing(ControllerSeat)...)
|
sub = append(sub, announcing(ControllerSeat)...)
|
||||||
|
// And the verbs the self-check reads with, of any machine's holder (novox/hq to-be 45 §4).
|
||||||
|
for _, v := range VerbsTheSelfCheckAsks {
|
||||||
|
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*")
|
||||||
|
}
|
||||||
|
// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
|
||||||
|
// the same discovery the console reads. The question only; the answers come to its own inbox.
|
||||||
|
pub = append(pub, "$SRV.INFO")
|
||||||
|
|
||||||
// The two events it reacts to, and its ack subject on the stream they arrive from
|
// The events it reacts to, and its ack subject on the stream they arrive from
|
||||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||||
// controller a subscriber to every event in the mesh, and its permission list would stop
|
// controller a subscriber to every event in the mesh, and its permission list would stop
|
||||||
// saying what it is for. The ack grant below is scoped per stream because the controller's
|
// saying what it is for. The one wildcard is the emitter of a provider's standing (ADR
|
||||||
|
// 0224) — still two named events, from whichever module provides. The ack grant below is scoped per stream because the controller's
|
||||||
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
|
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
|
||||||
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
||||||
// ever, refused by the list it already has.
|
// ever, refused by the list it already has.
|
||||||
@@ -265,6 +311,18 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// enrolments and can reach nothing else.
|
// enrolments and can reach nothing else.
|
||||||
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
||||||
|
|
||||||
|
// **And its own buckets** (novox/hq to-be 45 §1): the calls it served and the acts done by
|
||||||
|
// hand, which it alone writes. A put is a publish to the bucket's subject, which `$JS.API.>`
|
||||||
|
// does not cover; each bucket named, not `$KV.>`, which would let it write any module's state.
|
||||||
|
for _, bucket := range ControllerBuckets() {
|
||||||
|
pub = append(pub, "$KV."+bucket+".>")
|
||||||
|
}
|
||||||
|
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
|
||||||
|
// consumer that gave up on a message, or one that was deleted. The server already publishes
|
||||||
|
// both in the mesh's own account; the controller says each as a condition in the mesh's words.
|
||||||
|
// Named, not `$JS.EVENT.>`: the other advisories are every API call the mesh makes.
|
||||||
|
sub = append(sub, BusAdvisories...)
|
||||||
|
|
||||||
case KindPerson:
|
case KindPerson:
|
||||||
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||||
// who could publish an event would be able to claim a module said something.
|
// who could publish an event would be able to claim a module said something.
|
||||||
@@ -398,10 +456,26 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
|
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
|
||||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
|
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
|
||||||
"$JS.ACK."+stream+"."+worker+".>")
|
"$JS.ACK."+stream+"."+worker+".>")
|
||||||
|
// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
|
||||||
|
// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
|
||||||
|
// moment the controller cancelled it ends it instead of building it. Read, never written:
|
||||||
|
// the set is the controller's, and only the work queues the controller asks — and so may
|
||||||
|
// cancel from — have one.
|
||||||
|
if hasCancelledSet(s.Name) {
|
||||||
|
set := CancelledSetName(s.Name)
|
||||||
|
pub = append(pub, "$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+".>")
|
||||||
|
}
|
||||||
for _, a := range s.Accepts {
|
for _, a := range s.Accepts {
|
||||||
sub = append(sub, seatSubject(s, "accept", a))
|
sub = append(sub, seatSubject(s, "accept", a))
|
||||||
}
|
}
|
||||||
for _, e := range s.Emits {
|
for _, e := range s.Emits {
|
||||||
|
// **A machine says its own state and no other's** (novox/hq ADR 0219): on a node-scoped
|
||||||
|
// seat, an event about the holder itself carries the machine as its last token, and
|
||||||
|
// each holder is granted its own machine's alone.
|
||||||
|
if s.Scope == "node" && p.Node != "" && perMachineEvents[e] {
|
||||||
|
pub = append(pub, seatSubject(s, "event", strings.TrimSuffix(e, "*")+p.Node))
|
||||||
|
continue
|
||||||
|
}
|
||||||
pub = append(pub, seatSubject(s, "event", e))
|
pub = append(pub, seatSubject(s, "event", e))
|
||||||
}
|
}
|
||||||
for _, t := range s.Serves {
|
for _, t := range s.Serves {
|
||||||
@@ -421,6 +495,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 5. Its state, and the state of others it reads (novox/hq ADR 0201): every one read and
|
||||||
|
// watched, its own written too.
|
||||||
|
pub = append(pub, stateGrants(p.Module, p.State, p.Reads)...)
|
||||||
|
|
||||||
case KindNodeTools:
|
case KindNodeTools:
|
||||||
// **One process serves what every module on the machine would have served for itself**
|
// **One process serves what every module on the machine would have served for itself**
|
||||||
// (novox/hq ADR 0175). Each carried module's whole tool namespace — the same grant that
|
// (novox/hq ADR 0175). Each carried module's whole tool namespace — the same grant that
|
||||||
@@ -456,6 +534,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// that varies is the module, so the pattern is the machine's own assignments.
|
// that varies is the module, so the pattern is the machine's own assignments.
|
||||||
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
||||||
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
||||||
|
// And that it is there (novox/hq to-be 45 §3, S11): its own heartbeat, under its machine's
|
||||||
|
// name and no other's, on core NATS like the host's.
|
||||||
|
pub = append(pub, "mesh.control."+p.Node+".tools-alive")
|
||||||
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
||||||
// node, as the console already could — the runtime is the console's serving mode.
|
// node, as the console already could — the runtime is the console's serving mode.
|
||||||
invoked, err := invokedSubjects([]string{"*"})
|
invoked, err := invokedSubjects([]string{"*"})
|
||||||
@@ -487,6 +568,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+durable,
|
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+durable,
|
||||||
"$JS.ACK."+stream+"."+durable+".>")
|
"$JS.ACK."+stream+"."+durable+".>")
|
||||||
}
|
}
|
||||||
|
// **And it keeps and reads state for the modules it carries** (novox/hq ADR 0201): the union
|
||||||
|
// of what each may do with a bucket — an owner's write, a reader's read. That one module's code
|
||||||
|
// does not write another's bucket through it is the runtime's to keep, from the membership
|
||||||
|
// each assignment is issued, as it keeps each module's events under that module's own name.
|
||||||
|
for _, d := range p.Carries {
|
||||||
|
pub = append(pub, stateGrants(d.Module, stateNames(d.State), d.Reads)...)
|
||||||
|
}
|
||||||
sub = unique(sub)
|
sub = unique(sub)
|
||||||
pub = unique(pub)
|
pub = unique(pub)
|
||||||
}
|
}
|
||||||
@@ -736,7 +824,7 @@ func ComposeAccounts(principals []Principal) (string, error) {
|
|||||||
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
|
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
|
||||||
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
|
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
|
||||||
if perms.AllowResponses {
|
if perms.AllowResponses {
|
||||||
b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n")
|
fmt.Fprintf(&b, " allow_responses: { max: 1, ttl: \"%dm\" }\n", int(ResponseTTL/time.Minute))
|
||||||
}
|
}
|
||||||
b.WriteString(" } }\n")
|
b.WriteString(" } }\n")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,16 +5,16 @@ import "testing"
|
|||||||
// A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
|
// A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
|
||||||
// node-scoped seat derive two addresses, and a user of the seat may publish any node's.
|
// node-scoped seat derive two addresses, and a user of the seat may publish any node's.
|
||||||
func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
|
func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
|
||||||
seat := Seat{Name: "node-dns-resolver", Scope: "node", Serves: []string{"lookup"}}
|
seat := Seat{Name: "node-hostname", Scope: "node", Serves: []string{"entries"}}
|
||||||
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
|
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||||
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
|
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||||
has(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.one")
|
has(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.one")
|
||||||
has(t, two.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
|
has(t, two.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
|
||||||
hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup")
|
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries")
|
||||||
hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
|
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
|
||||||
|
|
||||||
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
|
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
|
||||||
has(t, user.Publish, "mesh.seat.node-dns-resolver.tool.lookup.*")
|
has(t, user.Publish, "mesh.seat.node-hostname.tool.entries.*")
|
||||||
}
|
}
|
||||||
|
|
||||||
// A mesh-scoped seat's tool stays flat: nothing about it changes.
|
// A mesh-scoped seat's tool stays flat: nothing about it changes.
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's state on the bus (novox/hq ADR 0201, design 32 §4, design 25 §3).
|
||||||
|
//
|
||||||
|
// A module names the state it keeps (`state`) and the state of others it reads (`reads`), and each
|
||||||
|
// is a key-value bucket: the server's own last-per-subject stream with direct reads, delete markers
|
||||||
|
// and watches, which is the state relationship the mesh already uses for declarations, opened to
|
||||||
|
// modules. The controller creates every bucket from the catalogue — from registration, like a
|
||||||
|
// seat's stream, so a reader may watch before the owner runs anywhere — and no module can.
|
||||||
|
//
|
||||||
|
// Pure, like everything else in this package that decides what the bus holds; jetstream.go is the
|
||||||
|
// part that asks a server.
|
||||||
|
|
||||||
|
// The mesh's caps on a bucket, the same for every module: a value is a piece of state, not a file,
|
||||||
|
// and a bucket that grew without bound would be one module filling the bus's disk for everyone.
|
||||||
|
const (
|
||||||
|
StateMaxValueBytes = 256 * 1024
|
||||||
|
StateMaxBytes = 64 * 1024 * 1024
|
||||||
|
)
|
||||||
|
|
||||||
|
// A Bucket is one module's declared state as the bus holds it.
|
||||||
|
type Bucket struct {
|
||||||
|
Module string
|
||||||
|
Name string
|
||||||
|
// History is how many values a key keeps; zero is one.
|
||||||
|
History int
|
||||||
|
// TTLSeconds is how long a value lives; zero is until replaced or deleted.
|
||||||
|
TTLSeconds int
|
||||||
|
}
|
||||||
|
|
||||||
|
// BucketName is the bucket a module's state lives in: the module and the local name joined by an
|
||||||
|
// underscore, which neither may contain, so two modules can never derive one bucket.
|
||||||
|
func BucketName(module, name string) string { return module + "_" + name }
|
||||||
|
|
||||||
|
// Bucket is this bucket's name on the bus.
|
||||||
|
func (b Bucket) Bucket() string { return BucketName(b.Module, b.Name) }
|
||||||
|
|
||||||
|
// Why is carried into the server's description of the bucket, so somebody reading the server's
|
||||||
|
// own state finds whose it is and why it is kept.
|
||||||
|
func (b Bucket) Why() string {
|
||||||
|
return fmt.Sprintf("%s's state %q (novox/hq ADR 0201): its current value per key, written by %s, "+
|
||||||
|
"read by whatever declares it reads it; kept when %s is unassigned, because it is data",
|
||||||
|
b.Module, b.Name, b.Module, b.Module)
|
||||||
|
}
|
||||||
|
|
||||||
|
// bucketOfRead is the bucket a read names, `<module>.<name>`, or false when it names none.
|
||||||
|
func bucketOfRead(read string) (string, bool) {
|
||||||
|
at := strings.LastIndex(read, ".")
|
||||||
|
if at <= 0 || at == len(read)-1 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
module, name := read[:at], read[at+1:]
|
||||||
|
if !safeSubject.MatchString(module) || !safeSubject.MatchString(name) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return BucketName(module, name), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// stateGrants is what a principal publishes to reach the state its modules keep and read: for every
|
||||||
|
// bucket, binding to it, reading a key directly, and an ordered consumer for listing and watching,
|
||||||
|
// created and deleted on the bucket's own stream, with its flow control answered; for a bucket an
|
||||||
|
// owner keeps, writing under the bucket's own subjects too.
|
||||||
|
//
|
||||||
|
// **Measured against a running server, 2026-10-04** (novox/hq research 024), and each one is there
|
||||||
|
// because leaving it out failed: without STREAM.INFO nothing binds; without DIRECT.GET nothing is
|
||||||
|
// read; without CONSUMER.CREATE no key is listed and nothing is watched; without CONSUMER.DELETE a
|
||||||
|
// watch cannot be stopped and lingers on the server. A write outside these is refused by the server
|
||||||
|
// — and reaches the writer as a timeout, not a refusal, which is why the runtime refuses first.
|
||||||
|
func stateGrants(module string, keeps []string, reads []string) []string {
|
||||||
|
var out []string
|
||||||
|
read := func(bucket string) {
|
||||||
|
stream := "KV_" + bucket
|
||||||
|
out = append(out,
|
||||||
|
"$JS.API.STREAM.INFO."+stream,
|
||||||
|
"$JS.API.DIRECT.GET."+stream+".>",
|
||||||
|
"$JS.API.CONSUMER.CREATE."+stream+".>",
|
||||||
|
"$JS.API.CONSUMER.DELETE."+stream+".>",
|
||||||
|
"$JS.FC."+stream+".>")
|
||||||
|
}
|
||||||
|
for _, name := range keeps {
|
||||||
|
if !safeSubject.MatchString(name) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bucket := BucketName(module, name)
|
||||||
|
read(bucket)
|
||||||
|
out = append(out, "$KV."+bucket+".>")
|
||||||
|
}
|
||||||
|
for _, r := range reads {
|
||||||
|
if bucket, ok := bucketOfRead(r); ok {
|
||||||
|
read(bucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// StateIssued is one bucket an assignment may reach, by the name its module uses for it: its own
|
||||||
|
// state by the local name, another's as `<module>.<name>` (novox/hq ADR 0201).
|
||||||
|
type StateIssued struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Bucket string `json:"bucket"`
|
||||||
|
Writes bool `json:"writes,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// stateIssuedFor is every bucket a module's code may reach, as its membership lists them.
|
||||||
|
func stateIssuedFor(d Declared) []StateIssued {
|
||||||
|
var out []StateIssued
|
||||||
|
for _, b := range d.State {
|
||||||
|
out = append(out, StateIssued{Name: b.Name, Bucket: BucketName(d.Module, b.Name), Writes: true})
|
||||||
|
}
|
||||||
|
for _, r := range d.Reads {
|
||||||
|
if bucket, ok := bucketOfRead(r); ok {
|
||||||
|
out = append(out, StateIssued{Name: r, Bucket: bucket})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// stateNames is the local names of a module's own buckets.
|
||||||
|
func stateNames(buckets []Bucket) []string {
|
||||||
|
out := make([]string, 0, len(buckets))
|
||||||
|
for _, b := range buckets {
|
||||||
|
out = append(out, b.Name)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// A BucketAsserter is the part of a JetStream connection bucket assertion needs.
|
||||||
|
type BucketAsserter interface {
|
||||||
|
// EnsureBucket creates the bucket if absent and brings its options to match if present, never
|
||||||
|
// discarding what it holds.
|
||||||
|
EnsureBucket(b Bucket) error
|
||||||
|
// BucketNames is every key-value bucket on the server.
|
||||||
|
BucketNames() ([]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RaiseBuckets asserts every declared bucket and answers the buckets on the server that nothing
|
||||||
|
// declares any more.
|
||||||
|
//
|
||||||
|
// **Those are reported, never removed** (novox/hq ADR 0201, ADR 0030): what a module stored is
|
||||||
|
// data, and a manifest edited, a module renamed or a catalogue entry dropped is an ordinary day's
|
||||||
|
// work that must not take data with it. Removing one is a person's act.
|
||||||
|
func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err error) {
|
||||||
|
sorted := append([]Bucket(nil), buckets...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Bucket() < sorted[j].Bucket() })
|
||||||
|
declared := map[string]bool{}
|
||||||
|
for _, b := range sorted {
|
||||||
|
if err := a.EnsureBucket(b); err != nil {
|
||||||
|
return nil, fmt.Errorf("asserting %s's state %q: %w", b.Module, b.Name, err)
|
||||||
|
}
|
||||||
|
declared[b.Bucket()] = true
|
||||||
|
}
|
||||||
|
names, err := a.BucketNames()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
||||||
|
}
|
||||||
|
for _, n := range names {
|
||||||
|
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state; so are
|
||||||
|
// the controller's own buckets (novox/hq to-be 45 §1).
|
||||||
|
if !declared[n] && !IsCancelledSet(n) && !IsControllerBucket(n) {
|
||||||
|
undeclared = append(undeclared, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(undeclared)
|
||||||
|
return undeclared, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The grants measured against a running server (novox/hq research 024): an owner reads and writes
|
||||||
|
// its bucket, a reader only reads, and neither reaches any other bucket.
|
||||||
|
func TestAnOwnerWritesItsStateAndAReaderOnlyReads(t *testing.T) {
|
||||||
|
owner, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "claude-code",
|
||||||
|
State: []string{"servers"}, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range []string{
|
||||||
|
"$KV.claude-code_servers.>",
|
||||||
|
"$JS.API.STREAM.INFO.KV_claude-code_servers",
|
||||||
|
"$JS.API.DIRECT.GET.KV_claude-code_servers.>",
|
||||||
|
"$JS.API.CONSUMER.CREATE.KV_claude-code_servers.>",
|
||||||
|
"$JS.API.CONSUMER.DELETE.KV_claude-code_servers.>",
|
||||||
|
"$JS.FC.KV_claude-code_servers.>",
|
||||||
|
} {
|
||||||
|
has(t, owner.Publish, s)
|
||||||
|
}
|
||||||
|
hasNot(t, owner.Publish, "$KV.>")
|
||||||
|
hasNot(t, owner.Publish, "$JS.API.>")
|
||||||
|
|
||||||
|
reader, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "console",
|
||||||
|
Reads: []string{"claude-code.servers"}, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
has(t, reader.Publish, "$JS.API.DIRECT.GET.KV_claude-code_servers.>")
|
||||||
|
has(t, reader.Publish, "$JS.API.CONSUMER.CREATE.KV_claude-code_servers.>")
|
||||||
|
hasNot(t, reader.Publish, "$KV.claude-code_servers.>")
|
||||||
|
for _, s := range reader.Subscribe {
|
||||||
|
if s == "$KV.claude-code_servers.>" {
|
||||||
|
t.Fatalf("a reader subscribes the bucket's subjects directly: %v", reader.Subscribe)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One runtime carries every module on its machine, so its grant is the union: the owner's write
|
||||||
|
// where an owner is carried, a read where only a reader is.
|
||||||
|
func TestTheRuntimeKeepsAndReadsStateForItsModules(t *testing.T) {
|
||||||
|
perms, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "one", Module: RuntimeModule,
|
||||||
|
Carries: []Declared{
|
||||||
|
{Module: "claude-code", State: []Bucket{{Module: "claude-code", Name: "servers"}},
|
||||||
|
Reads: []string{"licence-manager.bindings"}},
|
||||||
|
{Module: "audit"},
|
||||||
|
}, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
has(t, perms.Publish, "$KV.claude-code_servers.>")
|
||||||
|
has(t, perms.Publish, "$JS.API.DIRECT.GET.KV_licence-manager_bindings.>")
|
||||||
|
hasNot(t, perms.Publish, "$KV.licence-manager_bindings.>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module with no state is granted nothing of any bucket — the composition of every module that
|
||||||
|
// existed before this is unchanged.
|
||||||
|
func TestAModuleWithNoStateReachesNoBucket(t *testing.T) {
|
||||||
|
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||||
|
Emits: []string{"order.placed"}, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range perms.Publish {
|
||||||
|
if strings.HasPrefix(s, "$KV.") || strings.HasPrefix(s, "$JS.FC.") || strings.Contains(s, ".KV_") {
|
||||||
|
t.Fatalf("granted %q without declaring state", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A read that names no bucket grants nothing rather than something that happens to parse.
|
||||||
|
func TestAReadThatNamesNoBucketGrantsNothing(t *testing.T) {
|
||||||
|
if got := stateGrants("a", nil, []string{"nodot", "x.", ".y", "a.b>"}); len(got) != 0 {
|
||||||
|
t.Fatalf("granted %v for reads that name no bucket", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The membership lists every bucket the module's code may reach, by the name the module uses for
|
||||||
|
// it, and whether it may write it — the list the runtime refuses from.
|
||||||
|
func TestAMembershipListsTheStateItsModuleMayReach(t *testing.T) {
|
||||||
|
m := MembershipFor("one", Declared{Module: "claude-code",
|
||||||
|
State: []Bucket{{Module: "claude-code", Name: "servers"}},
|
||||||
|
Reads: []string{"licence-manager.bindings"}}, Placements{})
|
||||||
|
want := []StateIssued{
|
||||||
|
{Name: "servers", Bucket: "claude-code_servers", Writes: true},
|
||||||
|
{Name: "licence-manager.bindings", Bucket: "licence-manager_bindings"},
|
||||||
|
}
|
||||||
|
if !slices.Equal(m.State, want) {
|
||||||
|
t.Fatalf("issued %+v, want %+v", m.State, want)
|
||||||
|
}
|
||||||
|
if none := MembershipFor("one", Declared{Module: "audit"}, Placements{}); none.State != nil {
|
||||||
|
t.Fatalf("a module with no state was issued %+v", none.State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type buckets struct {
|
||||||
|
ensured []string
|
||||||
|
on []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *buckets) EnsureBucket(x Bucket) error {
|
||||||
|
b.ensured = append(b.ensured, x.Bucket())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (b *buckets) BucketNames() ([]string, error) { return b.on, nil }
|
||||||
|
|
||||||
|
// Every declared bucket is asserted; one on the server that nothing declares is said, not removed.
|
||||||
|
func TestRaisingStateReportsWhatNothingDeclares(t *testing.T) {
|
||||||
|
b := &buckets{on: []string{"claude-code_servers", "gone_old", "ours_by_hand"}}
|
||||||
|
undeclared, err := RaiseBuckets(b, []Bucket{{Module: "claude-code", Name: "servers"}, {Module: "a", Name: "b"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Equal(b.ensured, []string{"a_b", "claude-code_servers"}) {
|
||||||
|
t.Fatalf("asserted %v", b.ensured)
|
||||||
|
}
|
||||||
|
if !slices.Equal(undeclared, []string{"gone_old", "ours_by_hand"}) {
|
||||||
|
t.Fatalf("reported %v", undeclared)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Against a real server: a bucket is created with the owner's options and the mesh's caps,
|
||||||
|
// asserting it again changes nothing and keeps what it holds, and a changed option is brought to
|
||||||
|
// match in place.
|
||||||
|
func TestABucketIsAssertedInPlace(t *testing.T) {
|
||||||
|
js := aLiveBus(t)
|
||||||
|
b := Bucket{Module: "statetest", Name: "servers"}
|
||||||
|
if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
|
||||||
|
t.Fatalf("a real server refused a module's bucket: %v", err)
|
||||||
|
}
|
||||||
|
kv, err := js.Context().KeyValue(b.Bucket())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put("all.one", []byte(`{"kept":true}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.History = 3
|
||||||
|
if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
|
||||||
|
t.Fatalf("asserting the bucket again failed, so a restart would: %v", err)
|
||||||
|
}
|
||||||
|
got, err := kv.Get("all.one")
|
||||||
|
if err != nil || string(got.Value()) != `{"kept":true}` {
|
||||||
|
t.Fatalf("asserting again lost what the bucket held: %v %v", got, err)
|
||||||
|
}
|
||||||
|
status, err := kv.Status()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if status.History() != 3 {
|
||||||
|
t.Fatalf("history is %d, the owner declared 3", status.History())
|
||||||
|
}
|
||||||
|
if s, ok := status.(*nats.KeyValueBucketStatus); ok {
|
||||||
|
if c := s.StreamInfo().Config; c.MaxMsgSize != StateMaxValueBytes || c.MaxBytes != StateMaxBytes {
|
||||||
|
t.Fatalf("the mesh's caps are not on the bucket: value %d, bucket %d", c.MaxMsgSize, c.MaxBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Against a real server: the handle over a connection the control plane already holds asserts a
|
||||||
|
// bucket as Dial's does — what a push uses, so a module registered since the last start has its
|
||||||
|
// bucket before its membership names it.
|
||||||
|
func TestABucketIsAssertedOverAHeldConnection(t *testing.T) {
|
||||||
|
js := aLiveBus(t)
|
||||||
|
held := OnConn(js.Conn())
|
||||||
|
if _, err := RaiseBuckets(held, []Bucket{{Module: "statetest", Name: "held"}}); err != nil {
|
||||||
|
t.Fatalf("asserting over a held connection failed: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := js.Context().KeyValue("statetest_held"); err != nil {
|
||||||
|
t.Fatalf("the bucket is not there: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -20,12 +21,15 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
|||||||
t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q",
|
t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q",
|
||||||
broker.ControllerSeat, link.MeshControllerSeat)
|
broker.ControllerSeat, link.MeshControllerSeat)
|
||||||
}
|
}
|
||||||
for _, event := range []string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore} {
|
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||||
|
states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...)
|
||||||
|
states = append(states, conditions.HeartbeatEvent)
|
||||||
|
for _, event := range states {
|
||||||
if !slices.Contains(broker.ControllerStates, event) {
|
if !slices.Contains(broker.ControllerStates, event) {
|
||||||
t.Errorf("the mesh states %q and its account may not publish it", event)
|
t.Errorf("the mesh states %q and its account may not publish it", event)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(broker.ControllerStates) != 3 {
|
if len(broker.ControllerStates) != len(states) {
|
||||||
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
||||||
}
|
}
|
||||||
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
||||||
|
|||||||
@@ -201,7 +201,23 @@ const ControllerName = "controller"
|
|||||||
// something to say.
|
// something to say.
|
||||||
const ControllerSeat = "mesh-controller"
|
const ControllerSeat = "mesh-controller"
|
||||||
|
|
||||||
var ControllerStates = []string{"applied", "refused", "built-before"}
|
var ControllerStates = []string{"applied", "refused", "built-before",
|
||||||
|
// What is wrong, said as it changes (novox/hq to-be 45 §2): a condition raised, changed in
|
||||||
|
// severity, resolver or silence, and cleared. The operator-channel's holder and any other surface
|
||||||
|
// consume them; the controller tells nobody itself.
|
||||||
|
"condition-raised", "condition-changed", "condition-cleared",
|
||||||
|
// And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a
|
||||||
|
// machine that is not the control node, so the controller going quiet is itself said.
|
||||||
|
"doctor-heartbeat"}
|
||||||
|
|
||||||
|
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
||||||
|
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
||||||
|
// may and gave up on it, and one that was deleted. Read-only: an advisory is the server's to
|
||||||
|
// publish, and the controller's subscription changes nothing on the bus.
|
||||||
|
var BusAdvisories = []string{
|
||||||
|
"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>",
|
||||||
|
"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>",
|
||||||
|
}
|
||||||
|
|
||||||
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
||||||
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
||||||
@@ -226,8 +242,23 @@ var ControllerFollows = []string{
|
|||||||
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
|
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
|
||||||
// with the retired seat row.
|
// with the retired seat row.
|
||||||
seatEventSubject("mesh-build-machine", "built"),
|
seatEventSubject("mesh-build-machine", "built"),
|
||||||
|
// **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and
|
||||||
|
// that consumer recovered. The one pattern on this list, and a narrow one — two named events,
|
||||||
|
// from whichever module provides — because the rule is about every provider, and a list of
|
||||||
|
// providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider
|
||||||
|
// failed every consumer for a day and only its journal said so (issue 179). Appended, because
|
||||||
|
// the index is a name.
|
||||||
|
moduleEventSubject("*", ProvisionerFailing),
|
||||||
|
moduleEventSubject("*", ProvisionerRecovered),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The provider standing events, by their local names. Written here as well as in the catalogue
|
||||||
|
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
|
||||||
|
const (
|
||||||
|
ProvisionerFailing = "provisioner.failing"
|
||||||
|
ProvisionerRecovered = "provisioner.recovered"
|
||||||
|
)
|
||||||
|
|
||||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||||
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
|
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
|
||||||
func moduleEventSubject(module, event string) string {
|
func moduleEventSubject(module, event string) string {
|
||||||
@@ -270,7 +301,8 @@ func MeshConsumers() []Consumer {
|
|||||||
// announcement handed over behind it must wait on the server, not time out on the
|
// announcement handed over behind it must wait on the server, not time out on the
|
||||||
// client and come back to be acted on again.
|
// client and come back to be acted on again.
|
||||||
MaxAckPending: 1,
|
MaxAckPending: 1,
|
||||||
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
|
FromNow: true,
|
||||||
|
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
||||||
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
||||||
"become actionable",
|
"become actionable",
|
||||||
},
|
},
|
||||||
|
|||||||
+2
-2
@@ -24,8 +24,8 @@ accounts {
|
|||||||
jetstream: enabled
|
jetstream: enabled
|
||||||
users = [
|
users = [
|
||||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] }
|
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||||
|
|||||||
@@ -33,6 +33,14 @@ type Declared struct {
|
|||||||
Watches []Seat
|
Watches []Seat
|
||||||
// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
|
// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
|
||||||
Invokes []string
|
Invokes []string
|
||||||
|
// State is the state it keeps, each a bucket its instances write (novox/hq ADR 0201).
|
||||||
|
State []Bucket
|
||||||
|
// Reads are other modules' state it reads, each `<module>.<name>` (novox/hq ADR 0201).
|
||||||
|
Reads []string
|
||||||
|
// NoAccount says the module declares no own secret named broker, so no account could ever be
|
||||||
|
// delivered to it and nothing can connect as it (novox/hq issue 195). Said in the negative so a
|
||||||
|
// record that does not say is composed as it always was.
|
||||||
|
NoAccount bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Records is what composing a user list needs to know about the mesh, and nothing more.
|
// Records is what composing a user list needs to know about the mesh, and nothing more.
|
||||||
@@ -77,10 +85,20 @@ func Users(r Records) ([]Principal, error) {
|
|||||||
if runtimeHere && d.Module == RuntimeModule {
|
if runtimeHere && d.Module == RuntimeModule {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// **A module with nowhere to read an account is no user** (novox/hq issue 195). `module
|
||||||
|
// issue` refuses it one (issue 078: an account nothing reads is an orphan), so its user
|
||||||
|
// could only ever be left out of the file for want of a password — and every such module
|
||||||
|
// was named, on every status and plan, as a credential the mesh had not minted. Where the
|
||||||
|
// runtime is, it speaks for the module; where it is not, the module cannot speak at all,
|
||||||
|
// and a user would not change that.
|
||||||
|
if d.NoAccount {
|
||||||
|
continue
|
||||||
|
}
|
||||||
out = append(out, Principal{
|
out = append(out, Principal{
|
||||||
Kind: KindModule, Node: node, Module: d.Module,
|
Kind: KindModule, Node: node, Module: d.Module,
|
||||||
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
|
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
|
||||||
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
||||||
|
State: stateNames(d.State), Reads: d.Reads,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if runtimeHere {
|
if runtimeHere {
|
||||||
|
|||||||
@@ -296,3 +296,57 @@ func TestTheRuntimeModuleBecomesTheMachinesRuntimePrincipal(t *testing.T) {
|
|||||||
t.Error("telegram lost its own principal when the runtime arrived on its node")
|
t.Error("telegram lost its own principal when the runtime arrived on its node")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A module that declares nowhere to read an account is no user: it could never be issued one, so it
|
||||||
|
// was only ever named as a credential the mesh had not minted (novox/hq issue 195). Where the runtime
|
||||||
|
// is, the runtime still carries it — its grants are the runtime's.
|
||||||
|
func TestAModuleThatCannotReadAnAccountIsNoUser(t *testing.T) {
|
||||||
|
r := someRecords()
|
||||||
|
r.Assigned["one"] = append(r.Assigned["one"],
|
||||||
|
Declared{Module: "packet-filter", NoAccount: true, Emits: []string{"rule.changed"}},
|
||||||
|
Declared{Module: RuntimeModule})
|
||||||
|
users, err := Users(r)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, u := range users {
|
||||||
|
if u.Username() == "one.packet-filter" {
|
||||||
|
t.Fatalf("packet-filter declares no broker secret and was composed as a user: %v", namesOf(t, r))
|
||||||
|
}
|
||||||
|
if u.Kind == KindNodeTools {
|
||||||
|
carried := false
|
||||||
|
for _, d := range u.Carries {
|
||||||
|
carried = carried || d.Module == "packet-filter"
|
||||||
|
}
|
||||||
|
if !carried {
|
||||||
|
t.Error("the runtime stopped carrying a module that has no account of its own")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if names := strings.Join(namesOf(t, r), ","); !strings.Contains(names, "one.telegram") {
|
||||||
|
t.Errorf("a module that does read an account lost its user: %s", names)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A carried module with no account still has its consumer made: the runtime reads it on the module's
|
||||||
|
// behalf (ADR 0198), and the consumer was derived from the module's own user until issue 195 took
|
||||||
|
// that user away.
|
||||||
|
func TestACarriedModuleWithNoAccountStillHasItsConsumer(t *testing.T) {
|
||||||
|
r := someRecords()
|
||||||
|
r.Assigned["one"] = append(r.Assigned["one"],
|
||||||
|
Declared{Module: "listener", NoAccount: true, Consumes: []string{"shop.order.placed"}},
|
||||||
|
Declared{Module: RuntimeModule})
|
||||||
|
r.Assigned["two"] = append(r.Assigned["two"],
|
||||||
|
Declared{Module: "auditor", Consumes: []string{"shop.order.placed"}})
|
||||||
|
users, err := Users(r)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := map[string]int{}
|
||||||
|
for _, c := range ConsumersOf(users) {
|
||||||
|
got[c.Node+"/"+c.Module]++
|
||||||
|
}
|
||||||
|
if got["one/listener"] != 1 || got["two/auditor"] != 1 || len(got) != 2 {
|
||||||
|
t.Fatalf("consumers: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -215,7 +215,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
||||||
for _, a := range artifacts {
|
for _, a := range artifacts {
|
||||||
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
||||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, seatBases, say)
|
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, say)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
say("artifact", "%s FAILED: %v", a.Name, err)
|
say("artifact", "%s FAILED: %v", a.Name, err)
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
@@ -443,7 +443,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
|
|||||||
|
|
||||||
func one(ctx context.Context, run Runner, publish Publisher,
|
func one(ctx context.Context, run Runner, publish Publisher,
|
||||||
module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
|
module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
|
||||||
held map[string]string, npmrc string, seats map[string]string,
|
held map[string]string, npmrc string, registry Npmrc, seats map[string]string,
|
||||||
say func(step, format string, args ...any)) (catalogue.Built, error) {
|
say func(step, format string, args ...any)) (catalogue.Built, error) {
|
||||||
|
|
||||||
switch a.Kind {
|
switch a.Kind {
|
||||||
@@ -582,6 +582,11 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
"holds no copy of it. Build %s first",
|
"holds no copy of it. Build %s first",
|
||||||
module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
|
module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
|
||||||
}
|
}
|
||||||
|
// The module's own packages first, where the compiler and the bundler resolve them from
|
||||||
|
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
|
||||||
|
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
|
||||||
|
return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
|
||||||
|
}
|
||||||
say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base)
|
say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base)
|
||||||
compiled, err := compile(ctx, run, tree, chain, base, a)
|
compiled, err := compile(ctx, run, tree, chain, base, a)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -756,6 +761,9 @@ func Command(ctx context.Context, dir, name string, args ...string) (string, err
|
|||||||
tell("run", "$ (%s) %s %s", short(filepath.Base(dir)), name, strings.Join(args, " "))
|
tell("run", "$ (%s) %s %s", short(filepath.Base(dir)), name, strings.Join(args, " "))
|
||||||
cmd := exec.CommandContext(ctx, name, args...)
|
cmd := exec.CommandContext(ctx, name, args...)
|
||||||
cmd.Dir = dir
|
cmd.Dir = dir
|
||||||
|
// **Ended whole when the build is** (novox/hq ADR 0219): its own process group, killed as one
|
||||||
|
// when the context ends, so a build killed by hand leaves no `git` or `docker` child running.
|
||||||
|
inItsOwnGroup(cmd)
|
||||||
out, err := cmd.CombinedOutput()
|
out, err := cmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
tell("run", "! %s %s failed after %s", name, args[0], since(started))
|
tell("run", "! %s %s failed after %s", name, args[0], since(started))
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's own packages, installed before its bundle is compiled, so the bundler inlines them.
|
||||||
|
//
|
||||||
|
// **A bundle could only import what the toolchain happened to carry.** The compiler and the bundler
|
||||||
|
// resolve an import by walking up from the module's source: the module's own directory first, then
|
||||||
|
// the toolchain image's node_modules. Nothing ever put anything in the first, so a module needing a
|
||||||
|
// database driver (`pg`, `mongodb`, `mssql`) could not be a bundle at all, and kept a container whose
|
||||||
|
// recipe installed it by hand (novox/hq ADR 0198 §4: "the backend's own driver inside the bundle").
|
||||||
|
// Now the module's `package.json` says what it depends on, as any Node package does, and the build
|
||||||
|
// installs exactly that into the module's own directory before compiling.
|
||||||
|
//
|
||||||
|
// **The SDK the toolchain carries is the one a bundle is built with, whatever the module says**
|
||||||
|
// (novox/hq issue 212: the toolchain is rebuilt on every SDK release and every bundle after it). A
|
||||||
|
// module's `package.json` names `@novox/mesh-sdk` with a range — it has to, to type-check on a
|
||||||
|
// workstation — and installing that range would shadow the toolchain's copy for this module alone:
|
||||||
|
// one module compiled against an older SDK than its neighbours, chosen by a caret nobody re-reads.
|
||||||
|
// So the SDK is taken out of what is installed (and never fetched), and any copy something else
|
||||||
|
// pulls in is removed afterwards; every import of it resolves past the module's node_modules to the
|
||||||
|
// toolchain's. A module therefore cannot pin a different SDK, by design: the toolchain is the pin.
|
||||||
|
//
|
||||||
|
// **Correctness before speed.** Every build installs afresh into a fresh clone, from the lockfile
|
||||||
|
// when the module has one (`npm ci`, exact) and from its ranges otherwise; nothing installed is kept
|
||||||
|
// between builds. What is shared is npm's own download cache, a named volume, which is
|
||||||
|
// content-addressed and verified by integrity on every read — it saves the network, never the
|
||||||
|
// install. Install scripts do not run: the build node runs nobody's postinstall, and what a script
|
||||||
|
// would build natively could not be inlined into one file anyway.
|
||||||
|
|
||||||
|
// sdkPackage is the package a TypeScript bundle's launcher serves through, and the one package a
|
||||||
|
// module's own dependencies never supply (above).
|
||||||
|
const sdkPackage = "@novox/mesh-sdk"
|
||||||
|
|
||||||
|
// npmCache is the named volume npm's download cache lives in across builds on one build node.
|
||||||
|
const npmCache = "mesh-builder-npm-cache"
|
||||||
|
|
||||||
|
// ownDependencies is what a module's package.json depends on beyond the SDK, sorted; nothing when
|
||||||
|
// the module has no package.json or depends on nothing else — which builds exactly as before.
|
||||||
|
func ownDependencies(tree string) ([]string, error) {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(tree, "package.json"))
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var p struct {
|
||||||
|
Dependencies map[string]string `json:"dependencies"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &p); err != nil {
|
||||||
|
return nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for name := range p.Dependencies {
|
||||||
|
if name != sdkPackage {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// installSteps is the script run inside the toolchain image, from the module's own directory ($0).
|
||||||
|
// It works in a scratch copy so the module's package.json and lockfile are never rewritten, takes
|
||||||
|
// the SDK out of what is installed, installs production dependencies only, removes any copy of the
|
||||||
|
// SDK something pulled in, and puts the result at the module's node_modules.
|
||||||
|
const installSteps = `set -e
|
||||||
|
work="$(mktemp -d)"
|
||||||
|
cp "$0/package.json" "$work/"
|
||||||
|
if [ -f "$0/package-lock.json" ]; then cp "$0/package-lock.json" "$work/"; fi
|
||||||
|
cd "$work"
|
||||||
|
node -e '
|
||||||
|
const fs = require("fs"), sdk = process.argv[1];
|
||||||
|
const p = JSON.parse(fs.readFileSync("package.json", "utf8"));
|
||||||
|
for (const k of ["dependencies", "peerDependencies", "optionalDependencies"]) if (p[k]) delete p[k][sdk];
|
||||||
|
delete p.devDependencies; delete p.scripts;
|
||||||
|
fs.writeFileSync("package.json", JSON.stringify(p));
|
||||||
|
' "$1"
|
||||||
|
shift
|
||||||
|
if [ -f package-lock.json ]; then
|
||||||
|
npm ci --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund "$@"
|
||||||
|
else
|
||||||
|
npm install --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund --no-package-lock "$@"
|
||||||
|
fi
|
||||||
|
find node_modules -depth -type d -path "*/node_modules/@novox/mesh-sdk" -exec rm -rf {} +
|
||||||
|
rm -rf "$0/node_modules"
|
||||||
|
cp -a node_modules "$0/node_modules"
|
||||||
|
`
|
||||||
|
|
||||||
|
// installOwn installs a TypeScript module's own production dependencies into its directory, in the
|
||||||
|
// toolchain image, before the compile — or does nothing at all for a module that has none.
|
||||||
|
func installOwn(ctx context.Context, run Runner, tree string, chain Toolchain, base string,
|
||||||
|
registry Npmrc, say func(step, format string, args ...any)) error {
|
||||||
|
if chain.Language != "typescript" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
deps, err := ownDependencies(tree)
|
||||||
|
if err != nil || len(deps) == 0 {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
scoped := strings.TrimSpace(registry.Scope)
|
||||||
|
if !registry.Enabled() {
|
||||||
|
// **No registry, no scoped package.** Without the mesh's registry a scoped name resolves on
|
||||||
|
// the public one, where anybody may have published it: a dependency that installs is not
|
||||||
|
// the dependency the module meant.
|
||||||
|
for _, d := range deps {
|
||||||
|
if strings.HasPrefix(d, "@novox/") {
|
||||||
|
return fmt.Errorf("the module depends on %s, and this build knows no package registry "+
|
||||||
|
"for its scope; it would resolve from the public registry, which is not where the "+
|
||||||
|
"mesh publishes it", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const within = "/app/modules/module"
|
||||||
|
invocation := []string{"run", "--rm",
|
||||||
|
"--volume", tree + ":" + within,
|
||||||
|
"--volume", npmCache + ":/root/.npm",
|
||||||
|
"--workdir", within}
|
||||||
|
var flags []string
|
||||||
|
if registry.Enabled() {
|
||||||
|
// The registry is reached where the binding says it is, which may be this machine's own
|
||||||
|
// loopback — the reason an image build that resolves packages runs on the host network too.
|
||||||
|
invocation = append(invocation, "--network", "host")
|
||||||
|
reg := strings.TrimSpace(registry.Registry)
|
||||||
|
if !strings.HasSuffix(reg, "/") {
|
||||||
|
reg += "/"
|
||||||
|
}
|
||||||
|
flags = append(flags, "--"+scoped+":registry="+reg)
|
||||||
|
}
|
||||||
|
invocation = append(invocation, base, "sh", "-c", installSteps, within, sdkPackage)
|
||||||
|
invocation = append(invocation, flags...)
|
||||||
|
say("bundle", "installing the module's own packages: %s", strings.Join(deps, ", "))
|
||||||
|
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
|
||||||
|
return fmt.Errorf("installing the module's own packages (%s): %w", strings.Join(deps, ", "), err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's own packages (dependencies.go): installed into its own directory, in the toolchain,
|
||||||
|
// before the compile, so the bundler inlines them — the SDK always the toolchain's.
|
||||||
|
|
||||||
|
func buildWithPackageJSON(t *testing.T, pkg string, extra map[string]string, registry Npmrc) (*recorded, error) {
|
||||||
|
t.Helper()
|
||||||
|
files := map[string]string{"index.ts": "console.log(1)"}
|
||||||
|
if pkg != "" {
|
||||||
|
files["package.json"] = pkg
|
||||||
|
}
|
||||||
|
for k, v := range extra {
|
||||||
|
files[k] = v
|
||||||
|
}
|
||||||
|
r, workspace := aRepository(t, aBundle, files)
|
||||||
|
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
|
||||||
|
_, err := Build(context.Background(), compiling{r}.run, r,
|
||||||
|
"https://forge.invalid/greeter.git", "", "", workspace, held, registry, GitCredential{}, nil)
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func installs(r *recorded) []string {
|
||||||
|
var out []string
|
||||||
|
for _, line := range r.ran {
|
||||||
|
if strings.HasPrefix(line, "docker run") && strings.Contains(line, "npm ci") {
|
||||||
|
out = append(out, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func compileIndex(r *recorded) int {
|
||||||
|
for i, line := range r.ran {
|
||||||
|
if strings.Contains(line, "--outDir") {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAModulesOwnPackagesAreInstalledInTheToolchainBeforeTheCompile(t *testing.T) {
|
||||||
|
r, err := buildWithPackageJSON(t, `{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0","pg":"^8"},"devDependencies":{"typescript":"^5"}}`,
|
||||||
|
nil, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := installs(r)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("want one install of the module's own packages:\n%s", strings.Join(r.ran, "\n"))
|
||||||
|
}
|
||||||
|
line := got[0]
|
||||||
|
for _, want := range []string{
|
||||||
|
"mesh-tools/build@sha256:", // in the toolchain image
|
||||||
|
":/app/modules/module", // into the module's own directory
|
||||||
|
"--workdir /app/modules/module", //
|
||||||
|
npmCache + ":/root/.npm", // npm's verified download cache, and only that
|
||||||
|
"--omit=dev", "--ignore-scripts", // production packages, no build-node scripts
|
||||||
|
"npm ci", "npm install", "--no-package-lock", // the lockfile when there is one, else the ranges
|
||||||
|
"--@novox:registry=https://forge.invalid/api/packages/novox/npm/", // the scope from the mesh's registry
|
||||||
|
"--network host",
|
||||||
|
"@novox/mesh-sdk", // named, to be taken out of what is installed
|
||||||
|
} {
|
||||||
|
if !strings.Contains(line, want) {
|
||||||
|
t.Errorf("the install lacks %q:\n%s", want, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The SDK is the toolchain's: never installed from the module's range, and any copy removed.
|
||||||
|
if !strings.Contains(line, `delete p[k][sdk]`) || !strings.Contains(line, `-path "*/node_modules/@novox/mesh-sdk" -exec rm -rf`) {
|
||||||
|
t.Errorf("the module's own SDK range could shadow the toolchain's SDK:\n%s", line)
|
||||||
|
}
|
||||||
|
if i, c := strings.Index(strings.Join(r.ran, "\n"), "npm ci"), compileIndex(r); c < 0 ||
|
||||||
|
i > strings.Index(strings.Join(r.ran, "\n"), "--outDir") {
|
||||||
|
t.Fatalf("the install did not run before the compile:\n%s", strings.Join(r.ran, "\n"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A module with nothing beyond the SDK builds exactly as before**: the same commands, no install.
|
||||||
|
func TestAModuleDependingOnlyOnTheSDKBuildsExactlyAsBefore(t *testing.T) {
|
||||||
|
without, err := buildWithPackageJSON(t, "", nil, Npmrc{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, pkg := range []string{
|
||||||
|
`{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0"},"devDependencies":{"typescript":"^5"}}`,
|
||||||
|
`{"type":"module"}`,
|
||||||
|
} {
|
||||||
|
with, err := buildWithPackageJSON(t, pkg, map[string]string{"package-lock.json": "{}"}, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/npm/"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.Join(with.ran, "\n"), "npm ") {
|
||||||
|
t.Fatalf("a module depending on nothing but the SDK ran npm:\n%s", strings.Join(with.ran, "\n"))
|
||||||
|
}
|
||||||
|
if len(with.ran) != len(without.ran) {
|
||||||
|
t.Fatalf("a module depending only on the SDK built differently from one with no package.json:\n%s\n---\n%s",
|
||||||
|
strings.Join(with.ran, "\n"), strings.Join(without.ran, "\n"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without the mesh's registry a scoped package would resolve on the public one: refused by name.
|
||||||
|
func TestAScopedPackageWithNoRegistryIsRefused(t *testing.T) {
|
||||||
|
r, err := buildWithPackageJSON(t, `{"dependencies":{"@novox/mesh-sdk":"^0.1.0","@novox/other":"^1"}}`, nil, Npmrc{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "@novox/other") {
|
||||||
|
t.Fatalf("a scoped package was installed with no registry for its scope: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.Join(r.ran, "\n"), "--outDir") {
|
||||||
|
t.Fatal("the compile ran after the refusal")
|
||||||
|
}
|
||||||
|
// A public package installs without one, from the public registry and nothing else.
|
||||||
|
r, err = buildWithPackageJSON(t, `{"dependencies":{"mssql":"^11"}}`, nil, Npmrc{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := installs(r); len(got) != 1 || strings.Contains(got[0], ":registry=") || strings.Contains(got[0], "--network host") {
|
||||||
|
t.Fatalf("a public package's install: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnUnreadablePackageJSONIsRefusedByName(t *testing.T) {
|
||||||
|
_, err := buildWithPackageJSON(t, `{"dependencies":`, nil, Npmrc{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "package.json") {
|
||||||
|
t.Fatalf("a broken package.json was not refused by name: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A build killed by hand (novox/hq ADR 0219).
|
||||||
|
//
|
||||||
|
// A build is a tree of commands — git, then docker, and docker's own children — and a container the
|
||||||
|
// docker client started keeps running when the client dies. So ending one by hand is three things:
|
||||||
|
// the build's context is cancelled, which kills each command's whole process group rather than the
|
||||||
|
// one process the context knows; every container the build started carries the build's id as a
|
||||||
|
// label, so what outlived its client is found and removed by that label; and the holder announces
|
||||||
|
// the outcome itself, since nothing else will.
|
||||||
|
|
||||||
|
// BuildLabel is the label every container a build starts carries, valued with the build's id.
|
||||||
|
const BuildLabel = "mesh.build"
|
||||||
|
|
||||||
|
// KillWait is how long a command killed with its build may take to let go of its output before it
|
||||||
|
// is abandoned: a grandchild holding the pipe open must not hold the build open with it.
|
||||||
|
const KillWait = 10 * time.Second
|
||||||
|
|
||||||
|
// inItsOwnGroup makes a command the leader of its own process group, killed as a group when its
|
||||||
|
// context ends.
|
||||||
|
func inItsOwnGroup(cmd *exec.Cmd) {
|
||||||
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
|
cmd.Cancel = func() error {
|
||||||
|
if cmd.Process == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// The negative pid is the group: the command and everything it started.
|
||||||
|
if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
|
||||||
|
return cmd.Process.Kill()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cmd.WaitDelay = KillWait
|
||||||
|
}
|
||||||
|
|
||||||
|
// Labelled is a Runner that marks every container a build starts with the build's id, so a kill
|
||||||
|
// finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
|
||||||
|
// passes rather than at each `docker run` the builder composes: a run added later is labelled too.
|
||||||
|
func Labelled(run Runner, id string) Runner {
|
||||||
|
return func(ctx context.Context, dir string, name string, args ...string) (string, error) {
|
||||||
|
return run(ctx, dir, name, LabelledArgs(name, args, id)...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// LabelledArgs is a command's arguments with the build's label added, when it is a `docker run`.
|
||||||
|
func LabelledArgs(name string, args []string, id string) []string {
|
||||||
|
if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(args)+2)
|
||||||
|
out = append(out, "run", "--label", BuildLabel+"="+id)
|
||||||
|
return append(out, args[1:]...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveContainersOf removes every container labelled with the build's id, running or not, and
|
||||||
|
// says how many. Run with a context of its own: the build's is the one that was just cancelled.
|
||||||
|
func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error) {
|
||||||
|
out, err := run(ctx, "", "docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
ids := strings.Fields(out)
|
||||||
|
if len(ids) == 0 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return len(ids), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A build killed by hand (novox/hq ADR 0219) ends every command it started: the context's end kills
|
||||||
|
// the command's whole process group, not the one process the context knows about.
|
||||||
|
func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
child := filepath.Join(dir, "child")
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
began := time.Now()
|
||||||
|
go func() {
|
||||||
|
// A shell that starts a grandchild and waits on it: killing the shell alone would leave the
|
||||||
|
// grandchild running, holding the output open.
|
||||||
|
_, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
var pid int
|
||||||
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
|
||||||
|
if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" {
|
||||||
|
pid, _ = strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pid == 0 {
|
||||||
|
t.Fatal("the command never started its child")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a killed command reported success")
|
||||||
|
}
|
||||||
|
case <-time.After(KillWait + 5*time.Second):
|
||||||
|
t.Fatal("the killed command never returned")
|
||||||
|
}
|
||||||
|
if took := time.Since(began); took > KillWait {
|
||||||
|
t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took)
|
||||||
|
}
|
||||||
|
for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
|
||||||
|
if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = syscall.Kill(pid, syscall.SIGKILL)
|
||||||
|
t.Fatalf("the grandchild %d outlived the kill", pid)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every `docker run` a build starts carries its id as a label; nothing else is touched.
|
||||||
|
func TestEveryContainerABuildStartsCarriesItsID(t *testing.T) {
|
||||||
|
got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
|
||||||
|
if want := []string{"run", "--label", "mesh.build=build-1", "--rm", "img", "sh"}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("docker run became %v", got)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
}{{"docker", []string{"build", "."}}, {"git", []string{"run"}}, {"docker", nil}} {
|
||||||
|
if got := LabelledArgs(c.name, c.args, "build-1"); !reflect.DeepEqual(got, c.args) {
|
||||||
|
t.Errorf("%s %v became %v", c.name, c.args, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var ran [][]string
|
||||||
|
run := Labelled(func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||||
|
ran = append(ran, append([]string{name}, args...))
|
||||||
|
return "", nil
|
||||||
|
}, "build-2")
|
||||||
|
_, _ = run(context.Background(), "", "docker", "run", "img")
|
||||||
|
if want := [][]string{{"docker", "run", "--label", "mesh.build=build-2", "img"}}; !reflect.DeepEqual(ran, want) {
|
||||||
|
t.Errorf("ran %v", ran)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What a kill removes is found by the label, and only what it finds.
|
||||||
|
func TestAKillRemovesTheContainersLabelledWithTheBuild(t *testing.T) {
|
||||||
|
var ran []string
|
||||||
|
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||||
|
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||||
|
if args[0] == "ps" {
|
||||||
|
return "c1\nc2\n", nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
n, err := RemoveContainersOf(context.Background(), run, "build-3")
|
||||||
|
if err != nil || n != 2 {
|
||||||
|
t.Fatalf("%d %v", n, err)
|
||||||
|
}
|
||||||
|
if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker rm -f c1 c2"}; !reflect.DeepEqual(ran, want) {
|
||||||
|
t.Errorf("ran %v", ran)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,6 +7,8 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Where built artifacts go.
|
// Where built artifacts go.
|
||||||
@@ -66,22 +68,32 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string)
|
|||||||
return pinned, nil
|
return pinned, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PublishArchive stores bytes as a blob and returns where to fetch them from.
|
// PublishArchive stores bytes as a blob, holds it by a manifest, and returns where to fetch them
|
||||||
|
// from.
|
||||||
//
|
//
|
||||||
// Two steps, which is the registry's own protocol: ask for somewhere to put it, then put it there
|
// Two steps for the blob, which is the registry's own protocol: ask for somewhere to put it, then
|
||||||
// naming the digest. The registry verifies the digest itself, so a blob that arrived corrupted is
|
// put it there naming the digest. The registry verifies the digest itself, so a blob that arrived
|
||||||
// refused by the thing storing it rather than by the machine unpacking it a week later.
|
// corrupted is refused by the thing storing it rather than by the machine unpacking it a week
|
||||||
|
// later.
|
||||||
|
//
|
||||||
|
// **Then a manifest that names it** (novox/hq issue 253, ADR 0189). The store's own collector
|
||||||
|
// marks only from manifests, and a blob no manifest names is collected however much the mesh
|
||||||
|
// means to keep it — so an archive published bare is an archive the first nightly collection
|
||||||
|
// deletes. The holder is composed from the digest and size alone (artifacts.Holder), which is
|
||||||
|
// what lets the sweep recompute it to backfill or let go without anything being recorded here.
|
||||||
|
// What a machine is told to fetch is the blob, exactly as before.
|
||||||
func (r Registry) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
func (r Registry) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
||||||
base := "http://" + r.Address + "/v2/" + repository
|
base := "http://" + r.Address + "/v2/" + repository
|
||||||
final := base + "/blobs/" + digest
|
final := base + "/blobs/" + digest
|
||||||
|
|
||||||
// Already there. Blobs are immutable and named by their content, so this is not an
|
// Already there. Blobs are immutable and named by their content, so this is not an
|
||||||
// optimisation — re-uploading would be asking the registry to store what it already has under
|
// optimisation — re-uploading would be asking the registry to store what it already has under
|
||||||
// the name it already has.
|
// the name it already has. It is still held: a blob published before holders existed is
|
||||||
|
// exactly the one a rebuild of the same source finds already there.
|
||||||
if there, err := r.has(ctx, final); err != nil {
|
if there, err := r.has(ctx, final); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
} else if there {
|
} else if there {
|
||||||
return final, nil
|
return final, r.hold(ctx, repository, digest, len(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/blobs/uploads/", nil)
|
start, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/blobs/uploads/", nil)
|
||||||
@@ -119,7 +131,17 @@ func (r Registry) PublishArchive(ctx context.Context, repository string, body []
|
|||||||
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||||
return "", fmt.Errorf("%s refused the blob: %s %s", base, done.Status, strings.TrimSpace(string(said)))
|
return "", fmt.Errorf("%s refused the blob: %s %s", base, done.Status, strings.TrimSpace(string(said)))
|
||||||
}
|
}
|
||||||
return final, nil
|
return final, r.hold(ctx, repository, digest, len(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// hold puts the manifest holding an archive beside it. A build whose archive could not be held is
|
||||||
|
// a failed build: recorded as published, it would be an archive the store's collector takes.
|
||||||
|
func (r Registry) hold(ctx context.Context, repository, digest string, size int) error {
|
||||||
|
store := artifacts.Store{Address: r.Address, HTTP: r.client()}
|
||||||
|
if _, err := store.HoldBlob(ctx, repository, digest, int64(size)); err != nil {
|
||||||
|
return fmt.Errorf("published %s/blobs/%s and could not hold it by a manifest: %w", repository, digest, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// has is whether this registry already holds what is at that URL.
|
// has is whether this registry already holds what is at that URL.
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user