novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 7 Releases Wiki Activity

Compare commits

merge into: novox/mesh-controller:fix/gate-baseline-composes
Branches Tags
novox/mesh-controller:main novox/mesh-controller:feat/delivery-checks-verb novox/mesh-controller:fix/a-recorded-build-waits-for-a-person novox/mesh-controller:feat/node-uplink-verbs novox/mesh-controller:feat/journal-window-on-the-seat novox/mesh-controller:feat/machine-network-health novox/mesh-controller:fix/vendor-the-merged-engine novox/mesh-controller:fix/gate-baseline-composes novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module novox/mesh-controller:fix/the-runtime-file-is-the-runtimes novox/mesh-controller:feat/no-change-takes-effect-unseen novox/mesh-controller:feat/build-edges-are-recorded novox/mesh-controller:fix/several-contributions-collide-in-grants-v2
..
pull from: novox/mesh-controller:fix/the-derived-value-record-is-0202
Branches Tags
novox/mesh-controller:feat/delivery-checks-verb novox/mesh-controller:fix/a-recorded-build-waits-for-a-person novox/mesh-controller:feat/node-uplink-verbs novox/mesh-controller:feat/journal-window-on-the-seat novox/mesh-controller:feat/machine-network-health novox/mesh-controller:fix/vendor-the-merged-engine novox/mesh-controller:main novox/mesh-controller:fix/gate-baseline-composes novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module novox/mesh-controller:fix/the-runtime-file-is-the-runtimes novox/mesh-controller:feat/no-change-takes-effect-unseen novox/mesh-controller:feat/build-edges-are-recorded novox/mesh-controller:fix/several-contributions-collide-in-grants-v2
1 Commits
fix/gate-baseline-composes .. fix/the-derived-value-record-is-0202
Author SHA1 Message Date
jschoubben 75b2676d32 The derived-value record is ADR 0202 (was 0201) 
The key-value-buckets record took 0201 on main while this waited to merge.
Only the comments citing the derived-value work move; this repository's other
0201 citations are the buckets record's own and stay.
 2026-10-04 04:33:56 +02:00
1432 changed files with 1491 additions and 671885 deletions
Expand all files Collapse all files
Dockerfile
+3 -5
View File
@@ -21,15 +21,13 @@ ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7
FROM ${GO_BASE} AS build
WORKDIR /src
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
# the image builds from this repository alone — the host's validator among them, whose module no
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
# Dependencies first, so a change to the source does not refetch them.
COPY go.mod go.sum ./
COPY vendor/ vendor/
RUN go mod download
COPY . .
ARG VERSION=development
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
RUN CGO_ENABLED=0 go build -trimpath \
-ldflags "-s -w -X main.version=${VERSION}" \
-o /mesh-controller ./cmd/mesh-controller
Makefile
+9 -9
View File
@@ -102,18 +102,18 @@ proxy-image:
# The whole gate. Raises a database, runs everything against it, and takes it down again --
# including when the tests fail, which is why the teardown is not conditional.
#
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
# It was one package at a time against one shared bus, because the live tests assert, read and remove
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
# was reading, as "no response from stream". That reads as a bug in the code under test.
check: fmt vet postgres
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
# the gate.
# Without a database the live tests skip rather than fail, so this is the honest subset and not
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
test:
go test -timeout 15m ./...
go test -p 1 ./...
vet:
go vet ./...
cmd/mesh-builder/holder.go
-386
View File
@@ -1,386 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/micro"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
//
// **The queue is the controller's; the build running here is this machine's.** The controller can
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
// on this machine and containers in this machine's runtime, and only this machine can end them. So
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
// it, pause, resume.
//
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
// set per build — so "the build running here" is one or none, and kill names it by id so a call
// that arrives as one build ends and the next begins cannot end the wrong one.
// holder is this machine's state as a holder of the build seat.
type holder struct {
on, seat string
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
// rather than silently taking work again.
workspace string
// say publishes this machine's state: whether it takes work (link.HolderState).
say func(link.HolderState) error
// remove runs what a kill needs outside the build: the containers left behind.
remove builder.Runner
mu sync.Mutex
paused bool
running *running
}
// running is the build this machine is doing.
type running struct {
request link.BuildRequest
step string
started time.Time
cancel context.CancelFunc
killed bool
// returned is the build's own work having ended, before a kill or not; outcome is what was then
// announced, and announced whether it went out.
returned bool
outcome string
announced bool
done chan struct{}
}
// pausedFile is where the flag lives in the workspace.
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
// newHolder reads the paused flag the workspace keeps.
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
if _, err := os.Stat(pausedFile(workspace)); err == nil {
h.paused = true
}
return h
}
// Paused is asked by the taking loop before every fetch.
func (h *holder) Paused() bool {
h.mu.Lock()
defer h.mu.Unlock()
return h.paused
}
// setPaused records the flag in the workspace first and then in memory, so what this holder says
// it is and what it would be after a restart never differ.
func (h *holder) setPaused(paused bool) error {
path := pausedFile(h.workspace)
if paused {
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
return err
}
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
}
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
}
h.mu.Lock()
h.paused = paused
h.mu.Unlock()
h.announce()
return nil
}
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
// controller reading an older state is a plan read as late rather than a build lost.
func (h *holder) announce() {
if h.say == nil {
return
}
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
}
}
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
// a pause outlives the events stream's retention.
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
h.announce()
tick := time.NewTicker(every)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if h.Paused() {
h.announce()
}
}
}
}
// begin records the build this machine took, with the cancel that ends it.
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
h.mu.Lock()
h.running = r
h.mu.Unlock()
return r
}
// end clears it, and lets a kill waiting on it know it is over.
func (h *holder) end(r *running) {
h.mu.Lock()
if h.running == r {
h.running = nil
}
h.mu.Unlock()
close(r.done)
}
// stepped records the step a build is at, for `current`.
func (h *holder) stepped(r *running, step string) {
h.mu.Lock()
r.step = step
h.mu.Unlock()
}
// currentBuild is what `current` answers.
type currentBuild struct {
On string `json:"on"`
Paused bool `json:"paused"`
Running *struct {
ID string `json:"id"`
Repository string `json:"repository"`
Path string `json:"path,omitempty"`
Ref string `json:"ref,omitempty"`
Step string `json:"step,omitempty"`
Started string `json:"started"`
Elapsed string `json:"elapsed"`
} `json:"running,omitempty"`
Said string `json:"said"`
}
func (h *holder) current() currentBuild {
h.mu.Lock()
defer h.mu.Unlock()
out := currentBuild{On: h.on, Paused: h.paused}
taking := "taking builds"
if h.paused {
taking = "paused, taking no new build"
}
if h.running == nil {
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
return out
}
r := h.running
elapsed := time.Since(r.started).Round(time.Second)
out.Running = &struct {
ID string `json:"id"`
Repository string `json:"repository"`
Path string `json:"path,omitempty"`
Ref string `json:"ref,omitempty"`
Step string `json:"step,omitempty"`
Started string `json:"started"`
Elapsed string `json:"elapsed"`
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
r.started.UTC().Format(time.RFC3339), elapsed.String()}
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
return out
}
// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
// (killAnswer in the controller's queue.go, 75s).
var (
killRemoves = 15 * time.Second
killWaits = 20 * time.Second
)
// kill ends the build with this id, if it is the one running here and still working: its context
// cancelled — which kills each command's process group — and the containers it started removed by
// their label, once at once and again after the build has ended, so one created while it was being
// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
// ask so it is not redelivered; the answer says whether that happened.
func (h *holder) kill(id string) (string, error) {
h.mu.Lock()
r := h.running
if r == nil || r.request.ID != id {
doing := "nothing"
if r != nil {
doing = r.request.ID
}
h.mu.Unlock()
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
}
if r.returned {
h.mu.Unlock()
return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
}
r.killed = true
cancel, done := r.cancel, r.done
h.mu.Unlock()
cancel()
removed, removeErr := h.removeContainers(id)
ended := false
select {
case <-done:
ended = true
case <-time.After(killWaits):
}
again, againErr := h.removeContainers(id)
removed += again
if removeErr == nil {
removeErr = againErr
}
containers := fmt.Sprintf("%d container(s) it started removed", removed)
if removeErr != nil {
containers = "its containers could not all be listed or removed: " + removeErr.Error()
}
if !ended {
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
"its outcome is in", id, h.on, containers), nil
}
h.mu.Lock()
outcome, announced := r.outcome, r.announced
h.mu.Unlock()
if !announced {
return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
containers), nil
}
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
}
// removeContainers is one pass of removing what the build left, bounded.
func (h *holder) removeContainers(id string) (int, error) {
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
defer stop()
return builder.RemoveContainersOf(cleanup, h.remove, id)
}
// returned records that the build's work ended, and says whether a kill came first — only then is
// the build killed; an error it ended with on its own is its own outcome.
func (h *holder) returned(r *running) bool {
h.mu.Lock()
defer h.mu.Unlock()
r.returned = true
return r.killed
}
// said records the outcome announced, and whether it went out, for a kill to answer with.
func (h *holder) said(r *running, outcome string, announced bool) {
h.mu.Lock()
r.outcome, r.announced = outcome, announced
h.mu.Unlock()
}
// handlers are the seat's verbs, as this machine answers them.
func (h *holder) handlers() map[string]link.ToolHandler {
return map[string]link.ToolHandler{
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
var args struct {
ID string `json:"id"`
}
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
return nil, errors.New("kill needs the build's id")
}
said, err := h.kill(strings.TrimSpace(args.ID))
if err != nil {
return nil, err
}
return map[string]any{"said": said}, nil
},
"pause": func(context.Context, json.RawMessage) (any, error) {
if err := h.setPaused(true); err != nil {
return nil, err
}
said := h.on + " is paused: it takes no new build until resumed"
if c := h.current(); c.Running != nil {
said += "; " + c.Running.ID + " runs on and finishes"
}
return map[string]any{"said": said, "paused": true}, nil
},
"resume": func(context.Context, json.RawMessage) (any, error) {
if err := h.setPaused(false); err != nil {
return nil, err
}
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
},
}
}
func orNothing(s string) string {
if s == "" {
return "its start"
}
return s
}
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
// log it would be is the one being ended.
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
cmd := exec.CommandContext(ctx, name, args...)
cmd.Dir = dir
out, err := cmd.CombinedOutput()
if err != nil {
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
}
return string(out), nil
}
// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
// the module answering, the seat, scope node, the machine, its description and argument schema — so
// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
//
// One service per machine, named for the seat and identified by the machine, so the answer is this
// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
// store, and what it serves is what this binary was built to serve.
func announcement(seat, module, node string) micro.Info {
s, _ := catalogue.SeatNamed(seat)
var endpoints []micro.EndpointInfo
for _, v := range s.Serves {
schema, _ := json.Marshal(v.Input)
endpoints = append(endpoints, micro.EndpointInfo{
Name: seat + "__" + v.Name,
Subject: link.NodeSeatToolSubject(seat, v.Name, node),
// The queue group the verbs are served in, as every runtime announces its own.
QueueGroup: "seat." + seat,
Metadata: map[string]string{
"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
},
})
}
return micro.Info{
ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
Endpoints: endpoints,
}
}
// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
func moduleOf(user string) string {
if _, module, ok := strings.Cut(user, "."); ok && module != "" {
return module
}
return "build-agent"
}
cmd/mesh-builder/holder_test.go
-148
View File
@@ -1,148 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
// 0219), and what it says about itself follows.
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
workspace := t.TempDir()
var said []link.HolderState
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
said = append(said, s)
return nil
})
if h.Paused() {
t.Fatal("a new holder starts paused")
}
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
t.Fatal(err)
}
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
}
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
if !again.Paused() {
t.Fatal("restarted, the holder forgot it was paused")
}
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
t.Fatal(err)
}
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
t.Fatal("resumed, the holder came back paused")
}
}
// kill ends the build with that id — its context, which ends its commands — removes what it left by
// label, and refuses an id it is not building.
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
var removed []string
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
removed = append(removed, name+" "+strings.Join(args, " "))
if args[0] == "ps" {
return "c1\n", nil
}
return "", nil
}
kill := h.handlers()["kill"]
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
t.Fatal("killed a build while none ran")
}
building, cancel := context.WithCancel(context.Background())
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
h.stepped(r, "image")
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
t.Fatalf("current says %+v", c)
}
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
!strings.Contains(err.Error(), "build-1") {
t.Fatalf("killed another id: %v", err)
}
// The build's own goroutine: it ends when its context does, as a build's commands do, and
// announces what came of it.
go func() {
<-building.Done()
if h.returned(r) {
h.said(r, link.KilledByHand, true)
}
h.end(r)
}()
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
if err != nil {
t.Fatal(err)
}
if building.Err() == nil {
t.Fatal("the build's context was not cancelled")
}
if !r.killed {
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
}
said := answer.(map[string]any)["said"].(string)
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
t.Errorf("kill said %q", said)
}
// Removed at the kill and again once the build had ended: a container made in between is caught.
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
t.Errorf("removed %v", removed)
}
if c := h.current(); c.Running != nil {
t.Errorf("after the kill current says %+v", c)
}
}
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
}
kill := info.Endpoints[1]
md := kill.Metadata
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
t.Fatalf("kill is announced as %+v", kill)
}
body, err := json.Marshal(info)
if err != nil || len(body) > 8*1024 {
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
}
}
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
_, cancel := context.WithCancel(context.Background())
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
if killed := h.returned(r); killed {
t.Fatal("a build nobody killed reads as killed")
}
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
t.Fatalf("killed a build that had ended: %v", err)
}
h.end(r)
building, cancel := context.WithCancel(context.Background())
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
go func() {
<-building.Done()
if h.returned(r) {
h.said(r, link.KilledByHand, false)
}
h.end(r)
}()
said, err := h.kill("build-2")
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
t.Fatalf("kill said %q (%v)", said, err)
}
}
cmd/mesh-builder/main.go
+13 -136
View File
@@ -19,13 +19,11 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"net/url"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builder"
@@ -109,96 +107,48 @@ func run() error {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
js, seat, err := dialFor(credential)
machine, err := takeWorkFrom(credential, on)
if err != nil {
return err
}
defer js.Close()
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
// paused flag is read from the workspace before anything is taken, so a holder restarted while
// paused takes nothing.
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
body, err := json.Marshal(state)
if err != nil {
return err
}
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
return err
})
if h.Paused() {
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
}
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
defer machine.Close()
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
// one serves none, and a subscription its holder has no grant for would be refused for ever.
if seat == link.TheBuildMachine {
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
log.New(os.Stderr, "", 0))
if err != nil {
return err
}
defer stopServing()
// And says so, for the console to find (novox/hq ADR 0197).
stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
if err != nil {
return err
}
defer stopAnnouncing()
go h.stateWhilePaused(ctx, time.Hour)
}
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
publisher := builder.Registry{Address: registry, Run: builder.Command}
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
answer(ctx, publisher, on, workspace, work, h)
answer(ctx, publisher, on, workspace, work)
})
}
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
// holds.
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
//
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
// machine told about both would take work from one and answer on the other, and every log line would
// say it was fine.
func dialFor(credential Credential) (*broker.JetStream, string, error) {
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
// and that is enough to dial it, pinned.
if !credential.onTheNewBus() {
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
}
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
if err != nil {
return nil, "", err
return nil, err
}
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
// handover): the mesh issues a build machine's credential naming the seat its module claims,
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
seat := link.BuildSeatClaimed(credential.seatsClaimed())
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
return js, seat, nil
return link.MachineOverNATSOn(js, on, seat), nil
}
// answer does one build and says what happened, whichever way it went.
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
request := work.Request()
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
// reaches it through the parent, and that keeps today's meaning below.
building, cancel := context.WithCancel(ctx)
defer cancel()
var mine *running
if h != nil {
mine = h.begin(request, cancel)
defer h.end(mine)
}
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
@@ -212,9 +162,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
say := func(step, message string) {
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
work.Say(step, message)
if mine != nil && step != "run" && step != "output" {
h.stepped(mine, step)
}
}
builder.Said = say
defer func() { builder.Said = nil }()
@@ -224,7 +171,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
result := link.BuildResult{
ID: request.ID, Repository: request.Repository, Path: request.Path,
Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
Ref: request.Ref, On: on, Source: request.Source,
}
what := "building " + request.Repository
if request.Path != "" {
@@ -237,48 +184,19 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
npmrc, err := packagesFrom()
var built builder.Result
if request.Check != nil {
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
// published or registered; the verdict is the outcome.
result.Checked = request.Check
registry := ""
if r, ok := publisher.(builder.Registry); ok {
registry = r.Address
}
var v builder.CheckVerdict
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
if err == nil {
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
}
} else if err == nil {
if err == nil {
// The package-registry credential is a build input, so it is resolved before the clone: a
// build that could not have resolved its dependencies is refused in front of the reason, not
// after a clone that then fails at npm ci.
// Every container it starts is labelled with its id, so a kill finds what outlived the
// docker client (ADR 0219).
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
built, err = builder.Build(ctx, builder.Command, publisher,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(), say, request.Seats)
}
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
killed := false
if mine != nil {
killed = h.returned(mine) && err != nil
}
if killed {
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
// error it ended with is the kill's consequence, not a fault of the source.
result.Failed = link.KilledByHand
say("failed", link.KilledByHand)
} else if err != nil {
if err != nil {
// A failure is a result. A build that fails and says nothing is indistinguishable from a
// builder that is not running, and those want completely different responses.
result.Failed = err.Error()
say("failed", err.Error())
} else if request.Check != nil {
say("checked", result.Check.Verdict+": "+result.Check.Summary)
} else {
manifest, marshalErr := json.Marshal(built.Manifest)
if marshalErr != nil {
@@ -292,8 +210,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
})
}
result.Against = built.Against
result.SourceFingerprint = built.Source
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
for _, r := range built.Read {
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
}
@@ -301,21 +217,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
}
}
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
// to be built again. A machine being stopped is the other case and keeps its meaning: the
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
announcing := ctx
if killed {
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
defer stop()
announcing = fresh
}
announceErr := work.Announce(announcing, result)
if mine != nil {
h.said(mine, result.Failed, announceErr == nil)
}
if err := announceErr; err != nil {
if err := work.Announce(ctx, result); err != nil {
// Said, not fatal: the build happened. A build reported as failed because announcing it
// failed is a lie about work that was done — and the request stays unsettled below only if
// nothing was said at all, so another machine can try.
@@ -330,31 +232,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
}
}
// checkSpecOf is a check request as the builder runs it.
func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
c := request.Check
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
for dir, b := range c.Beside {
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
}
for _, m := range c.Members {
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
Ref: m.Ref, Paths: m.Paths})
}
return spec
}
// layerOf is one layer of a check as the outcome carries it.
func layerOf(l *builder.Layer) *link.CheckLayer {
if l == nil {
return nil
}
return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules}
}
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
// (novox/hq ADR 0076, issue 053).
//
cmd/mesh-builder/once.go
-2
View File
@@ -105,7 +105,6 @@ func buildOnce(ctx context.Context, args []string) error {
Ref: *ref,
Manifest: built.Manifest,
Against: built.Against,
Source: built.Source,
}
for _, r := range built.Read {
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
@@ -136,7 +135,6 @@ type onceResult struct {
Made []madeArtifact `json:"made"`
Against []string `json:"against,omitempty"`
Read []readRepository `json:"read,omitempty"`
Source string `json:"source-fingerprint,omitempty"`
}
// readRepository is a repository this build read source from besides the module's own.
cmd/mesh-controller/acting.go
-353
View File
@@ -1,353 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
//
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
// not happen. Three ways a process stands to the lease:
//
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
// objects, which are the controller's to write — waiting while another holds it, and renews it.
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
// its service manager restarts it as a candidate (serve, in push.go).
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
// between makes it stale and refused like any other. **When nobody holds the lease, the command
// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
// it, as it would for another controller.
// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
// refused nothing: there is nothing to order against, and nothing it does reaches a machine.
//
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
// instead would be a controller that can never send the user list that lets it act.
// actor is this process's standing to the lease.
type actor struct {
mu sync.Mutex
// held is the lease this process holds: the serving controller's, or a command's own.
held *lease.Lease
// unleased is why a serving controller acts without the lease; empty while it holds it or is not
// serving.
unleased string
// serving is a serving controller, which never borrows another's epoch.
serving bool
// kv is the lease bucket, for a command to read the holder's epoch from.
kv jetstream.KeyValue
close func()
// noBus is a process with no bus configured.
noBus bool
// reset is when the lease bucket was found raised again from nothing and its revisions moved past
// the highest epoch issued, and what was said of it; zero when it was not (S12).
reset time.Time
resetSaid string
}
// theLease is this process's standing to the lease.
var theLease = &actor{}
// instance names this process among controller instances: its machine, its process and when it
// started. The lease's holder and every call this process keeps carry it.
var instance = func() string {
host, _ := os.Hostname()
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
}()
// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
func (a *actor) epoch(ctx context.Context) (uint64, error) {
a.mu.Lock()
held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
a.mu.Unlock()
switch {
case held != nil:
return held.Epoch()
case serving && unleased != "":
return 0, nil
case serving:
return 0, lease.ErrNotHeld
case noBus:
return 0, nil
}
return a.forACommand(ctx)
}
// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
a.mu.Lock()
defer a.mu.Unlock()
if a.held != nil {
return a.held.Epoch()
}
if a.kv == nil {
address, err := broker.BusAddress()
if err != nil {
// No bus: this process reaches no machine, and has nothing to order against.
a.noBus = true
return 0, nil
}
js, err := broker.Dial(address)
if err != nil {
return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
"read and nothing is done: %w", err)
}
api, err := jetstream.New(js.Conn())
if err != nil {
js.Close()
return 0, err
}
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := broker.EnsureLeaseBucket(reading, api); err != nil {
js.Close()
return 0, err
}
kv, err := api.KeyValue(reading, broker.LeaseBucket)
if err != nil {
js.Close()
return 0, err
}
a.kv, a.close = kv, js.Close
if _, found, err := lease.Current(reading, kv); err == nil && !found {
// Nobody: this command takes it for as long as it runs.
l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
if err != nil {
return 0, err
}
epoch, err := l.TryTake(reading)
if err != nil {
return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
}
keeping, stop := context.WithCancel(context.Background())
go l.Keep(keeping)
a.held = l
closeBus := a.close
a.close = func() {
stop()
l.Release(context.Background())
closeBus()
}
return epoch, nil
}
}
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
holder, found, err := lease.Current(reading, a.kv)
if err != nil {
return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
}
if !found {
return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
"more is done under an epoch nobody holds — run the command again")
}
return holder.Epoch, nil
}
// release gives back what this process holds, at its end.
func (a *actor) release() {
a.mu.Lock()
closing := a.close
a.close = nil
a.mu.Unlock()
if closing != nil {
closing()
}
}
// holderOf is this process as the lease's holder.
func holderOf(instance string) lease.Holder {
host, _ := os.Hostname()
return lease.Holder{Instance: instance, Host: host, Build: version}
}
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
a.mu.Lock()
a.serving = true
a.mu.Unlock()
js, err := broker.Dial(address)
if err != nil {
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
"lease: %w", broker.BareAddress(address), err)
}
api, err := jetstream.New(js.Conn())
if err != nil {
js.Close()
return nil, err
}
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
err = broker.EnsureLeaseBucket(asserting, api)
cancel()
if err != nil {
js.Close()
return nil, err
}
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
a.mu.Lock()
defer a.mu.Unlock()
a.reset = time.Now()
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
"no machine refuses the next epoch", was, floor, floor)
}})
if err != nil {
js.Close()
return nil, err
}
gone := make(chan struct{})
epoch, err := l.Take(ctx)
switch {
case ctx.Err() != nil:
js.Close()
return nil, ctx.Err()
case err != nil && !errors.Is(err, lease.ErrUnwritable):
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
js.Close()
return nil, err
case err != nil:
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
a.mu.Lock()
a.unleased = err.Error()
a.mu.Unlock()
say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
go a.takeWhenGranted(ctx, l, inv, gone)
default:
a.took(ctx, l, inv, epoch, gone)
}
a.mu.Lock()
a.close = func() {
if held, err := l.Epoch(); err == nil {
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
say("how epoch %d ended could not be recorded: %v", held, err)
}
cancel()
}
l.Release(context.Background())
js.Close()
}
a.mu.Unlock()
return gone, nil
}
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
a.mu.Lock()
a.held, a.unleased = l, ""
a.mu.Unlock()
h := holderOf(instance)
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
Build: h.Build, Taken: time.Now()})
cancel()
if err != nil {
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
}
for _, e := range expired {
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
}
go l.Keep(ctx)
go func() {
<-l.Lost()
if ctx.Err() == nil {
why := l.LostWhy()
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
cancel()
fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
"be started again as a candidate\n", epoch, why)
}
close(gone)
}()
}
// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
// controller if another took it meanwhile: two serving at once is what the lease is for.
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
tick := time.NewTicker(lease.RenewEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
}
epoch, err := l.TryTake(ctx)
if errors.Is(err, lease.ErrTaken) {
fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
"stops and exits\n", err)
close(gone)
return
}
if err != nil {
// Still not written, or not readable this time: unleased, said by S12, tried again.
a.mu.Lock()
a.unleased = err.Error()
a.mu.Unlock()
continue
}
a.took(ctx, l, inv, epoch, gone)
return
}
}
// standing is what `status` and the self-check say of this process and the lease.
type standing struct {
Epoch uint64
Held bool
Renewed time.Time
Unleased string
// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
Reset time.Time
ResetSaid string
}
func (a *actor) standing() standing {
a.mu.Lock()
held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
a.mu.Unlock()
st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
if held == nil {
return st
}
epoch, err := held.Epoch()
st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
return st
}
// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
func init() {
link.ActingGate = func(ctx context.Context) error {
_, err := theLease.epoch(ctx)
if err != nil {
return fmt.Errorf("this controller may not send: %w", err)
}
return nil
}
}
cmd/mesh-controller/acts.go
+22 -151
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"github.com/novox/mesh-controller/internal/broker"
"slices"
"sort"
"strings"
@@ -40,10 +39,7 @@ import (
//
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
// machines this just blocked is worth more than the milliseconds.
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
if len(modules) == 0 {
return "", fmt.Errorf("assign %s names no module", node)
}
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
// be taken without ever having been previewed (novox/hq ADR 0100).
ctx, release, err := holdNodes(ctx, open, []string{node})
@@ -51,16 +47,6 @@ func assign(ctx context.Context, open *stores, node string, modules ...string) (
return "", err
}
defer release()
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
// resources are applied through a seat nothing on the node holds is refused, because that order
// — the service manager, the package manager and the runtime before anything that installs,
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
// holders that depend on each other go on in one command.
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
if err != nil {
return "", err
}
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
// assignment resolves against a record rather than against a coincidence.
@@ -68,180 +54,65 @@ func assign(ctx context.Context, open *stores, node string, modules ...string) (
if err != nil {
return "", err
}
var lines []string
var added []string
for _, module := range modules {
fresh, err := open.inventory.Assign(ctx, node, module)
if err != nil {
return strings.Join(lines, "\n"), err
}
if !fresh {
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
lines = append(lines, fmt.Sprintf(
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
continue
}
added = append(added, module)
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
fresh, err := open.inventory.Assign(ctx, node, module)
if err != nil {
return "", err
}
if len(added) == 0 {
return strings.Join(lines, "\n"), nil
if !fresh {
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
return fmt.Sprintf("%s already runs %s — one node runs one of each (ADR 0115); nothing changed",
node, module), nil
}
answer := strings.Join(lines, "\n")
said := fmt.Sprintf("%s is assigned %s", node, module)
for _, line := range settled {
answer += "\n " + line
}
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
// node's list, and every other node's, is `status`'s.
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
answer += "\n " + line
said += "\n " + line
}
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
// no credential yet; kept when it has one, so re-assigning rotates nothing.
for _, module := range added {
if line := issueOnAssign(ctx, open, node, module); line != "" {
answer += "\n " + line
}
if line := issueOnAssign(ctx, open, node, module); line != "" {
said += "\n " + line
}
plan, _, err := planFor(ctx, open, node)
if err != nil {
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
// worth reporting: this machine's refusal is rarely the only consequence.
return answer + blockedElsewhere(ctx, open, node), err
return said + blockedElsewhere(ctx, open, node), err
}
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
// capability the machine lacks is on the wrong machine, and the assignment records what a person
// meant while this line says it will not run until it moves. The rest of the node still pushes.
isAdded := map[string]bool{}
for _, m := range added {
isAdded[m] = true
}
for _, u := range plan.Unhostable {
if !isAdded[u.Module] {
if u.Module != module {
continue
}
for _, c := range u.Missing {
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
said += "\n but " + catalogue.WrongMachine(u.Module, c, node)
}
}
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
return said + fmt.Sprintf("\n run `push %s` to send it", node) +
blockedElsewhere(ctx, open, node), nil
}
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
// and are not judged again.
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
map[string]catalogue.Manifest, []string, error) {
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return nil, nil, err
}
assigned, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, nil, err
}
already := map[string]bool{}
for _, a := range assigned {
already[a] = true
}
var adding []string
for _, m := range modules {
if !already[m] {
adding = append(adding, m)
}
}
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
// with everything else this act changed, so they are not said twice.
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
return nil, nil, err
}
// Two modules declaring one package, path or unit is refused before anything is recorded
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
return nil, nil, err
}
return shelf, assigned, nil
}
// unassign takes modules off a node. What they leave behind is the host's business: a directory
// unassign takes a module off a node. What it leaves behind is the host's business: a directory
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
//
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
// module off one machine is the ordinary way to stop providing something to another, and nothing
// about the command's own output would ever have said so.
//
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
// modules in one act are judged together, so a holder and its dependents come off in one command.
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
if len(modules) == 0 {
return "", fmt.Errorf("unassign %s names no module", node)
}
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
ctx, release, err := holdNodes(ctx, open, []string{node})
if err != nil {
return "", err
}
defer release()
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
if err := open.inventory.Unassign(ctx, node, module); err != nil {
return "", err
}
assigned, err := open.inventory.Assigned(ctx, node)
if err != nil {
return "", err
}
// Every one checked before any is taken off, so a refusal leaves the node as it was.
runs := map[string]bool{}
for _, a := range assigned {
runs[a] = true
}
for _, module := range modules {
if !runs[module] {
return "", fmt.Errorf("%s is not assigned to %s", module, node)
}
}
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
return "", err
}
for _, module := range modules {
if err := open.inventory.Unassign(ctx, node, module); err != nil {
return "", err
}
}
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
node, strings.Join(modules, ", "), node)
var left []string
for _, a := range assigned {
if !slices.Contains(modules, a) {
left = append(left, a)
}
}
for _, line := range unheldChange(shelf, node, assigned, left) {
answer += "\n " + line
}
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
answer += "\n " + line
}
return answer + blockedElsewhere(ctx, open, node), nil
}
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
// command API and the controller seat's verbs as they do from the command line.
func splitModules(field string) []string {
var out []string
for _, m := range strings.Split(field, ",") {
if m = strings.TrimSpace(m); m != "" {
out = append(out, m)
}
}
return out
return fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
node, module, node) + blockedElsewhere(ctx, open, node), nil
}
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
cmd/mesh-controller/addresses_test.go
-8
View File
@@ -111,14 +111,6 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
t.Fatal(err)
}
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
// asks where this machine reaches the store, as the docker module does.
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
t.Fatal(err)
}
on := map[string]bool{"anchor": true, "laptop": true}
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
cmd/mesh-controller/adopting_test.go
+1 -1
View File
@@ -85,7 +85,7 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
t.Fatal(err)
}
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
cmd/mesh-controller/api.go
+2 -2
View File
@@ -95,10 +95,10 @@ func commands(who Authenticator) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
return assign(ctx, open, in.Node, splitModules(in.Module)...)
return assign(ctx, open, in.Node, in.Module)
}))
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
return unassign(ctx, open, in.Node, splitModules(in.Module)...)
return unassign(ctx, open, in.Node, in.Module)
}))
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
cmd/mesh-controller/awaiting_push_test.go
-281
View File
@@ -1,281 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
func aKeeper() catalogue.Manifest {
return catalogue.Manifest{Module: "keeper", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
Resources: []map[string]any{
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
}}
}
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
// are made — without a bus to carry it.
func pushedBy(t *testing.T, open *stores, node string) string {
t.Helper()
ctx := t.Context()
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, node, plan, settings)
if err != nil {
t.Fatal(err)
}
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
digest := digestOf(body)
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
t.Fatal(err)
}
return digest
}
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
func pushedAndApplied(t *testing.T, open *stores, node string) {
t.Helper()
digest := pushedBy(t, open, node)
record, err := open.inventory.NodeByName(t.Context(), node)
if err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
t.Fatal(err)
}
}
// d1 runs D1 once.
func d1(t *testing.T, open *stores) []conditions.Observation {
t.Helper()
got, err := probeDeclarations(t.Context(), &doctor{open: open})
if err != nil {
t.Fatal(err)
}
return got
}
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
// can be compared about a secret that does not exist (status), and nothing here string-matches.
plan, settings, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
var notMade *catalogue.NotMadeError
if !errors.As(err, &notMade) || notMade.Module != "keeper" || notMade.Name != "repository" {
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
}
// Foreseen, it composes, names what the push makes, and made nothing.
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
}
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
}
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
if got := d1(t, open); len(got) != 0 {
t.Fatalf("a machine waiting for a push raised %+v", got)
}
// Past the bound: a warning, not urgent, saying what the push will make.
before := awaitingPushBound
awaitingPushBound = -time.Minute
t.Cleanup(func() { awaitingPushBound = before })
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
}
// Pushed: the secret is made and D1 says nothing.
pushedBy(t, open, "laptop")
if got := d1(t, open); len(got) != 0 {
t.Fatalf("a pushed machine still raised %+v", got)
}
}
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aTalker())
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
t.Fatal(err)
}
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
}
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
// uncomposable for that — the stand-in hides nothing.
register(t, open, aKeeper())
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
t.Fatal(err)
}
one, two := rivals()
register(t, open, one)
register(t, open, two)
_, _ = assign(ctx, open, "anchor", "rival-one")
_, _ = assign(ctx, open, "anchor", "rival-two")
keys := map[string]conditions.Severity{}
for _, o := range d1(t, open) {
keys[o.Key()] = o.Severity
}
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
t.Fatalf("a real failure beside a waiting secret: %v", keys)
}
}
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
// refused on it, so D1 is too.
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
t.Fatalf("a given secret under an old key: %+v", got)
}
}
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop")
sent := time.Now()
since, err := open.inventory.AwaitingSince(ctx, "laptop")
if err != nil || since.After(sent) {
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
}
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
since, err = open.inventory.AwaitingSince(ctx, "laptop")
if err != nil || since.Before(sent.Add(-time.Second)) {
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
}
}
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
// it and had time to report — never between assign and push.
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
now := time.Now()
sent := now.Add(-time.Minute)
long := now.Add(-time.Hour)
cases := []struct {
name string
send lastSend
want bool
}{
{"never sent", lastSend{}, false},
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
}
for _, c := range cases {
if got := c.send.settled("keeper", now); got != c.want {
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
}
}
}
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop")
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
sends, err := readDeliveries(ctx, open.inventory)
if err != nil {
t.Fatal(err)
}
if sends["laptop"].settled("keeper", time.Now()) {
t.Fatal("a holder assigned and not pushed is expected to answer")
}
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
t.Fatal("a module the machine was sent long ago is not expected to answer")
}
pushedBy(t, open, "laptop")
sends, err = readDeliveries(ctx, open.inventory)
if err != nil {
t.Fatal(err)
}
if sends["laptop"].settled("keeper", time.Now()) {
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
}
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
t.Fatal("pushed past the grace, a holder is not expected")
}
if _, ok := sends["laptop"].carried["keeper"]; !ok {
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
}
pushedAndApplied(t, open, "laptop")
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
}
}
cmd/mesh-controller/bindings.go
-140
View File
@@ -1,140 +0,0 @@
package main
import (
"context"
"fmt"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
)
// A binding to data moves only by a person (novox/hq ADR 0232, issue 273).
//
// The resolver keeps each consumer of a provision that keeps its data at the provider it was last
// sent (catalogue/bound.go) and says what it would have moved. This is where that is said: on the
// push that composed it, and by the self-check's D12 every run, as an urgent condition naming the
// consumer, both providers and the pin that confirms the move.
// The condition kinds of D12.
const (
// kindBindingKept is a move the resolver refused: the consumer is still where its data is.
kindBindingKept = "binding-kept"
// kindBindingMoved is a consumer about to be sent another provider than the one on record with
// no pin naming it — what the resolver exists to make impossible, said if it ever is not.
kindBindingMoved = "binding-moved"
// kindBindingMoving is a move a pin asked for, not yet sent: a person's act, said so that the
// data is moved before the push that carries it.
kindBindingMoving = "binding-moving"
)
// probeBindingsID is the self-check's id for this probe, and the source of what it raises.
const probeBindingsID = "D12"
// keptObservation is the urgent condition for one refused move.
func keptObservation(k catalogue.KeptBinding) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeMachine, ID: bindingID(k.Machine, k.Consumer, k.Provision),
Token: kindBindingKept, Kind: kindBindingKept, Machine: k.Machine, Also: otherMachines(k.Machine, k.Bound.Node, k.Would.Node),
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("on %s, the mesh %s", k.Machine, k.String())}
}
func bindingID(machine, consumer, provision string) string {
return machine + "." + consumer + "." + provision
}
func otherMachines(machine string, nodes ...string) []string {
var out []string
seen := map[string]bool{machine: true}
for _, n := range nodes {
if n != "" && !seen[n] {
seen[n] = true
out = append(out, n)
}
}
return out
}
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
// condition at once where this process keeps the conditions: a push is when a person is looking.
func reportKept(ctx context.Context, plan catalogue.Resolution) {
for _, k := range plan.Kept {
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
if conditionsFrom != nil {
o := keptObservation(k)
o.Source = probeBindingsID
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
}
}
}
}
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
// moves it (said, so the data goes first), and never anything else.
func probeBindings(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
inv := d.open.inventory
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
}
var out []conditions.Observation
for _, n := range nodes {
plan, _, err := planFor(ctx, d.open, n.Name)
if err != nil {
if unresolvable(err) {
// D1 says it, with the binding that refused it when that is why.
continue
}
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
}
bound, err := inv.BindingsFor(ctx, n.Name)
if err != nil {
return nil, err
}
pins, err := inv.PinsFor(ctx, n.Name)
if err != nil {
return nil, err
}
out = append(out, bindingFindings(plan, bound, pins)...)
}
return out, nil
}
// bindingFindings is what one machine's resolution says against its record.
func bindingFindings(plan catalogue.Resolution, bound map[string]map[string]catalogue.Chosen,
pins map[string]catalogue.Chosen) []conditions.Observation {
var out []conditions.Observation
for _, k := range plan.Kept {
out = append(out, keptObservation(k))
}
said := map[string]bool{}
for _, need := range plan.Needs {
if !need.KeepsData || need.ByRecord {
continue
}
was, recorded := bound[need.For][need.Name]
now := catalogue.Chosen{Node: need.From, Module: need.Module}
if !recorded || was == now || (was.Module == "" && was.Node == now.Node) {
continue
}
id := bindingID(plan.Node, need.For, need.Name)
if said[id] {
continue
}
said[id] = true
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Machine: plan.Node,
Also: otherMachines(plan.Node, was.Node, now.Node), Resolver: conditions.ResolverOperator}
if pin, pinned := pins[need.Name]; pinned && pin.Node == now.Node && (pin.Module == "" || pin.Module == now.Module) {
o.Token, o.Kind, o.Severity = kindBindingMoving, kindBindingMoving, conditions.Warning
o.Summary = fmt.Sprintf("on %s, %s's %s moves from %s to %s at the next push, by the pin — its data "+
"is on %s: move it first", plan.Node, need.For, need.Name, was, now, was)
} else {
o.Token, o.Kind, o.Severity = kindBindingMoved, kindBindingMoved, conditions.Urgent
o.Summary = fmt.Sprintf("on %s, %s's %s would be sent %s, and it is bound to %s, where its data is, "+
"with no pin naming %s — a move nothing asked for", plan.Node, need.For, need.Name, now, was, now)
}
out = append(out, o)
}
return out
}
cmd/mesh-controller/bindings_test.go
-183
View File
@@ -1,183 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
)
// novox/hq issue 273, ADR 0232: a consumer of a provision that keeps its data moves only by a pin.
func storeManifests() []catalogue.Manifest {
return []catalogue.Manifest{
{Module: "store", Version: "1",
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
}
}
func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) catalogue.Needed {
t.Helper()
for _, n := range plan.Needs {
if n.For == consumer && n.Name == provision {
return n
}
}
t.Fatalf("no %s for %s: %+v", provision, consumer, plan.Needs)
return catalogue.Needed{}
}
// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
for _, m := range storeManifests() {
register(t, open, m)
}
assignAll := func(pairs ...[2]string) {
for _, a := range pairs {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
}
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
t.Fatal(err)
}
}
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
[2]string{"laptop", "board"})
plan, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
}
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
}
// Sent, and recorded: only the binding to data.
bindings := boundToData(plan, nil)
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
t.Fatalf("recorded %+v", bindings)
}
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
t.Fatal(err)
}
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
}
// The laptop's store taken away: refused, not moved to the anchor's empty one.
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
t.Fatal(err)
}
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
}
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
t.Fatal(err)
}
plan, _, err = planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
t.Fatalf("pinned to the anchor and bound to %s", n.From)
}
found, err := probeBindings(ctx, &doctor{open: open})
if err != nil {
t.Fatal(err)
}
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
}
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
t.Fatal(err)
}
all, err := inv.Bindings(ctx)
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
t.Fatalf("%+v, %v", all, err)
}
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
t.Fatalf("a move sent is still said: %+v, %v", found, err)
}
}
// What one machine's resolution says against its record.
func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
home, anchor := catalogue.Chosen{Node: "home", Module: "store"}, catalogue.Chosen{Node: "anchor", Module: "store"}
plan := catalogue.Resolution{Node: "laptop",
Kept: []catalogue.KeptBinding{{Machine: "laptop", Consumer: "board", Provision: "postgres-database",
Bound: home, Would: anchor}},
Needs: []catalogue.Needed{
{Name: "postgres-database", For: "board", From: "home", Module: "store", KeepsData: true},
{Name: "postgres-database", For: "game", From: "anchor", Module: "store", KeepsData: true},
{Name: "wildcard-resolution", For: "network", From: "anchor", Module: "resolver"},
}}
bound := map[string]map[string]catalogue.Chosen{
"board": {"postgres-database": home},
"game": {"postgres-database": home},
"network": {"wildcard-resolution": {Node: "home", Module: "resolver"}},
}
found := linted(bindingFindings(plan, bound, nil))
if len(found) != 2 {
t.Fatalf("found %+v", found)
}
kept, moved := found[0], found[1]
if kept.Kind != kindBindingKept || kept.Severity != conditions.Urgent || kept.Machine != "laptop" ||
!strings.Contains(kept.Summary, "would move board's postgres-database from home/store to anchor/store") ||
!strings.Contains(kept.Summary, "its data is on home/store") ||
!strings.Contains(kept.Summary, "`pin laptop postgres-database anchor store` to confirm a move (and move the data first)") {
t.Errorf("kept: %+v", kept)
}
if moved.Kind != kindBindingMoved || moved.Severity != conditions.Urgent ||
!strings.Contains(moved.Summary, "game's postgres-database would be sent anchor/store") {
t.Errorf("moved: %+v", moved)
}
if kept.Key() == moved.Key() {
t.Error("two consumers, one condition")
}
}
// A push says the move it refused, and raises its condition at once.
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
k, _ := withConditionsInMemory(t)
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
open[0].Source != probeBindingsID {
t.Fatalf("raised %+v", open)
}
}
cmd/mesh-controller/build.go
+10 -128
View File
@@ -6,9 +6,7 @@ import (
"errors"
"flag"
"fmt"
"github.com/novox/mesh-controller/internal/conditions"
"os"
"slices"
"strings"
"time"
@@ -149,8 +147,6 @@ func buildFrom(result link.BuildResult) inventory.Build {
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
// rebuild the graph rather than a list of names.
Path: result.Path,
// What it was made from (novox/hq issue 280): two builds with one are one build.
SourceFingerprint: result.SourceFingerprint,
}
// When it was asked, which is what orders it against another build of the same module
// (novox/hq 04-ISSUES/219) — not when it was heard.
@@ -396,34 +392,22 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
//
// Separated from the command so `--behind` can walk a list without a second path to the same act.
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
return err
}
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
dryRun bool) (string, error) {
if dryRun && wait != 0 {
return "", errors.New("a dry run waited for is `build --dry-run`")
}
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
// the reason, rather than sent to a machine to fail at `git clone`.
repository, err := cloneFrom(ctx, source)
if err != nil {
return "", err
return err
}
ident, err := openIdentity(ctx)
if err != nil {
return "", err
return err
}
defer ident.Close()
server, err := connectLink(ctx, nil, nil, nil)
if err != nil {
return "", err
return err
}
defer server.Close()
@@ -436,7 +420,6 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
Ref: ref,
Held: heldBy(ctx),
Seats: seatBases(ctx),
DryRun: dryRun,
}
fmt.Printf("asked for %s", source)
if source.Seat != "" {
@@ -455,7 +438,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
seat := buildSeatHeld(ctx)
ask, err := askOverOn(seat)
if err != nil {
return "", err
return err
}
defer ask.Close()
fmt.Printf(" of %s\n", seat)
@@ -466,31 +449,26 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
// follows the build by its id instead.
if err := ask.Ask(ctx, request); err != nil {
return "", err
}
if dryRun {
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
"its outcome is not taken in\n", request.ID)
return request.ID, nil
return err
}
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
return request.ID, nil
return nil
}
result, err := ask.Submit(ctx, request, wait)
if err != nil {
return request.ID, err
return err
}
open, err := openStores(ctx)
if err != nil {
return request.ID, err
return err
}
defer open.Close()
manifest, kept, err := takeIn(ctx, open.inventory, result)
if err != nil {
return request.ID, err
return err
}
// Said as recorded: what each artifact is, not where this builder happened to push it.
for _, made := range kept.Made {
@@ -500,7 +478,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
manifest.Module, manifest.Version, result.On, short(result.Commit))
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
return request.ID, nil
return nil
}
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
@@ -573,46 +551,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
result.On, result.Repository, short(result.Commit), err)
}
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
// The branch the module already follows is its trunk — never the branch a build was asked at, or a
// build of a feature branch by name would make that branch its trunk.
follows := ""
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
follows = followedBranch(was.Ref)
}
if err := publishable(result, follows); err != nil {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
manifest.Module, short(result.Commit), err)
}
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
// and the next push would send it.
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
return manifest, kept, err
} else if failed {
return manifest, kept, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
short(result.Commit))
}
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
// what the build the mesh stands on was made from registers that build's artifacts at the new
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
// byte-reproducible, and the bus rebuilt for another module's merge demanded a planned upgrade.
if kept.SourceFingerprint != "" {
stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID)
if err != nil {
return manifest, kept, err
}
if stands != "" && stands != kept.ID && len(raw) > 0 {
if same, err := catalogue.ParseManifest(raw); err == nil && same.Module == manifest.Module {
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
manifest.Module, short(result.Commit), stands)
manifest = same
}
}
}
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
if errors.Is(err, inventory.ErrSuperseded) {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
@@ -627,34 +565,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return manifest, kept, nil
}
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
"never published (ADR 0238) — merge it, and the merge builds it")
// publishable says whether a build's outcome may become a module's version: its commit on the module's
// trunk, as the build seat read the forge at the build — the branch the module follows when its source
// names one, else its repository's default branch. A build seat that could not say — one older than the
// rule, building its own successor — is let through and said, so the rule can reach the mesh.
func publishable(result link.BuildResult, follows string) error {
if result.Check != nil || result.Checked != nil || result.DryRun {
return errors.New("a check or a dry run is never published")
}
if result.Trunk == "" {
fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+
"registered as before\n", result.ID, short(result.Commit))
return nil
}
trunk := result.Trunk
if follows != "" {
trunk = follows
}
on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk)
if !on {
return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk)
}
return nil
}
// buildAndShow builds and prints the manifest without recording anything.
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
repository, err := cloneFrom(ctx, source)
@@ -682,7 +592,6 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
ID: link.NewBuildID(time.Now()),
Repository: repository, Path: path, Ref: ref,
Held: heldBy(ctx), Seats: seatBases(ctx),
DryRun: true,
}, wait)
if err != nil {
return err
@@ -718,8 +627,6 @@ type answers struct {
reported []inventory.Reported
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
plans []inventory.Plan
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
paused pauseView
// refused is why a machine cannot be worked out at all, by name. A different thing from every
// other answer here: those are about a machine that was told something, and this is about one
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
@@ -743,29 +650,6 @@ type answers struct {
// public name on the machine went dark. The holds were correct; they were recorded only in the
// machine's own state file, and the one visible symptom was a count that did not add up.
untaken map[string]map[string]int
// unheld is every module on a machine whose resources are applied through a seat nothing on
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
// refused, until the switch — and while there is any, the mesh is not all well: the order the
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
unheld []catalogue.Unheld
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
// not be read when they could not — never read as none.
conditions []conditions.Condition
conditionsUnread string
// overflowing is every module whose identity overflows the bound of a provision it requires
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
// this is the one place it is said across the mesh. Not well while there is any.
overflowing []catalogue.Overflow
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
// where the log is not on hand; handActsUnread why it could not be read when it could not.
handActs *int
handActsUnread string
// heals is what the healers did in the last seven days (novox/hq to-be 45 §7): acts, and conditions
// handed to the operator; healsUnread why it could not be read.
heals *healsCount
healsUnread string
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
@@ -782,14 +666,12 @@ func heldBy(ctx context.Context) map[string]string {
if err != nil {
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
"base will be told that base is missing: %v\n", err)
// empty-on-error: said above; a build that names a base is refused by name for want of it
return nil
}
defer open.Close()
held, err := open.inventory.Held(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
// empty-on-error: said above; a build that names a base is refused by name for want of it
return nil
}
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
cmd/mesh-controller/bus_step.go
-410
View File
@@ -1,410 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236).
//
// **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the
// controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything,
// and a version whose data format moved (2.10 → 2.11) cannot be undone by putting the old one back.
// So nothing sends a new bus build on its own: its policy is `record` whatever anyone says (the
// catalogue's DerivedUpgrade, and `upgrade` refuses a roll-out), a plan builds it and sends nothing, a
// cascade holds the machine (ADR 0221), and a push naming that machine is refused while a new bus build
// waits for it (busHeld). The one way is this verb: a person, with why, the streams snapshotted first,
// whether it can be reverted said before it starts, the step said as `bus-maintenance` while it runs,
// and every stream, durable consumer and a round trip checked after (H-bus) — or the step said failed,
// with its snapshot as the way back.
// busStepProbe is the registry's row for the step; its kinds.
const (
busStepProbe = "DB"
kindBusMaintenance = "bus-maintenance"
kindBusUpgradeFailed = "bus-upgrade-failed"
)
// busStepBound is how long after its start a bus upgrade must be followed by a healthy bus.
var busStepBound = 15 * time.Minute
// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is: the bus
// machine's backup holder backs the bus module up now, whose dump is the streams' snapshot (novox/hq ADR
// 0235). A variable so a test takes none. A person who took one by hand says where with --snapshot-taken,
// and then none is taken — for a bus whose module does not yet carry the snapshot program.
var takeBusSnapshot = snapshotTheBusNow
// busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per
// machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds
// no bus module.
type busPending struct {
module string
machines []string
from map[string]string
to string
// same are the commits whose build was made from the same source as the build the mesh holds, or
// made the same artifacts and manifest (novox/hq issue 280).
same map[string]bool
}
// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the
// two builds were made from the same source, or made the same artifacts from the same manifest — a
// rebuild of the same source for another module's merge changes nothing the machine runs, whatever
// image digest it made (novox/hq issue 280).
func (b busPending) moves(machine string) bool {
from, known := b.from[machine]
if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) {
return false
}
return !known || !b.same[from]
}
// pendingBus reads what a bus upgrade would do.
func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, error) {
var b busPending
shelf, err := inv.Catalogue(ctx)
if err != nil {
return b, err
}
for name, m := range shelf {
if catalogue.ProvidesBus(m) {
b.module = name
}
}
if b.module == "" {
return b, nil
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
return b, err
}
b.to = current[b.module].Commit
if b.machines, err = inv.Running(ctx, b.module); err != nil {
return b, err
}
b.from, b.same = map[string]string{}, map[string]bool{}
made, err := inv.BuildFingerprints(ctx, b.module)
if err != nil {
return b, err
}
for commit, refs := range made {
b.same[commit] = refs != "" && refs == made[b.to]
}
sources, err := inv.BuildSourceFingerprints(ctx, b.module)
if err != nil {
return b, err
}
for commit, src := range sources {
if src != "" && src == sources[b.to] {
b.same[commit] = true
}
}
for _, n := range b.machines {
sent, known, err := inv.SentBuilds(ctx, n)
if err != nil {
return b, err
}
if known {
if c, carried := sent[b.module]; carried {
b.from[n] = c
}
}
}
return b, nil
}
// busHeld names the machines a push may not send because sending them would replace the bus: the
// planned step's, not a push's (ADR 0236). Said with the remedy.
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
b, err := pendingBus(ctx, inv)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, n := range machines {
for _, holder := range b.machines {
if n == holder && b.moves(n) {
out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+
"`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)",
n, b.module, short(orNotKnown(b.from[n])), short(b.to))
}
}
}
return out, nil
}
// busCommand is `bus` — what a bus upgrade would do and how the last went — and `bus upgrade`.
func busCommand(ctx context.Context, args []string) error {
sub := ""
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
set := flag.NewFlagSet("bus", flag.ContinueOnError)
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
irreversible := set.Bool("irreversible", false, "the new version cannot be undone by putting the old one back, "+
"and this is the person's explicit word that it runs anyway")
why := addHandActFlags(set)
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>]]")
}
switch sub {
case "":
return busStatus(ctx)
case "upgrade":
default:
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub)
}
// Everything refused before anything is done.
if err := why.require("bus upgrade"); err != nil {
return err
}
if *reversible == *irreversible {
return errors.New("bus upgrade says, before it starts, whether the new version can be undone by putting the " +
"old one back: --reversible, or --irreversible as your explicit word that it runs anyway (to-be 45 §8). " +
"Nothing was done")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
b, err := pendingBus(ctx, inv)
if err != nil {
return err
}
if b.module == "" {
return errors.New("the mesh holds no module that provides its bus: there is nothing to upgrade")
}
var moving []string
for _, n := range b.machines {
if b.moves(n) {
moving = append(moving, n)
}
}
if len(moving) == 0 {
fmt.Printf("every machine running %s runs the build the mesh holds (%s): nothing to upgrade\n", b.module, short(b.to))
return nil
}
where := strings.TrimSpace(*snapshot)
if where == "" {
for _, n := range moving {
fmt.Printf("snapshotting the bus's streams on %s first (its backup holder, ADR 0235)…\n", n)
if where, err = takeBusSnapshot(ctx, b.module, n); err != nil {
return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w — a snapshot "+
"taken by hand is said with --snapshot-taken <where>", err)
}
}
}
from := map[string]bool{}
for _, n := range moving {
from[orNotKnown(b.from[n])] = true
}
step, err := inv.StartBusStep(ctx, inventory.BusStep{Module: b.module, Machines: moving,
From: strings.Join(sortedKeys(from), ", "), To: b.to, Snapshot: where, Reversible: *reversible,
By: link.Caller(), Why: strings.TrimSpace(*why.why)})
if err != nil {
return err
}
cause := "bus-upgrade"
if strings.TrimSpace(*why.cause) == "" {
why.cause = &cause
}
why.record(ctx, "bus upgrade", append([]string{b.module}, moving...))
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
if err != nil {
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
}
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
return nil
}
// busStatus is `bus`: what an upgrade would do, and the last step.
func busStatus(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
b, err := pendingBus(ctx, open.inventory)
if err != nil {
return err
}
if b.module == "" {
fmt.Println("the mesh holds no module that provides its bus")
} else {
fmt.Printf("the bus is %s, built %s, on %s; never rolled out — a planned step (`bus upgrade`)\n", b.module,
short(b.to), orNone(strings.Join(b.machines, ", ")))
for _, n := range b.machines {
state := "runs it"
if b.moves(n) {
state = "runs " + short(orNotKnown(b.from[n])) + ": `bus upgrade` replaces it"
}
fmt.Printf(" %-10s %s\n", n, state)
}
}
fmt.Println(" `bus upgrade` has the bus machine's backup holder snapshot the streams first (ADR 0235)")
s, found, err := open.inventory.LatestBusStep(ctx)
if err != nil || !found {
return err
}
state := "running since " + s.Started.Local().Format("2006-01-02 15:04")
if s.Ended != nil {
state = s.Outcome + " at " + s.Ended.Local().Format("2006-01-02 15:04")
}
fmt.Printf("last step %d: %s → %s on %s by %s (%s): %s; snapshot %s\n", s.ID, s.From, short(s.To),
strings.Join(s.Machines, ", "), orNone(s.By), s.Why, state, s.Snapshot)
if s.Found != "" {
fmt.Printf(" %s\n", s.Found)
}
return nil
}
// probeBusStep is DB: a bus upgrade running is said as `bus-maintenance`; the bus healthy again after
// the machines reported the new build ends it done; past its bound, unhealthy, it ends failed and is
// said — urgent, with its snapshot — while the bus is still not healthy.
func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
inv := d.open.inventory
s, found, err := inv.LatestBusStep(ctx)
if err != nil || !found {
return nil, err
}
if s.Ended != nil && s.Outcome != "failed" {
return nil, nil
}
problems, err := busHealth(ctx, d)
if err != nil {
return nil, err
}
reports, err := inv.LastReports(ctx)
if err != nil {
return nil, err
}
applied := true
for _, r := range reports {
for _, n := range s.Machines {
if r.Node == n && (!r.Current || r.Outcome != inventory.OutcomeApplied || r.At == nil || r.At.Before(s.Started)) {
applied = false
problems = append(problems, n+" has not reported the new bus applied")
}
}
}
id := s.Module
if s.Ended == nil {
switch {
case applied && len(problems) == 0:
return nil, inv.EndBusStep(ctx, s.ID, "done", "the bus answered healthy after the upgrade")
case time.Since(s.Started) > busStepBound:
found := strings.Join(problems, "; ")
if err := inv.EndBusStep(ctx, s.ID, "failed", found); err != nil {
return nil, err
}
s.Found = found
default:
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "maintenance",
Kind: kindBusMaintenance, Severity: conditions.Warning,
Summary: fmt.Sprintf("the bus is being upgraded (step %d, %s → %s on %s, by %s: %s); its snapshot is %s",
s.ID, s.From, short(s.To), strings.Join(s.Machines, ", "), orNone(s.By), s.Why, s.Snapshot),
Said: orNone(strings.Join(problems, "; "))}}, nil
}
}
if len(problems) == 0 {
return nil, nil // failed, and healthy since: nothing wrong now
}
way := "put the old build back"
if !s.Reversible {
way = "restore the snapshot"
}
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "upgrade-failed",
Kind: kindBusUpgradeFailed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the bus upgrade (step %d, %s → %s) did not end healthy within %s: %s — the way back is to %s (%s)",
s.ID, s.From, short(s.To), busStepBound, strings.Join(problems, "; "), way, s.Snapshot)}}, nil
}
func sortedKeys(set map[string]bool) []string {
out := make([]string, 0, len(set))
for k := range set {
out = append(out, k)
}
sort.Strings(out)
return out
}
// busSnapshotWithin is how long the bus machine's backup holder is given to take the bus's snapshot.
var busSnapshotWithin = 15 * time.Minute
// snapshotTheBusNow asks the bus machine's backup holder to back the bus module up now — its dump is
// the streams' snapshot (novox/hq ADR 0235) — and waits until it says a backup newer than the ask:
// where the snapshot is, as a person reads it. The serving controller's connection, or one of its own.
func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) {
var where string
err := onTheBus(func(conn *nats.Conn) error {
asked := time.Now()
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", node,
map[string]any{"module": module}, 30*time.Second)
if err != nil {
return fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
}
if answer.Error != "" {
return fmt.Errorf("%s's backup holder would not take the bus's snapshot: %s", node, answer.Error)
}
deadline := time.Now().Add(busSnapshotWithin)
for {
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", node, map[string]any{}, 10*time.Second)
if err == nil && answer.Error == "" {
if measured, err := readHolder(answer.Result); err == nil {
for item, m := range measured[module] {
if m.LastBackup != nil && m.LastBackup.After(asked) && m.Error == "" {
where = fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, item,
m.LastBackup.UTC().Format(time.RFC3339))
return nil
}
}
}
}
if time.Now().After(deadline) {
return fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s; the bus is "+
"not replaced", node, busSnapshotWithin)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(10 * time.Second):
}
}
})
return where, err
}
// errBusWaits is a send refused because it would replace the bus outside its planned step.
var errBusWaits = errors.New("a new bus build waits for its planned step")
type busStepKey struct{}
// withBusStep marks a send as the bus's planned step: the one send that may replace the bus.
func withBusStep(ctx context.Context) context.Context {
return context.WithValue(ctx, busStepKey{}, true)
}
func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on }
cmd/mesh-controller/busobjects.go
-162
View File
@@ -1,162 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
// D6 and D7).
//
// **What the controller asserts at its start and what the self-check expects to find are one
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
// the drift rather than the fault.
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
// can now hear a declaration.
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
}
names := make([]string, 0, len(nodes))
for _, n := range nodes {
names = append(names, n.Name)
}
if err := broker.Raise(r, names); err != nil {
return nil, err
}
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
// after something started asking for builds flushes the backlog instead of having lost it.
// With the seats' holders, so each role's work queue gets the consumer its holder takes
// work from. Passed as nil until the first live raise, which left the build machine bound to a
// consumer nothing had created (2026-09-28).
holders, err := seatHolders(ctx, inv)
if err != nil {
return nil, err
}
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
return nil, err
}
// And how every module hears what it consumes. Derived from the same records the user list is
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
// Done on every raise, not only when a credential is issued: every module moved onto this bus
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
consumers, err := moduleConsumers(ctx, inv)
if err != nil {
return nil, err
}
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
var failed []error
for _, c := range consumers {
if err := r.EnsureConsumer(c.Consumer); err != nil {
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
}
}
if len(failed) > 0 {
return nil, errors.Join(failed...)
}
return names, nil
}
// What raises the condition a send says when the objects it implies could not be asserted, and its kind.
const (
sourceBusObjects = "bus-objects"
kindBusObjectsUnasserted = "bus-objects-unasserted"
)
// assertOnSend asserts, on the bus a send is about to use, every object assertBusObjects derives —
// **whenever a declaration is sent, not only when the controller starts** (novox/hq issue 208).
//
// A module assigned after the controller started was sent its declaration and found no consumer to
// bind (`consumer not found`, messenger on 2026-10-06), and a seat holder assigned after it found no
// worker: the objects a declaration implies were asserted at start and nowhere else, so they existed
// only for what was assigned before the last restart. The same derivation, not a second list of what
// a send needs: what start asserts, the self-check expects and a send asserts are one answer. Every
// part is idempotent, so asserting the whole of it again is the no-op a restart already relies on.
//
// **A failure is said and raised, and the send goes on.** The objects are the mesh's, not the
// machines' being sent: holding every machine back for one consumer that none of them may use would
// turn one fault into all of them, and the declarations are not what is wrong. It is never silent —
// said in the send's own output and raised as a condition, which the next send that asserts them
// clears — and the start-time raise still refuses to serve without them.
func assertOnSend(ctx context.Context, inv *inventory.Inventory, r broker.Raiser, indent string) error {
_, err := assertBusObjects(ctx, inv, r)
var observed []conditions.Observation
if err != nil {
fmt.Printf("%sTHE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES: %v\n", indent, err)
fmt.Printf("%s a module may find no consumer to bind, or a holder no worker; sent anyway, raised as "+
"condition %s, and asserted again by the next send\n", indent, unassertedObservation(err).Key())
observed = append(observed, unassertedObservation(err))
}
// Observed when it failed, cleared when it did not: a send that asserted everything is the
// observation that the bus holds what it should.
if kerr := withKeeper(ctx, func(k *conditions.Keeper) error {
return k.Reconcile(ctx, sourceBusObjects, observed)
}); kerr != nil {
fmt.Printf("%sand whether the bus holds what this send implies could not be kept as a condition: %v\n",
indent, kerr)
}
return err
}
// unassertedObservation is a send's failure to assert the bus's objects, as a condition.
func unassertedObservation(err error) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeBus, ID: "objects", Token: "unasserted",
Kind: kindBusObjectsUnasserted, Severity: conditions.Warning, Source: sourceBusObjects,
Summary: "the bus's streams and consumers could not be asserted when a declaration was sent: " +
"a module may find no consumer to bind, or a seat's holder no worker",
Said: err.Error()}
}
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
records, err := inv.BusRecords(ctx)
if err != nil {
return nil, err
}
users, err := broker.Users(records)
if err != nil {
return nil, err
}
return broker.ConsumersOf(users), nil
}
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
consumers, err := moduleConsumers(ctx, inv)
return len(consumers), err
}
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
var rec recordingRaiser
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
}
return rec.streams, rec.consumers, nil
}
// recordingRaiser keeps what it was asked to assert and asserts nothing.
type recordingRaiser struct {
streams []broker.Stream
consumers []broker.Consumer
}
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
r.streams = append(r.streams, s)
return nil
}
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
r.consumers = append(r.consumers, c)
return nil
}
cmd/mesh-controller/busobjects_test.go
-197
View File
@@ -1,197 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
// only when the controller starts.
// aCarriedConsumer is the runtime on laptop and, carried by it, a module that consumes an event: the
// shape of messenger on 2026-10-06, assigned after the controller started.
func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
t.Helper()
register(t, open, catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1",
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}})
register(t, open, catalogue.Manifest{Module: "messenger", Version: "1",
Consumes: []string{"billing.order.placed"}})
for _, m := range []string{catalogue.RuntimeModule, "messenger"} {
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
t.Fatal(err)
}
}
consumers, err := moduleConsumers(t.Context(), open.inventory)
if err != nil {
t.Fatal(err)
}
for _, c := range consumers {
if c.Module == "messenger" && c.Node == "laptop" {
return c.Consumer
}
}
t.Fatalf("messenger on laptop is derived no consumer: %+v", consumers)
return broker.Consumer{}
}
// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
// started, and the worker its seat's queue should have for it.
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
t.Helper()
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
t.Fatal(err)
}
for _, s := range inventory.MeshSeats() {
if s.Name == "node-build-agent" {
c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
if !needed {
t.Fatal("the build agent's seat needs no worker")
}
return c
}
}
t.Fatal("the mesh declares no build agent's seat")
return broker.Consumer{}
}
// asserted says whether a recording holds a consumer by stream and name.
func asserted(rec *recordingRaiser, want broker.Consumer) bool {
for _, c := range rec.consumers {
if c.Stream == want.Stream && c.Name == want.Name {
return true
}
}
return false
}
// What a send asserts includes what was assigned after the start's assertion: a carried module's
// consumer and a late holder's worker. The derivation is the start's own, so this holds without a bus.
func TestASendAssertsWhatWasAssignedAfterStart(t *testing.T) {
open := aMesh(t)
var atStart recordingRaiser
if _, err := assertBusObjects(t.Context(), open.inventory, &atStart); err != nil {
t.Fatal(err)
}
consumer := aCarriedConsumer(t, open)
worker := aLateHolder(t, open)
if asserted(&atStart, consumer) || asserted(&atStart, worker) {
t.Fatal("the start asserted what was not yet assigned; the test proves nothing")
}
var onSend recordingRaiser
if err := assertOnSend(t.Context(), open.inventory, &onSend, ""); err != nil {
t.Fatal(err)
}
if !asserted(&onSend, consumer) {
t.Fatalf("messenger's consumer %s on %s is not asserted by the send: %+v", consumer.Name, consumer.Stream, onSend.consumers)
}
if !asserted(&onSend, worker) {
t.Fatalf("the late holder's worker %s on %s is not asserted by the send: %+v", worker.Name, worker.Stream, onSend.consumers)
}
}
// failingRaiser refuses one consumer by name and records everything it was asked.
type failingRaiser struct {
recordingRaiser
refuse string
}
func (f *failingRaiser) EnsureConsumer(c broker.Consumer) error {
f.consumers = append(f.consumers, c)
if c.Name == f.refuse {
return errors.New("nats: API error: code=503 description=insufficient resources")
}
return nil
}
// A send whose objects cannot be asserted says so in its output and raises a condition — and the next
// send that asserts them clears it. The consumers after the refused one are still asked for.
func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
open := aMesh(t)
consumer := aCarriedConsumer(t, open)
worker := aLateHolder(t, open)
failing := &failingRaiser{refuse: consumer.Name}
var sendErr error
out := stdoutOf(t, func() error {
sendErr = assertOnSend(t.Context(), open.inventory, failing, " ")
return nil
})
if sendErr == nil || !strings.Contains(sendErr.Error(), "how messenger on laptop hears what it consumes") {
t.Fatalf("the failure is not answered: %v", sendErr)
}
if !strings.Contains(out, "THE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES") ||
!strings.Contains(out, "insufficient resources") || !strings.Contains(out, "bus.objects.unasserted") {
t.Fatalf("the failure is not said in the send's output:\n%s", out)
}
if !asserted(&failing.recordingRaiser, worker) {
t.Fatal("the seat's worker was not asked for")
}
c, open1, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted")
if err != nil || !open1 {
t.Fatalf("no condition raised: %v", err)
}
if c.Kind != kindBusObjectsUnasserted || c.Source != sourceBusObjects ||
len(c.Evidence) == 0 || !strings.Contains(c.Evidence[0].Said, "insufficient resources") {
t.Fatalf("the condition does not say what failed: %+v", c)
}
// The next send asserts them, and that observation clears it.
if err := assertOnSend(t.Context(), open.inventory, &recordingRaiser{}, ""); err != nil {
t.Fatal(err)
}
if _, stillOpen, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); err != nil || stillOpen {
t.Fatalf("a send that asserted everything left the condition open: %v", err)
}
}
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
// once a declaration is sent, and a holder assigned after start its seat's worker.
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
url := testbus.URL(t)
open := aMesh(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
_ = js.Context().DeleteStream(s)
}
// The controller starting, before either was assigned.
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
t.Fatal(err)
}
consumer := aCarriedConsumer(t, open)
worker := aLateHolder(t, open)
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
for _, c := range []broker.Consumer{consumer, worker} {
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
}
}
server := link.ConnectNats(js, nil, nil)
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
t.Fatal(err)
}
for _, c := range []broker.Consumer{consumer, worker} {
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
}
}
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
t.Fatal("a send that asserted everything raised a condition")
}
}
cmd/mesh-controller/calls_verb_test.go
-81
View File
@@ -1,81 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
// RequestTimeout) — the shortest wait of a caller the mesh ships.
const consoleWaits = 30 * time.Second
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
if link.AnswerWithin >= consoleWaits/2 {
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
}
if link.AnswerWithin >= broker.ResponseTTL {
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
}
}
// A push — named or through command — answers before it runs: it sends the machine holding the bus
// first, and the broker reloading its user list forgets the answer it was about to permit.
func TestAPushAnswersBeforeItSends(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want bool
}{
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
{"push", map[string]any{"why": "w"}, true},
{"command", map[string]any{"command": "push anchor --why w"}, true},
{"command", map[string]any{"command": "push --behind --why=w"}, true},
{"command", map[string]any{"command": "builds"}, false},
{"status", map[string]any{}, false},
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil {
t.Fatalf("%s %v: %v", c.verb, c.args, err)
}
if got := answersFirst(argv); got != c.want {
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
}
}
}
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
handlers, behind, err := seatToolHandlers()
if err != nil || len(behind) != 0 {
t.Fatalf("%v %v", behind, err)
}
calls, ok := handlers["calls"]
if !ok {
t.Fatal("calls is not served")
}
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
!strings.Contains(err.Error(), `"node"`) {
t.Errorf("calls took an argument it does not declare: %v", err)
}
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
!strings.Contains(err.Error(), "not across a restart") {
t.Errorf("an unknown call was not said plainly: %v", err)
}
got, err := calls(context.Background(), json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
if _, listed := got.(map[string]any)["calls"]; !listed {
t.Errorf("calls answered %v", got)
}
}
cmd/mesh-controller/changeplan.go
-155
View File
@@ -1,155 +0,0 @@
package main
import (
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
// planner, so the two can be compared.
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
type policyOf func(module string) (inventory.Upgrade, bool)
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
machines := map[string]*link.MachinePlan{}
machine := func(name string) *link.MachinePlan {
if machines[name] == nil {
machines[name] = &link.MachinePlan{Machine: name}
}
return machines[name]
}
for _, tier := range r.Plan.Tiers {
for _, name := range tier {
e, held := byName[name]
if !held {
continue
}
u, known := policy(name)
waits := known && !u.RollOut
for _, on := range e.On {
if waits {
machine(on).Waits = append(machine(on).Waits, name)
} else {
machine(on).Receives = append(machine(on).Receives, name)
}
}
switch {
case name == "nats":
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
case waits && len(e.On) > 0:
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
orNone(u.From)))
}
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
var offers []string
for _, o := range e.Manifest.Provides {
offers = append(offers, o.Name)
}
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
name, strings.Join(offers, ", ")))
}
if e.Manifest.Data != nil && len(e.On) > 0 {
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
}
}
}
var names []string
for name := range machines {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
p.Machines = append(p.Machines, *machines[name])
}
p.Summary = summaryOf(p)
return p
}
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
func summaryOf(p link.ChangePlan) string {
if len(p.Moved) == 0 && len(p.New) == 0 {
return "builds nothing: the change touches no module of the mesh's graph"
}
var parts []string
what := strings.Join(p.Moved, ", ")
if len(p.Dependents) > 0 {
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
}
if len(p.New) > 0 {
if what != "" {
what += ", "
}
what += "new: " + strings.Join(p.New, ", ")
}
var to []string
for _, m := range p.Machines {
if len(m.Receives) > 0 {
to = append(to, m.Machine)
}
}
if len(to) > 0 {
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
} else {
parts = append(parts, "builds "+what+", sent nowhere")
}
bus := "no bus step"
for _, s := range p.Steps {
if strings.HasPrefix(s, "a planned bus step") {
bus = "a bus step"
}
}
parts = append(parts, bus)
waiting := 0
for _, m := range p.Machines {
waiting += len(m.Waits)
}
if waiting > 0 {
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
}
return strings.Join(parts, "; ")
}
// planText is a change plan as a person reads it, for the pull request's comment.
func planText(p link.ChangePlan) string {
var b strings.Builder
fmt.Fprintf(&b, "change plan of %s at %.8s into %s: %s\n", p.Repository, p.Head, p.Base, p.Summary)
for i, tier := range p.Tiers {
fmt.Fprintf(&b, " tier %d: %s\n", i, strings.Join(tier, ", "))
}
for _, m := range p.Machines {
line := " " + m.Machine + ": "
if len(m.Receives) > 0 {
line += "receives " + strings.Join(m.Receives, ", ")
}
if len(m.Waits) > 0 {
if len(m.Receives) > 0 {
line += "; "
}
line += "waits for a person: " + strings.Join(m.Waits, ", ")
}
b.WriteString(line + "\n")
}
for _, s := range p.Steps {
b.WriteString(" - " + s + "\n")
}
if len(p.Unread) > 0 {
fmt.Fprintf(&b, " read by no module's build: %s\n", strings.Join(p.Unread, ", "))
}
return b.String()
}
cmd/mesh-controller/changeplan_test.go
-74
View File
@@ -1,74 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
// person, a provider whose consumers follow it.
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
entry := func(name, path string, on ...string) inventory.Entry {
e := fromRepo(name, catalogue_, path)
e.Source.BuiltFrom = "old"
e.On = on
return e
}
nats := entry("nats", "modules/nats", "anchor")
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
gitea := entry("gitea", "modules/gitea", "anchor")
held := entry("photos", "modules/photos", "anchor", "laptop")
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
tools.On = []string{"anchor", "laptop"}
entries := []inventory.Entry{nats, gitea, held, tools}
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
policy := func(module string) (inventory.Upgrade, bool) {
if module == "photos" {
return inventory.Upgrade{RollOut: false, From: "a person"}, true
}
return inventory.Upgrade{RollOut: true}, true
}
plan := func(paths ...string) link.ChangePlan {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
}
p := plan("modules/gitea/index.ts")
if p.Summary != "builds gitea → anchor; no bus step" {
t.Errorf("one module's change reads %q", p.Summary)
}
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
if !strings.Contains(p.Summary, "a bus step") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
t.Errorf("the bus and a held module read %q", p.Summary)
}
got := map[string]string{}
for _, m := range p.Machines {
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
}
// The bus first, what stands on it after: the build plan's order, per machine.
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
t.Errorf("the deploy plan reads %v", got)
}
text := strings.Join(p.Steps, "\n")
for _, want := range []string{"a planned bus step", "photos waits for a person", "nats provides mesh-bus"} {
if !strings.Contains(text, want) {
t.Errorf("the steps do not say %q:\n%s", want, text)
}
}
p = plan("merge-check.sh")
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
t.Errorf("a root file's plan reads %+v", p)
}
if !strings.Contains(planText(p), "read by no module's build: merge-check.sh") {
t.Errorf("the plan's text does not say why nothing is built:\n%s", planText(p))
}
}
cmd/mesh-controller/check.go
+1 -38
View File
@@ -25,17 +25,7 @@ import (
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
// refused what it could not see would teach people to ignore it.
//
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
// provider's machine when a real machine's name first meets the module's.
func moduleCheck(paths []string, out io.Writer) error {
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
}
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
if len(paths) == 0 {
return errors.New("module check <manifest.json>... — one file per module; pass every " +
"manifest of a repository together so the rules between them are checked too")
@@ -84,24 +74,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
}
failed += len(problems)
// Between the manifests too: an identity against the bounds of the provisions it wants, which
// only the provider's manifest states.
identities := catalogue.IdentityProblems(shelf, longestMachine)
sort.Strings(identities)
for _, p := range identities {
fmt.Fprintln(out, p)
}
failed += len(identities)
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
data := catalogue.DataProblems(shelf)
sort.Strings(data)
for _, p := range data {
fmt.Fprintln(out, p)
}
failed += len(data)
var names []string
for name := range shelf {
names = append(names, name)
@@ -130,14 +102,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
if len(m.Reads) > 0 {
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
}
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
if items := m.DataItems(); len(items) > 0 {
kept := make([]string, 0, len(items))
for _, it := range items {
kept = append(kept, it.ID+" ("+it.Class+")")
}
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
}
fmt.Fprintln(out)
}
if failed > 0 {
@@ -145,8 +109,7 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
}
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
"against the bounds of the providers given here\n", len(paths), longestMachine)
"module not given here reads as unknown\n", len(paths))
return nil
}
cmd/mesh-controller/check_here.go
-201
View File
@@ -1,201 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
// runs it** (novox/hq issue 286): the same code (builder.Check), the same ask the controller would make of
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
// the mesh runs.
//
// **The build seat is the reference.** A merge-check.sh that passed on an agent's machine failed on the
// seat for three reasons that were each the agent's environment, never the change: a newer Go whose gofmt
// lays a file out differently, a sibling checkout at the agent's feature branch where the seat had the
// commit the mesh runs, and a different user. Run here, a check sees what the seat will.
//
// check-here [--tree <checkout>] [--base main] [--registry <artifact store>] [--forge <url of the owner>]
// [--number <n>] [--user uid:gid] [--facts store|<file>] [--keep]
//
// The checkout's HEAD is what is checked, and it must be committed: the seat checks a commit, never a
// working tree.
func checkHereCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("check-here", flag.ContinueOnError)
tree := set.String("tree", ".", "the change's checkout; its HEAD is checked")
base := set.String("base", "main", "the branch the change would merge into")
registry := set.String("registry", os.Getenv("MESH_REGISTRY"), "the artifact store holding the facts and the toolchains")
forge := set.String("forge", "", "where the repositories beside it are cloned from, as <forge>/<repository>.git; "+
"the checkout's origin without its own name when not given")
number := set.Int("number", 0, "the pull request's number, when there is one")
// The build seat's service runs as root, and its check containers run as the builder does.
user := set.String("user", "0:0", "the user the check's containers run as: the build seat's")
keep := set.Bool("keep", false, "keep the workspace afterwards")
factsFrom := set.String("facts", "store", "the facts snapshot: `store`, the one the artifact store holds — "+
"what the seat reads — or a file")
if _, err := parseAround(set, args); err != nil {
return err
}
if *registry == "" {
return errors.New("check-here reads the facts and the toolchains from the artifact store: --registry <host:port> or MESH_REGISTRY")
}
dir, err := filepath.Abs(*tree)
if err != nil {
return err
}
git := func(args ...string) (string, error) {
cmd := exec.CommandContext(ctx, "git", args...)
cmd.Dir = dir
out, err := cmd.Output()
return strings.TrimSpace(string(out)), err
}
if dirty, err := git("status", "--porcelain", "--untracked-files=no"); err != nil {
return fmt.Errorf("%s is not a checkout: %w", dir, err)
} else if dirty != "" {
return errors.New("the checkout has changes not committed: the build seat checks a commit, so commit first")
}
head, err := git("rev-parse", "HEAD")
if err != nil {
return err
}
origin, err := git("remote", "get-url", "origin")
if err != nil {
return fmt.Errorf("the checkout has no origin to say which repository it is: %w", err)
}
owner, repo, prefix := ownerRepoOf(origin)
if *forge == "" {
*forge = prefix
}
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
}
changedText, err := git("diff", "--name-only", "origin/"+*base+"...HEAD")
if err != nil {
return err
}
var paths, removed []string
for _, p := range strings.Split(changedText, "\n") {
if p = strings.TrimSpace(p); p == "" {
continue
}
paths = append(paths, p)
if _, err := os.Stat(filepath.Join(dir, p)); os.IsNotExist(err) {
removed = append(removed, p)
}
}
_ = os.Setenv("MESH_REGISTRY", *registry)
f, err := readFacts(ctx, *factsFrom)
if err != nil {
return fmt.Errorf("the facts snapshot cannot be read: %w", err)
}
entries, read, edges, err := graphOfFacts(f)
if err != nil {
return err
}
p := link.PullUpdated{Owner: owner, Repo: repo, Number: *number, Base: *base, Commit: head, Paths: paths,
Removed: removed, ModuleDirs: moduleDirsIn(dir, paths), ModuleDirsSaid: true}
scope := pullScope(p, entries, read, edges)
_, scriptErr := os.Stat(filepath.Join(dir, builder.CheckScript))
if !scope.gated() && !scope.Mesh {
fmt.Printf("%s: %s, and the repository is not the mesh's: the build seat runs nothing for it\n",
owner+"/"+repo, noModuleTouched)
return nil
}
if !scope.gated() && scriptErr != nil {
fmt.Printf("%s: %s; %s\n", owner+"/"+repo, noModuleTouched, noMergeCheck)
return nil
}
toolchains := map[string]string{}
for language, reference := range f.Versions.Toolchains {
toolchains[language] = catalogue.Rerouted(reference, *registry)
}
if len(toolchains) == 0 {
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
"issue 286, and the seat's toolchain cannot be known here")
}
beside := map[string]builder.Beside{}
for d, ref := range f.Beside {
from := d
if d == "mesh-controller-main" {
from = "mesh-controller"
}
beside[d] = builder.Beside{Repository: strings.TrimSuffix(*forge, "/") + "/" + from + ".git", Ref: ref}
}
id := fmt.Sprintf("check-here-%d", time.Now().UnixNano())
spec := builder.CheckSpec{ID: id, Repository: dir, Ref: head, Owner: owner, Repo: repo, Number: *number,
Paths: paths, Beside: beside, Modules: scope.Modules, New: scope.New, Manifests: scope.Manifests,
Base: *base, Judge: scope.Judge, Toolchain: toolchains["go"], Toolchains: toolchains, User: *user}
workspace, err := os.MkdirTemp("", "mesh-check-here-")
if err != nil {
return err
}
if !*keep {
defer removeWorkspace(spec.Toolchain, workspace, *user)
}
fmt.Fprintf(os.Stderr, "checking %s/%s at %.8s as the build seat would, against the facts of %s, in %s\n",
owner, repo, head, f.Taken.Format(time.RFC3339), workspace)
v, err := builder.Check(ctx, builder.Command, spec, workspace, *registry, builder.GitCredential{},
func(step, message string) {
if step != "output" {
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
}
})
if err != nil {
return fmt.Errorf("the check could not run — on the seat an error, never a pass: %w", err)
}
fmt.Println(v.Report)
fmt.Println()
fmt.Printf("mesh/merge-gate: %s — %s\n", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary)
if v.Repo != nil {
fmt.Printf("mesh/repo-check: %s — %s\n", strings.ToUpper(v.Repo.Verdict), v.Repo.Summary)
}
for _, l := range []*builder.Layer{v.Gate, v.Repo} {
if l != nil && l.Verdict != "pass" && l.Verdict != "warning" {
return errors.New("the build seat would not pass this change")
}
}
return nil
}
// ownerRepoOf reads owner, repository and the owner's URL from a remote: ssh://git@host:222/novox/mesh-host.git
// → novox, mesh-host, ssh://git@host:222/novox.
func ownerRepoOf(remote string) (string, string, string) {
trimmed := strings.TrimSuffix(strings.TrimSuffix(remote, "/"), ".git")
cut := strings.LastIndexAny(trimmed, "/:")
if cut < 0 {
return "", trimmed, ""
}
repo, prefix := trimmed[cut+1:], trimmed[:cut]
owner := prefix
if at := strings.LastIndexAny(prefix, "/:"); at >= 0 {
owner = prefix[at+1:]
}
return owner, repo, prefix
}
// removeWorkspace removes what the check left, written as the seat's user: by a container of that user
// when it is not this one.
func removeWorkspace(image, workspace, user string) {
if image != "" && user != fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) {
// Everything in it — the check's HOME is the workspace, so the toolchain's own files are there too.
_ = exec.Command("docker", "run", "--rm", "--user", user, "--volume", workspace+":/workspace", image,
"sh", "-c", "rm -rf /workspace/* /workspace/.[!.]*").Run()
}
_ = os.RemoveAll(workspace)
}
cmd/mesh-controller/check_test.go
-11
View File
@@ -92,14 +92,3 @@ func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
t.Fatalf("a name declared on purpose passes; got %v", err)
}
}
// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged
// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it —
// refused it. Held here, so a verb added to the table without the manifest fails this repository's
// own suite rather than its next pull request's gate.
func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
var out strings.Builder
if err := moduleCheck([]string{"../../module.json"}, &out); err != nil {
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
}
}
cmd/mesh-controller/checks.go
-400
View File
@@ -1,400 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"path"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A pull request's merge check (novox/hq to-be 45 §9, ADR 0237 as amended 2026-10-06): the forge
// announces every pull request's new head, the controller decides what is checked, asks the build seat to
// check it, and says the verdict as `checked`, which the forge's holder sets as the pull request's
// statuses. **Before merge, never after**: every check the mesh had ran after a merge, on a machine.
//
// **The mesh's module graph decides, not the repository.** The controller holds the graph — every module,
// the repository and directory it is built from — and maps the pull request's changed paths onto it by
// the planner's own answer (reachOfMerge, touchedBy — the one place a changed file is mapped onto modules,
// for the merge handler, the release planner, the merge gate and this check; issue 280): **a changed file
// touches exactly the modules whose build reads it** — a module's own directory (the whole repository for
// one built from its root), or a repository its recipe packages. A file no build reads — a script at the
// root, a README — touches no module. A directory the change adds a module.json in, which the graph does
// not hold yet (said by the head, issue 278), is a new module and is checked too.
//
// - touches a module: the build seat runs **the gate** — `mesh/merge-gate`, the touched manifests, every
// machine composed with the change, the replays — and the repository's own merge-check.sh beside it;
// - touches none, in a repository that is the mesh's (it sources a module on some branch, or shares the
// core's owner): the gate is a pass that says so — a fact, not a missing check — and the repository's
// own merge-check.sh runs as `mesh/repo-check`, a warning when it has none;
// - touches none, anywhere else: the gate is a pass that says so, and nothing more is said.
//
// What is checked is decided here and run there (internal/builder/check.go).
// checkTimeout is how long one check may run on the build seat. Said here so the ask's watchdog (S6)
// and the builder agree on what late means.
const checkTimeout = 45 * time.Minute
// noModuleTouched is the gate's word for a change that touches nothing of the graph.
const noModuleTouched = "the change touches no module of the mesh's graph"
// noMergeCheck is the repository layer's word for a repository of the mesh with no merge-check.sh.
const noMergeCheck = "the repository declares no merge-check.sh: none of its own tests run before it merges"
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
// each beside it — and whose owner is the mesh's own.
var coreModules = map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host",
"node-tools": "mesh-tools", "nats": "mesh-catalog"}
// checkScope is what a pull request reaches of the mesh's graph, as the planner reckons it.
type checkScope struct {
// Modules are the modules a merge of the change would move itself — built from the repository into
// the pull request's base and reading a changed file, or packaging the repository's source — and
// Dependents those the plan would build after them; New the directories it adds a module in.
Modules []string
Dependents []string
New []string
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
Manifests []string
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
// module's build reads.
Width, Tiers int
Unread []string
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
// core's, or the change adds a module to it.
Mesh bool
// Judge is who judges the gate (link.JudgeSelf, link.JudgeValidator, or the running controller).
Judge string
// From is a module built from the repository, for how the mesh clones it; nil when none is.
From *inventory.Entry
// Reach is the planner's whole answer, which the change plan is made from.
Reach mergeReach
}
func (s checkScope) gated() bool { return len(s.Modules)+len(s.New) > 0 }
// pullScope is what a pull request reaches: **the planner's own answer** (reachOfMerge), asked as if the
// head were merged into the base — never a mapping of its own, so a change to what a merge touches
// changes what is checked with it (novox/hq ADR 0238).
func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
edges []inventory.Edge) checkScope {
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL, Commit: p.Commit,
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
ModuleDirsSaid: p.ModuleDirsSaid}
r := reachOfMerge(m, entries, read, edges)
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
switch e.Manifest.Module {
case "mesh-controller":
s.Judge = link.JudgeSelf
case "mesh-host":
if s.Judge == "" {
s.Judge = link.JudgeValidator
}
}
}
for _, d := range r.Added {
s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile))
}
sort.Strings(s.Manifests)
s.Manifests = slices.Compact(s.Manifests)
// Whose repository it is, for how it is cloned and whether its own check is the mesh's to run.
owners := map[string]bool{}
for i, e := range entries {
if e.Provided {
continue
}
if _, core := coreModules[e.Manifest.Module]; core {
if owner := sourceOwner(e.Source.Repository); owner != "" {
owners[owner] = true
}
}
if sameRepository(e.Source.Repository, m) && s.From == nil {
s.From = &entries[i]
}
}
s.Mesh = s.From != nil || owners[strings.ToLower(p.Owner)] || s.gated()
return s
}
// sourceOwner is the owner of a recorded repository, a path on the git seat or a URL: novox/mesh-host → novox.
func sourceOwner(repository string) string {
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
if len(parts) < 2 {
return ""
}
return strings.ToLower(parts[len(parts)-2])
}
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
inv := f.open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return err
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return err
}
scope := pullScope(p, entries, read, edges)
// The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted
// with the verdict whatever the verdict is.
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
u, err := inv.UpgradeOf(ctx, module)
return u, err == nil
})
fmt.Print(planText(plan))
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
switch {
case !scope.gated() && !scope.Mesh:
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
sayChecked(ctx, direct)
return nil
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
sayChecked(ctx, direct)
return nil
}
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
if err != nil {
return err
}
request.Check.Plan = &plan
seat := buildSeatHeld(ctx)
ask, err := askOverOn(seat)
if err != nil {
return err
}
defer ask.Close()
if err := ask.Ask(ctx, request); err != nil {
return err
}
what := "its own merge-check.sh alone: " + noModuleTouched
if scope.gated() {
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
scope.Width, scope.Tiers)
}
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
p.Commit, seat, what, request.ID)
return nil
}
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
// and what is read beside it.
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
entries []inventory.Entry) (link.BuildRequest, error) {
shelf := map[string]catalogue.Manifest{}
for _, e := range entries {
shelf[e.Manifest.Module] = e.Manifest
}
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
if err != nil {
return link.BuildRequest{}, err
}
clone := func(s inventory.Source) (string, error) {
if s.Seat == "" {
return s.Repository, nil
}
return clonedFromSeat(world, s.Seat, s.Repository)
}
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
source := inventory.Source{Seat: gitSeat, Repository: p.Owner + "/" + p.Repo}
if scope.From != nil {
source = scope.From.Source
}
repository, err := clone(source)
if err != nil {
return link.BuildRequest{}, err
}
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return link.BuildRequest{}, err
}
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
beside := map[string]link.CheckedOut{}
for _, e := range entries {
dir, core := coreModules[e.Manifest.Module]
if !core || e.Provided || e.Source.Repository == "" {
continue
}
url, err := clone(e.Source)
if err != nil {
return link.BuildRequest{}, err
}
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
if dir == "mesh-controller" {
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
if e.Source.Seat != "" {
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
"mesh-lab")}); err == nil {
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
}
}
}
}
return link.BuildRequest{
ID: link.NewBuildID(time.Now()),
Repository: repository,
Ref: p.Commit,
Held: heldBy(ctx),
Seats: seatBases(ctx),
Source: sourceOnSeat(source),
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
Manifests: scope.Manifests, Judge: scope.Judge},
}, nil
}
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
// builds from; and beside the controller its main, for a judge the running controller predates, and the
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
func besideRefs(dir, running string) map[string]string {
switch dir {
case "mesh-catalog":
return map[string]string{dir: "main"}
case "mesh-controller":
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
}
return map[string]string{dir: running}
}
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
func siblingOf(repository, name string) string {
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
return repository[:cut+1] + name
}
return name
}
// sourceOnSeat is a source's seat form, nil for one on no seat.
func sourceOnSeat(s inventory.Source) *link.SourceOnSeat {
if s.Seat == "" {
return nil
}
return &link.SourceOnSeat{Seat: s.Seat, Repository: s.Repository}
}
// checkEvents is where the serving controller says a check's verdict; nil in a command.
var checkEvents link.Bus
// maxCheckReport is how much of a check's report travels in its verdict: enough for the failures and
// the machines, never a log.
const maxCheckReport = 60 << 10
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
func checked(ctx context.Context, result link.BuildResult) {
sayChecked(ctx, checkedOf(result))
}
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
// could not run.
func checkedOf(result link.BuildResult) link.Checked {
c := link.Checked{ID: result.ID, On: result.On, Commit: result.Ref}
if result.Checked != nil {
c.Owner, c.Repo, c.Number = result.Checked.Owner, result.Checked.Repo, result.Checked.Number
}
switch {
case result.Check != nil:
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
if c.Gate == nil {
// A build seat from before the layers: its verdict is the gate's.
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
}
case result.Failed != "":
// The check could not run: an error, never read as a pass — on both layers it was asked for.
c.Verdict, c.Summary = "error", "the check could not run: "+firstLine(result.Failed)
default:
c.Verdict, c.Summary = "error", "the build seat answered the check with no verdict"
}
if c.Verdict == "" {
c.Verdict = "error"
}
if result.Check == nil {
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
}
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
}
if result.Checked != nil && c.Gate != nil && len(c.Gate.Dependents) == 0 {
c.Gate.Dependents = result.Checked.Dependents
}
if result.Checked != nil {
c.Plan = result.Checked.Plan
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
if g := result.Checked; g.Group != "" {
c.Group = g.Group
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
Commit: result.Ref})
for _, m := range g.Members {
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
Commit: m.Ref})
}
}
}
for _, l := range []*link.CheckLayer{c.Gate, c.RepoCheck} {
if l != nil && l.Verdict == "" {
l.Verdict = "error"
}
}
if len(c.Report) > maxCheckReport {
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
}
return c
}
func prefixedAll(prefix string, items []string) []string {
out := make([]string, 0, len(items))
for _, i := range items {
out = append(out, prefix+i)
}
return out
}
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
func sayChecked(ctx context.Context, c link.Checked) {
repo := "none"
if c.RepoCheck != nil {
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
}
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
if checkEvents == nil {
return
}
body, err := json.Marshal(c)
if err != nil {
return
}
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer stop()
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
}
}
cmd/mesh-controller/checks_test.go
-212
View File
@@ -1,212 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **The mesh's module graph decides what a pull request's check runs**, not the repository (novox/hq
// ADR 0237 as amended): a change is mapped onto the graph by the rule a merge is (issue 278), the gate
// runs when it touches a module — a new one included — and a repository that is the mesh's and touches
// none still has its own merge-check.sh run.
func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) {
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
const controller = "http://forge.internal:20000/novox/mesh-controller.git"
const host = "http://forge.internal:20000/novox/mesh-host.git"
photos := fromRepo("photos", "http://forge.internal:20000/novox/photos.git", "")
photos.Source.Ref = "nox-mesh"
snake := fromRepo("snake", "jschoubben/snake", "")
snake.Source.Seat = "git"
entries := []inventory.Entry{
fromRepo("gitea", catalogue, "modules/gitea"),
fromRepo("keycloak", catalogue, "modules/keycloak"),
fromRepo("nats", catalogue, "modules/nats"),
fromRepo("mesh-controller", controller, ""),
fromRepo("mesh-host", host, ""),
photos, snake,
}
pull := func(owner, repo, base string, paths []string, dirs ...string) link.PullUpdated {
return link.PullUpdated{Owner: owner, Repo: repo, Base: base, Commit: "abc", Paths: paths,
ModuleDirs: dirs, ModuleDirsSaid: true}
}
for _, c := range []struct {
what string
p link.PullUpdated
modules, new, manifest string
mesh bool
judge string
}{
{"one module's own files", pull("novox", "mesh-catalog", "main", []string{"modules/gitea/index.ts"}, "modules/gitea"),
"gitea", "", "modules/gitea/module.json", true, ""},
{"a file no module's build reads touches no module (issue 280)",
pull("novox", "mesh-catalog", "main", []string{"merge-check.sh", "README.md"}), "", "", "", true, ""},
{"files the announcer could not list: everything built from it",
link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", PathsTruncated: true,
Paths: []string{"README.md"}}, "gitea,keycloak,nats", "",
"modules/gitea/module.json,modules/keycloak/module.json,modules/nats/module.json", true, ""},
{"a new module, said by the head", pull("novox", "mesh-catalog", "main",
[]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, "modules/newmod"),
"", "modules/newmod", "modules/newmod/module.json", true, ""},
{"the controller judges itself", pull("novox", "mesh-controller", "main", []string{"cmd/x.go"}),
"mesh-controller", "", "module.json", true, "self"},
{"the node-engine is judged with its validator", pull("novox", "mesh-host", "main", []string{"validate/v.go"}),
"mesh-host", "", "module.json", true, "validator"},
{"the core's owner, no module: the mesh's, its own check alone", pull("novox", "hq", "main", []string{"README.md"}),
"", "", "", true, ""},
{"a branch nothing is built from: the mesh's repository, no module", pull("novox", "photos", "master",
[]string{"server/x.js"}), "", "", "", true, ""},
{"the branch a module is built from", pull("novox", "photos", "nox-mesh", []string{"server/x.js"}),
"photos", "", "module.json", true, ""},
{"a repository on the forge's seat", pull("jschoubben", "snake", "main", []string{"index.html"}),
"snake", "", "module.json", true, ""},
{"a repository of nobody's, touching nothing", pull("someone", "dotfiles", "main", []string{"x"}),
"", "", "", false, ""},
{"a repository adding a module at its root", pull("someone", "newapp", "main", []string{"module.json", "x.js"}),
"", ".", "module.json", true, ""},
} {
s := pullScope(c.p, entries, nil, nil)
got := []string{strings.Join(s.Modules, ","), strings.Join(s.New, ","), strings.Join(s.Manifests, ",")}
want := []string{c.modules, c.new, c.manifest}
for i, what := range []string{"modules", "new", "manifests"} {
if got[i] != want[i] {
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
}
}
if s.Mesh != c.mesh || s.Judge != c.judge {
t.Errorf("%s: the mesh's %v judged by %q, wanted %v by %q", c.what, s.Mesh, s.Judge, c.mesh, c.judge)
}
if s.gated() != (c.modules != "" || c.new != "") {
t.Errorf("%s: gated %v", c.what, s.gated())
}
}
}
// A module whose build packages another repository's source is touched by a change to it.
func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
entries := []inventory.Entry{fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")}
read := map[string][]inventory.ReadRepository{"node-tools": {{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}}
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "abc", Paths: []string{"go/x.go"}}, entries, read, nil)
if strings.Join(s.Modules, ",") != "node-tools" || len(s.Manifests) != 0 {
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
}
}
// Each layer is said; a check that could not run is an error on both, never a pass; a build seat from
// before the layers is read as the gate.
func TestAChecksLayersAreEachSaidAndAnErrorIsNeverAPass(t *testing.T) {
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3, Modules: []string{"gitea"}}
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
if c.Verdict != "error" || c.Gate == nil || c.Gate.Verdict != "error" || c.RepoCheck == nil || c.RepoCheck.Verdict != "error" {
t.Fatalf("a check that could not run said %+v", c)
}
if strings.Join(c.Gate.Modules, ",") != "gitea" {
t.Errorf("the gate names %v", c.Gate.Modules)
}
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "warning", Summary: "wide"}})
if c.Gate == nil || c.Gate.Verdict != "warning" || c.RepoCheck != nil {
t.Fatalf("an outcome without layers said %+v", c)
}
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "pass",
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "its merge-check.sh failed"}}})
if c.RepoCheck.Verdict != "fail" || c.Gate.Verdict != "pass" {
t.Fatalf("the layers said %+v / %+v", c.Gate, c.RepoCheck)
}
}
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
// module in both, and what stands on a touched module is named as its dependents in both.
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
nats := fromRepo("nats", catalogue, "modules/nats")
nats.Source.BuiltFrom = "old"
gitea := fromRepo("gitea", catalogue, "modules/gitea")
gitea.Source.BuiltFrom = "old"
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
entries := []inventory.Entry{nats, gitea, tools}
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
read := map[string][]inventory.ReadRepository{}
for _, c := range []struct {
what string
paths []string
moved, dependents string
width int
unread string
}{
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
} {
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
s := pullScope(p, entries, read, edges)
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
r := reachOfMerge(m, entries, read, edges)
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
}
if got := strings.Join(s.Dependents, ","); got != c.dependents {
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
}
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
}
if got := strings.Join(s.Unread, ","); got != c.unread {
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
}
}
// A repository whose build another module's recipe packages: that module is reached through the
// planner's `read`, and what stands on it after it.
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
Paths: []string{"src/index.ts"}}, entries, read, edges)
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
}
}
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a build of a commit off its repository's
// default branch — a pull request's head, a branch built by hand, a rebuild or replay of one — is recorded
// and never registered, so nothing can send it; a check or a dry run never is either.
func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
on := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main", OnTrunk: true}
if err := publishable(on, ""); err != nil {
t.Errorf("a build on the trunk was refused: %v", err)
}
off := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main"}
if err := publishable(off, ""); !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "main") {
t.Errorf("a build off the trunk was let through: %v", err)
}
for _, r := range []link.BuildResult{
{ID: "c", Trunk: "main", OnTrunk: true, Check: &link.CheckOutcome{Verdict: "pass"}},
{ID: "d", Trunk: "main", OnTrunk: true, Checked: &link.CheckRequest{}},
{ID: "e", Trunk: "main", OnTrunk: true, DryRun: true},
} {
if publishable(r, "") == nil {
t.Errorf("%s, a check or a dry run, was publishable", r.ID)
}
}
// A module that follows a branch other than the default: that branch is its trunk, and the default
// is not.
follows := link.BuildResult{ID: "g", Commit: "abc", Trunk: "master", Branches: []string{"nox-mesh"}}
if err := publishable(follows, "nox-mesh"); err != nil {
t.Errorf("a commit on the branch a module follows was refused: %v", err)
}
if err := publishable(link.BuildResult{ID: "h", Commit: "abc", Trunk: "master", OnTrunk: true,
Branches: []string{"master"}}, "nox-mesh"); err == nil {
t.Error("a commit on the default but not on the branch the module follows was published")
}
if err := publishable(link.BuildResult{ID: "i", Commit: "abc", Trunk: "main", Branches: []string{"feat/x"}}, ""); err == nil {
t.Error("a feature branch's commit was published")
}
// A build seat older than the rule says nothing: let through and said, so the rule can reach the mesh.
if err := publishable(link.BuildResult{ID: "f", Commit: "abc"}, ""); err != nil {
t.Errorf("a build seat that said nothing was refused: %v", err)
}
}
cmd/mesh-controller/collect.go
+6 -73
View File
@@ -31,12 +31,7 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
return
}
kept, err := inv.KeptArchives(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
return
}
if len(references) == 0 && len(kept) == 0 {
if len(references) == 0 {
return
}
shelf, err := inv.Catalogue(ctx)
@@ -64,27 +59,8 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
defer stop()
store := artifacts.Store{Address: address}
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
// manifest names, and archives were published as bare blobs, so every kept archive is first
// held by its manifest — which backfills the ones published before holders, a few at a time
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
// the sweep before it deletes anything: "everything kept is held" is the precondition the
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
wrote, missing, err := holdKept(within, store, kept)
if wrote > 0 {
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
}
if missing > 0 {
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
"`collection` lists them\n", missing)
}
if err != nil {
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
return
}
var done []string
var left, skipped int
var left int
for i, reference := range references {
if i >= mostPerSweep || within.Err() != nil {
left = len(references) - i
@@ -97,22 +73,10 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
done = append(done, reference)
continue
}
if errors.Is(err, artifacts.ErrNotOurs) {
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
// marked collected — the mesh did not remove it and should not claim to — and not a
// reason to stop, because the store was never asked. One of these at the front of
// the oldest-first order ended every sweep until this.
skipped++
if skipped == 1 {
fmt.Fprintf(os.Stderr,
"the sweep will not address %s and went on: %v\n", reference, err)
}
continue
}
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
// one refuses all of them — deletion disabled, the store down, the network gone — so
// going on would be a hundred identical failures and a hundred identical log lines in
// front of whoever was building something.
// **Stopped at the first refusal, not pushed through.** A store that refuses one refuses
// all of them — deletion disabled, the store down, the network gone — so going on would
// be a hundred identical failures and a hundred identical log lines in front of whoever
// was building something.
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
reference, err)
left = len(references) - i
@@ -133,37 +97,6 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
if left > 0 {
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
}
if skipped > 0 {
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
}
}
// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
// Answers how many holders it wrote and how many kept archives the store does not have.
//
// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
// reference the store cannot be asked about is skipped as the deletion loop skips one
// (novox/hq issue 226).
func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
for _, reference := range kept {
if err := ctx.Err(); err != nil {
return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
}
did, err := store.Hold(ctx, reference)
switch {
case err == nil:
if did {
wrote++
}
case errors.Is(err, artifacts.Gone):
missing++
case errors.Is(err, artifacts.ErrNotOurs):
default:
return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
}
}
return wrote, missing, nil
}
// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
cmd/mesh-controller/collection.go
-166
View File
@@ -1,166 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"strings"
"github.com/novox/mesh-controller/internal/artifacts"
)
// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
// (novox/hq issue 253, ADR 0189).
//
// **The question to answer before the store's collector runs for real.** The collector deletes
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
// dry run go.
//
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
// new verb in the seat's row.
type collectionReport struct {
// Store is the artifact store as this machine reached it; empty when it is not on the network.
Store string `json:"store"`
// KeptArchives is how many archives the mesh keeps, for either reason.
KeptArchives int `json:"kept_archives"`
// Held is how many of them the store holds by their manifest.
Held int `json:"held"`
// Unheld are the kept archives the collector would delete tonight if it ran for real.
Unheld []string `json:"unheld"`
// Missing are kept archives the store does not have at all.
Missing []string `json:"missing"`
// Unasked is how many could not be asked about, and why the asking stopped.
Unasked int `json:"unasked"`
Stopped string `json:"stopped,omitempty"`
// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
// collected — split by kind.
Eligible int `json:"eligible"`
EligibleImages int `json:"eligible_images"`
EligibleArchives int `json:"eligible_archives"`
// SafeToCollect is whether every kept archive was asked about and every one is held.
SafeToCollect bool `json:"safe_to_collect"`
}
func collectionCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("collection", flag.ContinueOnError)
asJSON := set.Bool("json", false, "answer as JSON")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 0 {
return errors.New("collection [--json]")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
kept, err := inv.KeptArchives(ctx)
if err != nil {
return err
}
eligible, err := inv.ToCollect(ctx)
if err != nil {
return err
}
report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
for _, reference := range eligible {
if strings.Contains(reference, "/blobs/") {
report.EligibleArchives++
} else {
report.EligibleImages++
}
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
if err != nil {
return err
}
if report.Store == "" {
report.Unasked = len(kept)
report.Stopped = "this mesh has no artifact store on its network"
} else {
report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
}
report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
if *asJSON {
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
return encoder.Encode(report)
}
printCollection(report)
return nil
}
// askHeld asks the store about each kept archive, stopping at the first answer that is not about
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
// identical failures says less than one line.
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
for i, reference := range kept {
held, err := store.Held(ctx, reference)
switch {
case err == nil && held:
report.Held++
case err == nil:
report.Unheld = append(report.Unheld, reference)
case errors.Is(err, artifacts.Gone):
report.Missing = append(report.Missing, reference)
case errors.Is(err, artifacts.ErrNotOurs):
// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
report.Unasked++
default:
return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
}
}
return report.Unasked, ""
}
func printCollection(r collectionReport) {
store := r.Store
if store == "" {
store = "(not on the network)"
}
fmt.Printf("artifact store %s\n", store)
fmt.Printf("kept archives %d\n", r.KeptArchives)
fmt.Printf(" held %d\n", r.Held)
fmt.Printf(" unheld %d\n", len(r.Unheld))
fmt.Printf(" missing %d\n", len(r.Missing))
if r.Unasked > 0 {
fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped)
}
fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
if len(r.Unheld) > 0 {
fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
for _, reference := range r.Unheld {
fmt.Printf(" %s\n", reference)
}
}
if len(r.Missing) > 0 {
fmt.Println("\nmissing — kept by the mesh, not in the store:")
for _, reference := range r.Missing {
fmt.Printf(" %s\n", reference)
}
}
fmt.Println()
if r.SafeToCollect {
fmt.Println("every kept archive is held: the store's collector may run for real")
} else {
fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
}
}
cmd/mesh-controller/conditions.go
-441
View File
@@ -1,441 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"os"
"slices"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
//
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
// while, with a reason, and that is recorded as a hand act.
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
var conditionsFrom *conditions.Keeper
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
store, history, err := conditions.OnTheBus(ctx, conn)
if err != nil {
return nil, err
}
js, err := conn.JetStream()
if err != nil {
return nil, err
}
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
Teller: link.OverNATS{Conn: conn, JS: js},
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
// What status leads with changed: composed again soon (a nudge outside the serving controller
// does nothing).
Changed: statusFrom.nudge,
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
}
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
// it was given before it returns.
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
if conditionsFrom != nil {
return f(conditionsFrom)
}
return onTheBus(func(conn *nats.Conn) error {
k, err := keeperOn(ctx, conn)
if err != nil {
return err
}
defer func() {
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
k.Close(flushing)
}()
return f(k)
})
}
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
if conditionsFrom != nil {
return conditionsFrom.Open(ctx)
}
if _, err := broker.BusAddress(); err != nil {
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
}
var out []conditions.Condition
err := onTheBus(func(conn *nats.Conn) error {
store, _, err := conditions.OnTheBus(ctx, conn)
if err != nil {
return err
}
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
out, err = conditions.Read(reading, store)
return err
})
return out, err
}
// conditionsUsage is how the verb is typed.
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
"conditions history [--days N] [--key K] [--json]"
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
func conditionsCommand(ctx context.Context, args []string) error {
sub := "list"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
switch sub {
case "list":
return listConditions(ctx, args)
case "show":
return showCondition(ctx, args)
case "silence":
return silenceCondition(ctx, args)
case "history":
return conditionHistory(ctx, args)
}
return errors.New(conditionsUsage)
}
func listConditions(ctx context.Context, args []string) error {
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
severity := set.String("severity", "", "only urgent, or only warning")
machine := set.String("machine", "", "only those about this machine")
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(conditionsUsage)
}
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
}
open, err := openConditions(ctx)
if err != nil {
return err
}
var out []conditions.Condition
for _, c := range open {
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
(*machine == "" || concerns(c, *machine)) {
out = append(out, c)
}
}
if *asJSON {
if out == nil {
out = []conditions.Condition{}
}
return printJSON(map[string]any{"conditions": out, "open": len(open),
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
}
if len(out) == 0 {
if len(open) == 0 {
fmt.Println("no open conditions")
} else {
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
}
return nil
}
for _, line := range conditionLines(out, time.Now()) {
fmt.Println(line)
}
return nil
}
// concerns says whether a condition is about a machine: it names it, or its key does.
func concerns(c conditions.Condition, machine string) bool {
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
return true
}
for _, part := range strings.Split(c.Key, ".") {
if part == machine {
return true
}
}
return false
}
// conditionLines is how a list of conditions reads: one line each, its silence under it.
func conditionLines(list []conditions.Condition, now time.Time) []string {
var out []string
for _, c := range list {
times := ""
if c.Count > 1 {
times = fmt.Sprintf(", raised %d times", c.Count)
}
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
if c.SilencedAt(now) {
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
}
}
return out
}
func showCondition(ctx context.Context, args []string) error {
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
rest, err := parseAround(set, args)
if err != nil {
return err
}
if len(rest) != 1 {
return errors.New("conditions show <key>")
}
key := rest[0]
var c conditions.Condition
var found bool
if conditionsFrom != nil {
c, found, err = conditionsFrom.Get(ctx, key)
} else {
err = onTheBus(func(conn *nats.Conn) error {
store, _, err := conditions.OnTheBus(ctx, conn)
if err != nil {
return err
}
c, found, err = conditions.ReadOne(ctx, store, key)
return err
})
}
if err != nil {
return err
}
if !found {
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
"history --key %s` what became of it", key, key)
}
if *asJSON {
return printJSON(c)
}
now := time.Now()
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
if c.Subject.Machine != "" {
fmt.Printf(", on %s", c.Subject.Machine)
}
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
now.Sub(c.LastObserved).Round(time.Second))
if c.Count > 1 {
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
}
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
if c.Silenced != nil {
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
c.Silenced.By, c.Silenced.Why)
}
if len(c.Tried) > 0 {
fmt.Println("\n tried:")
for _, t := range c.Tried {
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
}
}
fmt.Println("\n evidence, newest first:")
for _, e := range c.Evidence {
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
}
return nil
}
func resolverWords(r string) string {
switch r {
case conditions.ResolverSelf:
return "itself: it clears when observation says it is resolved"
case conditions.ResolverOperator:
return "the operator: nothing in the mesh will repair it"
case conditions.ResolverAgent:
return "an agent"
}
return r
}
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
// who and why before it is done, its cause the condition's kind unless one is given.
func silenceCondition(ctx context.Context, args []string) error {
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
acts := addHandActFlags(set)
rest, err := parseAround(set, args)
if err != nil {
return err
}
if len(rest) != 1 {
return errors.New("conditions silence <key> --for <duration> --why <text>")
}
key := rest[0]
if err := acts.require("conditions silence"); err != nil {
return err
}
d, err := parseFor(*forFlag)
if err != nil {
return err
}
if d > conditions.MaxSilence {
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
}
return withKeeper(ctx, func(k *conditions.Keeper) error {
c, found, err := k.Get(ctx, key)
if err != nil {
return err
}
if !found {
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
}
if strings.TrimSpace(*acts.cause) == "" {
*acts.cause = c.Kind
}
*acts.condition = key
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
if err != nil {
return err
}
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
"`status` still says it; it clears when observation says it is resolved\n",
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
return nil
})
}
// parseFor reads a duration, days included.
func parseFor(s string) (time.Duration, error) {
s = strings.TrimSpace(s)
if s == "" {
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
}
if days, ok := strings.CutSuffix(s, "d"); ok {
n, err := strconv.Atoi(days)
if err != nil || n <= 0 {
return 0, fmt.Errorf("%q is not a number of days", s)
}
return time.Duration(n) * 24 * time.Hour, nil
}
d, err := time.ParseDuration(s)
if err != nil || d <= 0 {
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
}
return d, nil
}
func conditionHistory(ctx context.Context, args []string) error {
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
days := set.Int("days", 7, "how many days back, at most 90")
key := set.String("key", "", "only this condition")
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("conditions history [--days N] [--key K] [--json]")
}
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
var events []conditions.Event
read := func(h conditions.History) error {
var err error
events, err = h.Since(ctx, since)
return err
}
var err error
if conditionsFrom != nil {
events, err = conditionsFrom.HistorySince(ctx, since)
} else {
err = onTheBus(func(conn *nats.Conn) error {
_, history, err := conditions.OnTheBus(ctx, conn)
if err != nil {
return err
}
return read(history)
})
}
if err != nil {
return err
}
var out []conditions.Event
for _, e := range events {
if *key == "" || e.Key == *key {
out = append(out, e)
}
}
if *asJSON {
if out == nil {
out = []conditions.Event{}
}
return printJSON(map[string]any{"history": out, "days": *days})
}
if len(out) == 0 {
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
return nil
}
for _, e := range out {
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
switch e.Change {
case conditions.ChangeRaised, conditions.ChangeReopened:
line += " — " + e.Summary
case conditions.ChangeSeverity:
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
case conditions.ChangeResolver:
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
default:
if e.Why != "" {
line += " — " + e.Why
}
}
fmt.Println(line)
}
return nil
}
// printConditions is the status section that leads it: every open condition, urgent first, oldest
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
// is not "none open".
func printConditions(list []conditions.Condition, unread string, now time.Time) {
if unread != "" {
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
return
}
if len(list) == 0 {
return
}
urgent := 0
for _, c := range list {
if c.Severity == conditions.Urgent {
urgent++
}
}
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
for _, line := range conditionLines(list, now) {
fmt.Println(line)
}
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
}
// orHealer is who tried, as an attempt names it.
func orHealer(by string) string {
if by == "" {
return "a healer"
}
return by
}
cmd/mesh-controller/conditions_test.go
-107
View File
@@ -1,107 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/conditions"
)
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
// by what is open.
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
for _, c := range []struct {
args map[string]any
want string
}{
{map[string]any{}, "conditions --json"},
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
"conditions history --json --days 3 --key machine.ace.silent"},
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
"conditions silence machine.ace.silent --for 2h --why on the train"},
{map[string]any{"run": "true"}, "doctor run --json"},
{map[string]any{}, "doctor --json"},
} {
verb := "conditions"
if strings.HasPrefix(c.want, "doctor") {
verb = "doctor"
}
argv, err := argvFor(verb, c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
}
}
for _, c := range []struct {
verb string
args map[string]any
}{
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
} {
if argv, err := argvFor(c.verb, c.args); err == nil {
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
}
}
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
t.Error("a silence is not a hand act")
}
}
// **A silence through the verb is recorded and bounded**; the condition stays open.
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
k, _ := withConditionsInMemory(t)
ctx := t.Context()
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
t.Fatal(err)
}
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
t.Fatal("silenced without saying why")
}
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
t.Fatal("silenced for more than a week")
}
said := printed(t, func() error {
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
})
if !strings.Contains(said, "is silenced until") {
t.Fatalf("%s", said)
}
c, found, _ := k.Get(ctx, "machine.ace.silent")
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
t.Fatalf("%+v", c)
}
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
t.Fatalf("%s", listed)
}
}
// **Conditions that cannot be read are not none open**: status says so, and is not well.
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
open := aMesh(t)
_, store := withConditionsInMemory(t)
store.Fail = errors.New("the bus is away")
asked, err := theThreeQuestions(t.Context(), open)
if err != nil {
t.Fatal(err)
}
if asked.well() || asked.conditionsUnread == "" {
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
}
said := printed(t, func() error { return printStatus(asked) })
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
t.Fatalf("%s", said)
}
store.Fail = nil
asked, _ = theThreeQuestions(t.Context(), open)
said = printed(t, func() error { return printStatus(asked) })
if asked.well() && !strings.Contains(said, "no open conditions;") {
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
}
}
cmd/mesh-controller/confirm.go
-67
View File
@@ -1,67 +0,0 @@
package main
import (
"context"
"sync"
"github.com/novox/mesh-controller/internal/conditions"
)
// **A finding one look can be wrong about is raised on the second look in a row** (novox/hq issue 277).
//
// D2 raised its resolver urgent on one unanswered question, asked once, while the resolver's machine
// was loaded by a push and a build starting; thirty questions right after were answered at once. A
// single sample of something that is answered over the network, or timed on a machine under load,
// is not the invariant failing. So a source marks such a finding `Confirm`, and this holds it back:
//
// - raised when the source's previous look saw it too — two runs of the self-check in a row (five
// minutes apart), or two ticks of the watchdogs (half a minute) — at the severity the source says;
// - kept while it is already open, however it is seen, so a condition the next look still sees is
// never cleared and raised again (flapping is not news; a reopening within ReopenWithin still is);
// - cleared, as everything is, by the look that no longer sees it.
//
// A finding held back is not a pass: the self-check's verdict names it as unconfirmed. A finding that is
// a definite answer — a resolver that answered wrongly, a stream that is not there — is not marked, and
// is raised at once.
type confirming struct {
mu sync.Mutex
// last is, by source, the keys of the Confirm findings its previous look saw.
last map[string]map[string]bool
}
// pass splits one source's findings into what is raised now and what is held for the next look, and
// remembers what it saw. An open condition that cannot be read is kept: unknown is not a reason to
// hold a finding back.
func (c *confirming) pass(ctx context.Context, keeper *conditions.Keeper, source string,
found []conditions.Observation) (raise, held []conditions.Observation) {
c.mu.Lock()
defer c.mu.Unlock()
if c.last == nil {
c.last = map[string]map[string]bool{}
}
before, seen := c.last[source], map[string]bool{}
for _, o := range found {
if !o.Confirm {
raise = append(raise, o)
continue
}
key := o.Key()
seen[key] = true
if before[key] || isOpen(ctx, keeper, key) {
raise = append(raise, o)
continue
}
held = append(held, o)
}
c.last[source] = seen
return raise, held
}
// isOpen says whether a condition is open; one that cannot be read is taken as open.
func isOpen(ctx context.Context, keeper *conditions.Keeper, key string) bool {
if keeper == nil {
return false
}
_, open, err := keeper.Get(ctx, key)
return open || err != nil
}
cmd/mesh-controller/data.go
-931
View File
@@ -1,931 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"log"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A module declares the data it holds, and the mesh protects and watches it from that declaration
// (novox/hq ADR 0233).
//
// The self-check's D13 composes what every machine declares, asks each machine's backup holder what
// it measured of every item — size, newest write, newest good backup, the redundant storage it is on —
// and keeps both. From that, and from what every provider says it holds for its consumers, it raises,
// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking):
//
// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least
// shrinkFloor less; `data-missing`: its path is gone;
// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a
// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on
// empty databases while their real ones sat on another machine (issue 273);
// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot
// be compared;
// - `data-quiet`: an item said to be written all the time has not been, within its bound;
// - `backup-stale`: an item's newest good backup is older than its bound, or there is none;
// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read;
// `protection-missing`: an item said to be protected by redundancy is on storage that is not;
// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person.
//
// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no
// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by
// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with
// anything in it; this is the record of what it kept. An operator's path is never retired or deleted.
// The condition kinds of D13.
const (
kindDataShrank = "data-shrank"
kindEmptyReplacement = "empty-replacement"
kindDataHeldTwice = "data-held-twice"
kindDataQuiet = "data-quiet"
kindBackupStale = "backup-stale"
kindDataUnmeasured = "data-unmeasured"
// kindDataMissing is a watched item whose path is gone.
kindDataMissing = "data-missing"
// kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read.
kindArrayDegraded = "array-degraded"
// kindProtectionMissing is an item said to be protected by redundancy, on storage that is not.
kindProtectionMissing = "protection-missing"
)
// probeDataID is the self-check's id for this probe.
const probeDataID = "D13"
// The bounds the findings are read against.
var (
// shrinkWindow is how far back the largest size is looked for.
shrinkWindow = 7 * 24 * time.Hour
// shrinkFloor is the least loss that is worth saying: two empty databases differ by a few
// megabytes, and half of almost nothing is noise.
shrinkFloor int64 = 16 << 20
// dataAsk is how long one machine's holder, or one provider, is given to answer.
dataAsk = 8 * time.Second
)
// keyOfItem is one item's condition id: its machine, module and item.
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data).
type holderAnswer struct {
Module string `json:"module"`
Data []holderItem `json:"data"`
}
// holderItem is one item as the holder measured it.
type holderItem struct {
Item string `json:"item"`
Class string `json:"class"`
Path string `json:"path"`
SizeBytes *int64 `json:"size_bytes"`
LastWrite *time.Time `json:"last_write"`
MeasuredAt *time.Time `json:"measured_at"`
LastBackup *time.Time `json:"last_backup"`
Error string `json:"error,omitempty"`
// Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233).
Precision string `json:"precision,omitempty"`
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
}
// readHolder reads a holder's answer into measurements by module and item.
func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) {
var modules []holderAnswer
if err := json.Unmarshal(raw, &modules); err != nil {
return nil, fmt.Errorf("its answer is not readable: %w", err)
}
out := map[string]map[string]inventory.Measurement{}
for _, m := range modules {
for _, it := range m.Data {
if out[m.Module] == nil {
out[m.Module] = map[string]inventory.Measurement{}
}
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy,
Precision: it.Precision}
}
}
return out, nil
}
// declaredOn is every data item a machine's composition declares, and the module there that holds
// node-backup to measure them — empty for none.
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) {
var out []inventory.DeclaredData
held := ""
for _, m := range plan.Modules {
for _, c := range m.Claims {
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
held = m.Module
}
}
for _, it := range m.DataItems() {
out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class,
Owned: it.OwnedByModule(), Protection: it.Protection()})
}
}
return out, held
}
// consumerCopy is one provider's account of one consumer: where, how big, and whether still active.
type consumerCopy struct {
Node, Module, Consumer string
Size *int64
Retired bool
// Class is how precious the consumer's data is: the stricter of what the provider keeps for its
// consumers and what the consumer says it keeps there (`kept-by`).
Class string
}
// probeData is D13.
func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
if d.js == nil {
return nil, errors.New("no bus to ask the machines over")
}
open := d.open
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return nil, err
}
nodes, err := open.inventory.Nodes(ctx)
if err != nil {
return nil, err
}
heard := heardMachines(d)
now := time.Now()
delivered, err := readDeliveries(ctx, open.inventory)
if err != nil {
return nil, err
}
type machine struct {
name string
declared []inventory.DeclaredData
held bool
measured map[string]map[string]inventory.Measurement
askErr error
}
var machines []*machine
for _, n := range nodes {
plan, _, err := planFor(ctx, open, n.Name)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
// A machine that cannot be worked out declares nothing this run — which is not the same as
// declaring nothing: retiring its data on that would be acting on an unreadable result.
continue
}
declared, holder := declaredOn(plan)
// **A holder is asked only once its machine has been sent it and had time to report** (novox/hq
// issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it
// measured" about it was a warning for a push nobody had made yet.
held := holder != "" && delivered[n.Name].settled(holder, now)
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
}
// Every holder asked at once, as D8 asks every ban list.
var wg sync.WaitGroup
for _, m := range machines {
if !m.held || !heard[m.name] {
continue
}
wg.Add(1)
go func(m *machine) {
defer wg.Done()
asking, cancel := context.WithTimeout(ctx, dataAsk)
defer cancel()
raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name)
if err == nil {
m.measured, err = readHolder(raw)
}
m.askErr = err
}(m)
}
wg.Wait()
var out []conditions.Observation
for _, m := range machines {
if m.askErr != nil {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning, Confirm: true,
Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
"nothing about that data is known", m.name),
Said: firstLine(m.askErr.Error())})
}
why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now)
if err != nil {
return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err)
}
for _, r := range change.Retired {
log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+
"delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why)
}
for _, r := range change.Reenabled {
log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine)
}
}
records, err := open.inventory.Data(ctx)
if err != nil {
return nil, err
}
peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow))
if err != nil {
return nil, err
}
bindings, err := open.inventory.Bindings(ctx)
if err != nil {
return nil, err
}
upgraded, keptBy := keptByClasses(bindings, shelf)
copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy)
if err != nil {
return nil, err
}
out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...)
return out, nil
}
func orUnknownPath(p string) string {
if p == "" {
return "a path its backup holder never named"
}
return p
}
// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at
// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding.
func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) {
instances, err := providerInstances(ctx, inv)
if err != nil {
return nil, err
}
var asked []providerInstance
for _, p := range instances {
m := shelf[p.Module]
keeps := false
for provision := range m.Grants {
keeps = keeps || m.KeepsConsumerData(provision)
}
if keeps {
asked = append(asked, p)
}
}
states := make([]*link.RetirementState, len(asked))
var wg sync.WaitGroup
for i, p := range asked {
wg.Add(1)
go func(i int, p providerInstance) {
defer wg.Done()
asking, cancel := context.WithTimeout(ctx, dataAsk)
defer cancel()
if s, err := askRetirement(asking, conn, p); err == nil {
states[i] = &s
}
}(i, p)
}
wg.Wait()
var out []consumerCopy
for i, p := range asked {
s := states[i]
if s == nil {
continue
}
class := consumersClass(shelf[p.Module])
for _, c := range s.Held {
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c,
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])}
if size, ok := s.HeldSizes[c]; ok && size >= 0 {
size := size
cp.Size = &size
}
out = append(out, cp)
}
for _, r := range s.Retired {
if r.Kind != "" && r.Kind != "consumer" {
continue
}
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true,
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])}
if r.SizeBytes != nil && *r.SizeBytes >= 0 {
cp.Size = r.SizeBytes
}
out = append(out, cp)
}
}
return out, nil
}
// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning
// for anything else watched (the operator's ranking, ADR 0233).
func severityOf(class string) conditions.Severity {
if class == catalogue.ClassIrreplaceable {
return conditions.Urgent
}
return conditions.Warning
}
// consumersClass is the most precious class a provider keeps any of its consumers' data as.
func consumersClass(m catalogue.Manifest) string {
class := catalogue.ClassNone
for provision := range m.Grants {
if c, ok := m.ConsumerDataOf(provision); ok {
class = catalogue.StricterClass(class, c.Class)
} else if m.KeepsConsumerData(provision) {
class = catalogue.StricterClass(class, catalogue.ClassValuable)
}
}
return class
}
// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read
// through where each is bound: by provider module and consumer identity, the class of that consumer's
// data there; and by provider item key (machine/module/item), the class the item holding it is held to.
func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) {
upgraded, keptBy := map[string]string{}, map[string]string{}
for _, b := range bindings {
m, ok := shelf[b.Consumer]
if !ok {
continue
}
k, said := m.KeptByOf(b.Provision)
if !said {
continue
}
identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module))
key := b.Provider.Module + "/" + identity
keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class)
if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" {
if _, own := shelf[b.Provider.Module].DataItem(pc.In); own {
item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In
upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class)
}
}
}
return upgraded, keptBy
}
// dataFindings is every condition the data on record raises now. A function of what is known, so the
// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer
// of its module keeps data in it more precious than its own class says (`kept-by`), by its key.
func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest,
copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation {
var out []conditions.Observation
byItem := map[string][]inventory.DataRecord{}
arrays := map[string][]inventory.DataRecord{}
type shrunk struct {
machine, dataset, class string
size, peak int64
items []string
}
shrunkDatasets := map[string]shrunk{}
for _, r := range records {
if r.DeletedAt != nil {
continue
}
class := catalogue.StricterClass(r.Class, upgraded[r.Key()])
r.Class = class
byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r)
item, declared := shelf[r.Module].DataItem(r.Item)
id := keyOfItem(r.Machine, r.Module, r.Item)
if r.Retired() {
if now.Sub(*r.RetiredAt) > cleanupAfter {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module),
Said: "kept at " + orUnknownPath(r.Path)})
}
continue
}
if !catalogue.Watched(class) {
continue
}
severity := severityOf(class)
if r.Redundancy != nil {
where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
arrays[where] = append(arrays[where], r)
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
"and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
r.Item, r.Module, r.Machine, class),
Said: orUnknownPath(r.Path) + " is on no redundant storage the backup holder can read"})
}
if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.Item,
r.Module, r.Machine, class),
Said: orUnknownPath(r.Path) + " does not exist: " + firstLine(r.MeasureError)})
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
// Several items on one dataset share its size: one condition for the dataset, as loud as the
// most precious item on it.
k := r.Machine + "/" + inventory.Dataset(r.Precision)
ds := shrunkDatasets[k]
ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak
ds.class = catalogue.StricterClass(ds.class, class)
ds.items = append(ds.items, r.Module+"/"+r.Item)
shrunkDatasets[k] = ds
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
"good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak),
int(shrinkWindow.Hours()/24))})
}
if !declared {
continue
}
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet,
Kind: kindDataQuiet, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
"whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339),
within)})
}
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
// plan, said only where the machine was measured — where a holder is there to take it.
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
within := item.BackupWithin()
switch {
case r.LastBackup == nil && now.Sub(r.FirstSeen) > within:
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))})
case r.LastBackup != nil && now.Sub(*r.LastBackup) > within:
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
"of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)})
}
}
}
for _, k := range keysSorted(shrunkDatasets) {
ds := shrunkDatasets[k]
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than half of "+
"what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak),
int(shrinkWindow.Hours()/24)),
Said: "the dataset " + ds.dataset})
}
// The redundant storage watched data is on: one condition per array, as loud as the most precious
// item on it — the array, not each item, is what degrades.
for _, where := range keysSorted(arrays) {
rs := arrays[where]
red := rs[0].Redundancy
if red.Healthy != nil && *red.Healthy {
continue
}
class, machine := catalogue.ClassValuable, rs[0].Machine
var names []string
for _, r := range rs {
class = catalogue.StricterClass(class, r.Class)
names = append(names, r.Module+"/"+r.Item)
}
state := "could not be read"
if red.Healthy != nil {
state = "is NOT healthy"
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, strings.Join(names, ", "),
state),
Said: fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said))})
}
// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
// an empty replacement. Only against a retired copy — a module running on two machines on purpose
// keeps two different sets of data.
for _, key := range keysSorted(byItem) {
rs := byItem[key]
for _, a := range rs {
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) {
continue
}
for _, o := range rs {
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) ||
!replacedByLess(*a.Size, *o.Size) {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
"an empty replacement of its data. Move the data, or assign it back where its data is",
a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)})
break
}
}
}
out = append(out, consumerFindings(copies)...)
return out
}
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than
// half of it, and at least shrinkFloor less.
func replacedByLess(inUse, kept int64) bool {
return inUse*2 < kept && kept-inUse >= shrinkFloor
}
// consumerFindings is the same question of consumers' data at providers: one consumer, the same
// provider module on two machines.
func consumerFindings(copies []consumerCopy) []conditions.Observation {
by := map[string][]consumerCopy{}
for _, c := range copies {
k := c.Module + "/" + c.Consumer
by[k] = append(by[k], c)
}
var out []conditions.Observation
for _, k := range keysSorted(by) {
cs := by[k]
if len(cs) < 2 {
continue
}
found := false
for _, a := range cs {
if a.Retired || a.Size == nil {
continue
}
for _, o := range cs {
if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
continue
}
state := "active"
if o.Retired {
state = "retired"
}
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)),
Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+
"consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+
"the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node,
sizeWords(o.Size), o.Node)})
found = true
break
}
if found {
break
}
}
if found {
continue
}
var active []consumerCopy
for _, c := range cs {
if !c.Retired {
active = append(active, c)
}
}
if len(active) >= 2 {
var where []string
var also []string
for _, c := range active {
where = append(where, c.Node+" ("+sizeWords(c.Size)+")")
also = append(also, c.Node)
}
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice,
Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses",
active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))})
}
}
return out
}
func keysSorted[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// ---- the `data` verb ---------------------------------------------------------------------------
const dataUsage = "data [--json] [--machine <name>] [--retired]"
// dataRow is one item as `data` lists it.
type dataRow struct {
Machine string `json:"machine"`
Module string `json:"module"`
Item string `json:"item"`
Class string `json:"class"`
Path string `json:"path,omitempty"`
Protection string `json:"protection,omitempty"`
// Array is the redundant storage it is on and its state, where its holder could tell.
Array string `json:"array,omitempty"`
Unmeasured string `json:"unmeasured,omitempty"`
// Precision says what the size is: exact, a dataset's whole size, partial, or none.
Precision string `json:"precision,omitempty"`
SizeBytes *int64 `json:"size-bytes,omitempty"`
LastWrite string `json:"last-write,omitempty"`
MeasuredAt string `json:"measured-at,omitempty"`
LastBackup string `json:"last-backup,omitempty"`
BackupDue string `json:"backup-within,omitempty"`
Retired string `json:"retired,omitempty"`
RetiredWhy string `json:"retired-why,omitempty"`
Deleted string `json:"deleted,omitempty"`
}
// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last
// found it.
func dataCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("data", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
only := set.String("machine", "", "one machine")
retiredOnly := set.Bool("retired", false, "only what is retired")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(dataUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
records, err := open.inventory.Data(ctx)
if err != nil {
return err
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return err
}
rows := dataRows(records, shelf, *only, *retiredOnly)
if *asJSON {
return printJSON(map[string]any{"data": rows})
}
if len(rows) == 0 {
fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run")
return nil
}
for _, r := range rows {
state := ""
switch {
case r.Deleted != "":
state = " DELETED " + r.Deleted
case r.Retired != "":
state = " RETIRED " + r.Retired + " — " + r.RetiredWhy
}
fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class,
sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state)
if r.Array != "" {
fmt.Printf(" on %s\n", r.Array)
}
if r.Precision != "" && r.Precision != "exact" {
fmt.Printf(" size: %s\n", r.Precision)
}
if r.Unmeasured != "" {
fmt.Printf(" not measured: %s\n", r.Unmeasured)
}
if r.Class == catalogue.ClassCache {
continue
}
fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt),
orNever(r.LastBackup), within(r.BackupDue))
}
return nil
}
func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow {
rows := []dataRow{}
stamp := func(t *time.Time) string {
if t == nil {
return ""
}
return t.UTC().Format(time.RFC3339)
}
for _, r := range records {
if only != "" && r.Machine != only {
continue
}
if retiredOnly && !r.Retired() {
continue
}
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
Deleted: stamp(r.DeletedAt)}
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
row.BackupDue = it.BackupWithin().String()
}
if red := r.Redundancy; red != nil {
state := "state unread"
if red.Healthy != nil && *red.Healthy {
state = "healthy"
} else if red.Healthy != nil {
state = "NOT HEALTHY"
}
row.Array = red.Kind + " " + red.Where + ", " + state
}
rows = append(rows, row)
}
return rows
}
func orNothingWord(s string) string {
if s == "" || s == "none" {
return "nothing"
}
return s
}
func orNever(s string) string {
if s == "" {
return "never"
}
return s
}
func within(s string) string {
if s == "" {
return " (not backed up)"
}
return " (bound " + s + ")"
}
// ---- cleanup of retired own data ---------------------------------------------------------------
// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first
// takes a last restore point of it, tagged as retired, and only then removes it — in the background,
// because a large item outlasts any call — and the second says how that went. Module tools, not seat
// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete.
const (
ToolDeleteRetired = "backup_delete_retired"
ToolDeletedOutcome = "backup_deleted"
)
// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person.
var deletionWait = 8 * time.Minute
// deletionPoll is how often it asks.
var deletionPoll = 5 * time.Second
// deletion is a holder's account of one deletion.
type deletion struct {
Started bool `json:"started"`
Running bool `json:"running"`
Done bool `json:"done"`
OK bool `json:"ok"`
Snapshot string `json:"snapshot"`
Error string `json:"error"`
}
// retiredData is every retired item on record, as `cleanup list` shows them.
func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow {
var out []retiredRow
for _, r := range records {
if !r.Retired() {
continue
}
out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind,
RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24),
SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class})
}
return out
}
// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer.
const retiredDataKind = "own-data"
// holderOn is the module holding node-backup on a machine.
func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) {
held, err := inv.Holdings(ctx)
if err != nil {
return "", err
}
for _, h := range held {
if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine {
return h.Module, nil
}
}
return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+
"(after a last restore point)", catalogue.BackupSeat, machine)
}
// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and
// never one not retired. The holder takes a last restore point of it first, so the deletion can be
// undone until a person forgets that restore point; the record says deleted only once the holder says
// it is.
func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error {
inv := open.inventory
if !r.Retired() {
return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done",
r.Item, r.Module, r.Machine)
}
if r.Path == "" {
return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted",
r.Item, r.Module, r.Machine)
}
if plan, _, err := planFor(ctx, open, r.Machine); err == nil {
for _, m := range plan.Modules {
if _, still := m.DataItem(r.Item); still && m.Module == r.Module {
return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done",
r.Module, r.Machine, r.Item)
}
}
}
holder, err := holderOn(ctx, inv, r.Machine)
if err != nil {
return err
}
f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item})
args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item,
"why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController}
ask := func(tool string) (deletion, error) {
var d deletion
answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second)
if err != nil {
return d, err
}
if answer.Error != "" {
return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error)
}
return d, unmarshalAnswer(answer, &d)
}
d, err := ask(ToolDeleteRetired)
if err != nil {
return err
}
for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(deletionPoll):
}
if d, err = ask(ToolDeletedOutcome); err != nil {
return err
}
}
switch {
case !d.Done:
fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+
"showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n",
holder, r.Machine, r.Path)
return nil
case !d.OK:
return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error)
}
if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil {
return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err)
}
fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n",
holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot))
return nil
}
// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its
// own directories on the machine:
// kept, and retired at the self-check's next run.
func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string {
records, err := inv.Data(ctx)
if err != nil {
return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()}
}
var out []string
for _, r := range records {
if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned {
continue
}
for _, m := range modules {
if r.Module == m {
out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+
"never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)",
r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class))
}
}
}
return out
}
cmd/mesh-controller/data_test.go
-465
View File
@@ -1,465 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/testbus"
)
func bytesOf(n int64) *int64 { return &n }
func when(t time.Time) *time.Time { return &t }
func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
t.Helper()
out := map[string]catalogue.Manifest{}
for _, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatal(err)
}
out[m.Module] = m
}
return out
}
const houseManifest = `{"module":"house","version":"1",
"data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]},
"resources":[{"id":"config","type":"directory","mode":"0700"}]}`
func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation {
out := map[string]conditions.Observation{}
for _, o := range linted(obs) {
out[o.Kind] = o
}
return out
}
// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound,
// by one changed rule, to the store on the control node, which made each an empty database — while
// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired
// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines
// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says
// its data there is irreplaceable (`kept-by`).
func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) {
var copies []consumerCopy
for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} {
copies = append(copies,
consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"},
consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"})
}
copies[1].Class = "irreplaceable" // the photo site's own database says so
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
if len(got) != 5 {
t.Fatalf("%d findings for five empty replacements: %+v", len(got), got)
}
for i, o := range got {
want := conditions.Warning
if strings.Contains(o.ID, "mesh_home_board") {
want = conditions.Urgent
}
_ = i
if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" ||
len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") {
t.Errorf("%+v", o)
}
}
// While the old copy is still active (the first ten minutes), it is the same finding.
copies[0].Retired = false
copies[1].Class = "valuable"
if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement {
t.Fatalf("with the old copy still active: %+v", got)
}
}
// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and
// a retired one at the old: nothing to say here; `cleanup` covers the old one.
func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) {
copies := []consumerCopy{
{Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true},
{Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)},
}
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
t.Fatalf("a deliberate move raised %+v", got)
}
// Two small databases differing by less than the floor are not a finding either.
copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20)
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
t.Fatalf("noise between two empty databases raised %+v", got)
}
}
// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning,
// since which one is empty cannot be told.
func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) {
copies := []consumerCopy{
{Node: "home", Module: "minio", Consumer: "mesh_home_photos"},
{Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"},
}
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning {
t.Fatalf("%+v", got)
}
}
// The same incident for a module's own data: a module unassigned from one machine and assigned on
// another starts over in an empty directory while its full one is kept, retired, where it was.
func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) {
now := time.Now()
retired := now.Add(-time.Hour)
records := []inventory.DataRecord{
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)},
{Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)},
}
got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
o, ok := got[kindEmptyReplacement]
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" {
t.Fatalf("%+v", got)
}
// The same of a valuable item is a warning.
records[0].Class, records[1].Class = "valuable", "valuable"
if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning {
t.Fatalf("a valuable empty replacement: %+v", o)
}
records[0].Class, records[1].Class = "irreplaceable", "irreplaceable"
// Two machines running a module on purpose, both active, keep two sets of data: nothing to say.
records[0].RetiredAt = nil
if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" {
t.Fatalf("two active copies were read as a replacement: %+v", got)
}
}
// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss,
// or a loss under the floor, is not a finding.
func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
now := time.Now()
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)}
shelf := shelfFor(t, houseManifest)
o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank]
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") {
t.Fatalf("%+v", o)
}
for _, peak := range []int64{500 << 20, 20 << 20} {
if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
t.Errorf("a peak of %d raised a shrink", peak)
}
}
small := r
small.Size = bytesOf(1 << 20)
if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
t.Error("a loss under the floor raised a shrink")
}
}
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
now := time.Now()
shelf := shelfFor(t, houseManifest)
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)),
LastBackup: when(now.Add(-72 * time.Hour))}
got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))
if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent {
t.Fatalf("irreplaceable: %+v", got)
}
valuable := r
valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup
if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning ||
got[kindBackupStale].Severity != conditions.Warning {
t.Fatalf("valuable: %+v", got)
}
never := r
never.LastBackup = nil
if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") {
t.Fatalf("never backed up: %+v", o)
}
fresh := never
fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now)
if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got)
}
}
// An item retired more than thirty days waits for a person; less, it is only listed.
func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) {
now := time.Now()
old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour)
records := []inventory.DataRecord{
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old},
{Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent},
}
got := linted(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") {
t.Fatalf("%+v", got)
}
if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind {
t.Fatalf("cleanup list: %+v", rows)
}
}
// The holder's answer reads into measurements, by module and item.
func TestTheHoldersAnswerIsRead(t *testing.T) {
raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3,
"data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps",
"size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`)
got, err := readHolder(raw)
if err != nil {
t.Fatal(err)
}
m := got["postgres"]["store"]
if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil {
t.Fatalf("%+v", m)
}
// An older holder, which says no data, reads as nothing measured rather than a failure.
if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 {
t.Fatalf("%v, %v", got, err)
}
}
// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it
// measured.
type fakeHolder struct {
mu sync.Mutex
modules []map[string]any
}
func (f *fakeHolder) set(modules ...map[string]any) {
f.mu.Lock()
defer f.mu.Unlock()
f.modules = modules
}
func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) {
t.Helper()
sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
f.mu.Lock()
defer f.mu.Unlock()
body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node})
_ = m.Respond(body)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
}
func measuredHouse(path string, size int64, at time.Time) map[string]any {
return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{
"item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size,
"last_write": at, "measured_at": at, "last_backup": at}}}
}
// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
// up on another machine with an empty directory while the full one waits retired, that is urgent.
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
house, err := catalogue.ParseManifest([]byte(houseManifest))
if err != nil {
t.Fatal(err)
}
register(t, open, house)
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
}
for _, node := range []string{"laptop", "anchor"} {
if _, err := assign(ctx, open, node, "keeper"); err != nil {
t.Fatal(err)
}
}
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
t.Fatal(err)
}
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
for _, node := range []string{"laptop", "anchor"} {
pushedAndApplied(t, open, node)
}
conn := onATestBus(t)
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
laptop, anchor := &fakeHolder{}, &fakeHolder{}
laptop.serve(t, conn, "laptop")
anchor.serve(t, conn, "anchor")
now := time.Now()
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
d := &doctor{open: open, js: js, watchdogs: heard}
var d13 probe
for _, p := range probeRegistry {
if p.ID == probeDataID {
d13 = p
}
}
run := func() []conditions.Observation {
t.Helper()
probing := context.WithValue(ctx, probeAsksKey{}, d13)
obs, err := probeData(probing, d)
if err != nil {
t.Fatal(err)
}
return obs
}
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
if obs := run(); len(obs) != 0 {
t.Fatalf("a measured, backed-up item raised %+v", obs)
}
r, err := inv.DataOf(ctx, "laptop", "house", "config")
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
}
said, err := unassign(ctx, open, "laptop", "house")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
}
laptop.set()
run()
r, err = inv.DataOf(ctx, "laptop", "house", "config")
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
}
records, _ := inv.Data(ctx)
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
t.Fatalf("cleanup list: %+v", rows)
}
// Assigned on the anchor, onto an empty directory.
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
t.Fatal(err)
}
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
obs := findingsByKind(run())
o, ok := obs[kindEmptyReplacement]
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
}
}
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
// whose path is gone is said.
func TestTheArrayUnderDataIsWatched(t *testing.T) {
now := time.Now()
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"},
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}`
shelf := shelfFor(t, media)
sick := false
on := func(item string, healthy *bool) inventory.DataRecord {
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false,
Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}}
}
got := linted(dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now))
if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
t.Fatalf("%+v", got)
}
well := true
if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("a healthy array raised %+v", got)
}
plain := on("films", nil)
plain.Redundancy = nil
if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent {
t.Fatalf("redundancy said and not found: %+v", o)
}
gone := on("films", &well)
gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0)
if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent {
t.Fatalf("a vanished library: %+v", o)
}
}
// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class:
// the photo site's objects make the object store's data an urgent matter.
func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"},
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`
photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`
shelf := shelfFor(t, objects, photos)
bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket",
Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}}
upgraded, keptBy := keptByClasses(bindings, shelf)
if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" {
t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy)
}
now := time.Now()
r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true,
Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))}
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent {
t.Fatalf("the photos' store without a backup: %+v", o)
}
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning {
t.Fatalf("a valuable store without a backup: %+v", o)
}
}
// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition
// naming every item on it, not one per item; and a partial walk's lower bound is never compared.
func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) {
now := time.Now()
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"},
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}`
shelf := shelfFor(t, media, houseManifest)
well := true
on := func(item string, size int64) inventory.DataRecord {
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable",
Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
Precision: "dataset tank/media: its whole size",
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}}
}
films, shows := on("films", 30<<40), on("shows", 30<<40)
peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40}
got := linted(dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now))
if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
t.Fatalf("%+v", got)
}
partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now),
LastBackup: when(now), Precision: "partial: measured partially"}
if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("a partial size was compared: %+v", got)
}
}
cmd/mesh-controller/delivery.go
-643
View File
@@ -1,643 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The controller's part in a delivery (novox/hq ADR 0239, to-be 47).
//
// **A delivery is mesh-delivery's; the walk is the controller's.** A delivery is one commit in one
// repository from its pull request's head to every machine, and a delivery group a few of them sharing a
// branch name; their states, order, holds and record belong to the module holding the `mesh-delivery`
// seat. The controller keeps what it already owned — the planner, the gate, registration, sending, the
// rollback — and the **walk**: one trunk commit's plan, tier by tier across the machines, one machine
// first and judged at the gate (ADR 0236). What changes here is when a walk starts.
//
// - A walk that moves the controller, the node-engine, the node tools, the bus or mesh-delivery itself is
// **on the controller's own path**: started by the merge, as every plan was. mesh-delivery cannot gate
// or deliver itself, and nothing the core needs to be repaired may wait for it.
// - Any other walk, **while the seat has a holder on record**, is opened by the merge and waits — nothing
// asked, nothing registered, nothing sent — until mesh-delivery says `deliver`, or a person says
// `plans go`. Nothing of it is registered before then, so no other send can carry it to a machine
// before its turn (ADR 0236 §4a carries everything registered).
// - With no holder on record, every walk starts at the merge, exactly as before this existed.
//
// The verbs mesh-delivery asks with are here: `delivery plan` (the planner's one answer for a diffset),
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
// go`, `delivery stop` and `delivery walks`.
// onTheControllersPath are the modules whose walk never waits for the delivery's owner, and what each is.
var onTheControllersPath = map[string]string{
"mesh-controller": "the controller",
"mesh-host": "the node-engine",
"node-tools": "the node tools",
"nats": "the bus",
catalogue.DeliverySeat: "the delivery's owner",
}
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
// holder on record. Read from the catalogue the merge handler already holds; never from whether the
// holder answers, so a walk does not start by itself because mesh-delivery is down — that wait is said.
func deliverySeatHeld(entries []inventory.Entry) bool {
for _, e := range entries {
if len(e.On) > 0 && e.Manifest.ClaimsSeat(catalogue.DeliverySeat) {
return true
}
}
return false
}
// awaitsFor is what a new walk waits for: nil when it starts at once — no holder on record, or a module
// on the controller's own path among those it moves.
func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDelivery {
if !deliverySeatHeld(entries) {
return nil
}
for _, m := range modules {
if _, own := onTheControllersPath[m]; own {
return nil
}
}
return &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}
}
// waitingNote is what a walk waiting for its word says, wherever it is read.
func waitingNote(p inventory.Plan) string {
return fmt.Sprintf("published; its walk waits for %s's word — `plans go %s --why …` starts it by hand",
p.Delivery.Awaits, p.ID)
}
// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its
// first tier. Refused for a walk that does not wait.
func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return inventory.Plan{}, err
}
defer release()
p, err := inv.PlanByID(ctx, id)
if err != nil {
return inventory.Plan{}, err
}
switch {
case !p.Open():
return p, fmt.Errorf("%s is %s: there is no walk to start", p.ID, p.State)
case p.Delivery == nil || p.Delivery.Awaits == "":
return p, fmt.Errorf("%s waits for nobody: it started at its merge", p.ID)
case p.Delivery.Go != nil:
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
p.Delivery.Go.Local().Format("15:04:05"))
}
now := time.Now().UTC()
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
p.Note = "let go by " + by + "; its first tier is asked next"
if err := inv.SavePlan(ctx, &p); err != nil {
return p, err
}
return p, nil
}
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
// builds and registers; nothing further is asked or sent.
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return inventory.Plan{}, err
}
defer release()
p, err := inv.PlanByID(ctx, id)
if err != nil {
return inventory.Plan{}, err
}
if !p.Open() {
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
}
if p.Delivery == nil {
p.Delivery = &inventory.PlanDelivery{}
}
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
if err := inv.SavePlan(ctx, &p); err != nil {
return p, err
}
return p, nil
}
// orderMember is one member of a group, as mesh-delivery says it.
type orderMember struct {
ID string `json:"id"`
Repository string `json:"repository"`
Base string `json:"base,omitempty"`
Head string `json:"head,omitempty"`
Number int `json:"number,omitempty"`
Paths []string `json:"paths,omitempty"`
PathsTruncated bool `json:"paths_truncated,omitempty"`
Removed []string `json:"removed,omitempty"`
ModuleDirs []string `json:"module_dirs,omitempty"`
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
CloneURL string `json:"clone_url,omitempty"`
// After are the repositories its pull request says it goes after (`after: <repository>`).
After []string `json:"after,omitempty"`
}
// pull is the member as the forge announced it.
func (m orderMember) pull() link.PullUpdated {
owner, repo, _ := strings.Cut(m.Repository, "/")
base := m.Base
if base == "" {
base = "main"
}
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head,
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
}
// orderPair is one "before" among a group's members, and why.
type orderPair struct {
Before string `json:"before"`
After string `json:"after"`
Why string `json:"why"`
}
// orderReach is what a member moves, as the planner says.
type orderReach struct {
Moved []string `json:"moved,omitempty"`
Dependents []string `json:"dependents,omitempty"`
New []string `json:"new,omitempty"`
Manifests []string `json:"manifests,omitempty"`
}
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one.
type groupOrder struct {
Order []string `json:"order"`
Pairs []orderPair `json:"pairs,omitempty"`
Cycle []string `json:"cycle,omitempty"`
Reach map[string]orderReach `json:"reach,omitempty"`
}
// The reasons a pair is ordered, in the words the delivery plan shows.
const (
orderDeclared = "declared"
orderBuiltBy = "built by"
orderVersionSkew = "version skew"
orderEngineFirst = "engine before controller"
)
// orderOf is a group's order (novox/hq ADR 0239 decision 4), pure. A member goes before another when:
//
// - its pull request is named in the other's `after:` lines (declared);
// - it moves a module the other's moved modules are built by, stand on, package or declare (built by);
// - it moves the controller and the other changes any module's manifest (version skew: the newer
// controller parses what the newer manifest says) — the node-engine's excepted, which goes first;
// - it moves the node-engine and the other moves the controller (the witness reads nothing the controller
// does not yet grant, and a controller sends nothing an older engine would refuse — ADR 0236's rollout
// order).
//
// Otherwise, by repository then id, so every reading gives one order. A pair both ways is a cycle: the
// members in it are named, and none of them is ordered.
func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge) groupOrder {
out := groupOrder{Reach: reach}
byID := map[string]orderMember{}
for _, m := range members {
byID[m.ID] = m
}
add := func(before, after, why string) {
if before == after {
return
}
for _, p := range out.Pairs {
if p.Before == before && p.After == after {
return
}
}
out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why})
}
named := func(said, repository string) bool {
said = strings.TrimSuffix(strings.TrimSpace(said), ".git")
if strings.EqualFold(said, repository) {
return true
}
_, repo, _ := strings.Cut(repository, "/")
return strings.EqualFold(said, repo)
}
for _, a := range members {
for _, b := range members {
if a.ID == b.ID {
continue
}
ra, rb := reach[a.ID], reach[b.ID]
for _, said := range b.After {
if named(said, a.Repository) {
add(a.ID, b.ID, orderDeclared)
}
}
for _, e := range edges {
if slices.Contains(ra.Moved, e.To) && slices.Contains(rb.Moved, e.From) && e.From != e.To {
add(a.ID, b.ID, orderBuiltBy)
break
}
}
if slices.Contains(ra.Moved, "mesh-controller") && len(rb.Manifests) > 0 &&
!slices.Contains(rb.Moved, "mesh-controller") && !slices.Contains(rb.Moved, "mesh-host") {
add(a.ID, b.ID, orderVersionSkew)
}
if slices.Contains(ra.Moved, "mesh-host") && slices.Contains(rb.Moved, "mesh-controller") &&
!slices.Contains(ra.Moved, "mesh-controller") {
add(a.ID, b.ID, orderEngineFirst)
}
}
}
sort.Slice(out.Pairs, func(i, j int) bool {
if out.Pairs[i].Before != out.Pairs[j].Before {
return out.Pairs[i].Before < out.Pairs[j].Before
}
return out.Pairs[i].After < out.Pairs[j].After
})
// Kahn's walk, the next always the first by repository and id among those with nothing before them.
waiting := map[string]int{}
for _, m := range members {
waiting[m.ID] = 0
}
for _, p := range out.Pairs {
waiting[p.After]++
}
done := map[string]bool{}
for len(done) < len(members) {
var ready []orderMember
for _, m := range members {
if !done[m.ID] && waiting[m.ID] == 0 {
ready = append(ready, m)
}
}
if len(ready) == 0 {
for _, m := range members {
if !done[m.ID] {
out.Cycle = append(out.Cycle, m.ID)
}
}
sort.Strings(out.Cycle)
return out
}
sort.Slice(ready, func(i, j int) bool {
if ready[i].Repository != ready[j].Repository {
return ready[i].Repository < ready[j].Repository
}
return ready[i].ID < ready[j].ID
})
next := ready[0]
done[next.ID] = true
out.Order = append(out.Order, next.ID)
for _, p := range out.Pairs {
if p.Before == next.ID {
waiting[p.After]--
}
}
}
return out
}
// reachOfMembers is each member's reach, as the planner says it.
func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
edges []inventory.Edge) map[string]orderReach {
out := map[string]orderReach{}
for _, m := range members {
s := pullScope(m.pull(), entries, read, edges)
out[m.ID] = orderReach{Moved: s.Modules, Dependents: s.Dependents, New: s.New, Manifests: s.Manifests}
}
return out
}
// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner:
//
// delivery plan --repository owner/repo --head <commit> [--base main] --paths a,b [--module-dirs …] [--removed …]
// delivery order --members <json>
// delivery check --group <id> --members <json>
// delivery go <plan> [--by <who>] [--why <text>]
// delivery stop <plan> --why <text> [--by <who>]
// delivery walks [-n 50] [--plan <id>]
func deliveryCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("delivery plan|order|check|go|stop|walks")
}
sub, rest := args[0], args[1:]
set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError)
repository := set.String("repository", "", "owner/repository")
base := set.String("base", "main", "the branch it merges into")
head := set.String("head", "", "the commit at hand")
paths := set.String("paths", "", "the files it changes, comma-separated")
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
removed := set.String("removed", "", "the files it deletes, comma-separated")
membersJSON := set.String("members", "", "a group's members, as JSON")
group := set.String("group", "", "a delivery group's id")
by := set.String("by", catalogue.DeliverySeat, "who says it")
why := set.String("why", "", "why")
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
planID := set.String("plan", "", "one walk")
positionals, err := parseAround(set, rest)
if err != nil {
return err
}
var members []orderMember
if *membersJSON != "" {
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
return fmt.Errorf("the members are not readable: %w", err)
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
answer := func(v any) error {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
return enc.Encode(v)
}
switch sub {
case "plan":
if *repository == "" || *head == "" && *paths == "" {
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
}
m := orderMember{ID: *repository + "@" + *head, Repository: *repository, Base: *base, Head: *head,
Paths: splitList(*paths), Removed: splitList(*removed)}
if d := moduleDirsOf(*moduleDirs); d != nil {
m.ModuleDirs, m.ModuleDirsSaid = *d, true
}
entries, read, edges, err := theGraph(ctx, inv)
if err != nil {
return err
}
s := pullScope(m.pull(), entries, read, edges)
plan := changePlanOf(*repository, *base, *head, s.Reach, entries, func(module string) (inventory.Upgrade, bool) {
u, err := inv.UpgradeOf(ctx, module)
return u, err == nil
})
return answer(map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()})
case "order":
if len(members) == 0 {
return errors.New("delivery order --members <json>: a group has members")
}
entries, read, edges, err := theGraph(ctx, inv)
if err != nil {
return err
}
return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges))
case "check":
if *group == "" && len(members) == 1 {
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
// own path for a pull request, run because the delivery's owner asked.
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
return err
}
return answer(map[string]any{"rechecked": members[0].ID})
}
if *group == "" || len(members) < 2 {
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
}
id, err := askGroupCheck(ctx, open, *group, members)
if err != nil {
return err
}
return answer(map[string]any{"asked": id, "group": *group})
case "go":
if len(positionals) != 1 {
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
}
var p inventory.Plan
if err := sayingOnTheBus(ctx, func() (err error) {
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
return err
}); err != nil {
return err
}
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
short(p.Commit), *by)
return nil
case "stop":
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
}
var p inventory.Plan
if err := sayingOnTheBus(ctx, func() (err error) {
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
return err
}); err != nil {
return err
}
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
return nil
case "walks":
var walks []inventory.Plan
if *planID != "" {
p, err := inv.PlanByID(ctx, *planID)
if err != nil {
return err
}
walks = []inventory.Plan{p}
} else {
if walks, err = inv.OpenPlans(ctx); err != nil {
return err
}
recent, err := inv.RecentPlans(ctx, *limit)
if err != nil {
return err
}
for _, p := range recent {
if !slices.ContainsFunc(walks, func(w inventory.Plan) bool { return w.ID == p.ID }) {
walks = append(walks, p)
}
}
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
"own-path": sortedKeysOf(ownPathWords())})
}
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
}
// ownPathWords is the controller's own path as words, for an answer.
func ownPathWords() map[string]string { return onTheControllersPath }
// theGraph is what the planner reads.
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
[]inventory.Edge, error) {
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, nil, nil, err
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return nil, nil, nil, err
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return nil, nil, nil, err
}
return entries, read, edges, nil
}
// groupPrimary is the head a group's composed check is run from: the one moving the controller, which then
// judges the group by itself; else the one moving the node-engine, whose validator judges; else the first.
func groupPrimary(members []orderMember, reach map[string]orderReach) int {
for _, want := range []string{"mesh-controller", "mesh-host"} {
for i, m := range members {
if slices.Contains(reach[m.ID].Moved, want) {
return i
}
}
}
return 0
}
// askGroupCheck asks the build seat for a group's composed check: every head laid over the mesh in turn,
// judged as one future state (novox/hq ADR 0239). The verdict comes back as `checked` with the group's id,
// for mesh-delivery; the forge's holder sets no status from it.
func askGroupCheck(ctx context.Context, open *stores, group string, members []orderMember) (string, error) {
entries, read, edges, err := theGraph(ctx, open.inventory)
if err != nil {
return "", err
}
reach := reachOfMembers(members, entries, read, edges)
primary := groupPrimary(members, reach)
p := members[primary].pull()
scope := pullScope(p, entries, read, edges)
// The gate runs over what any member moves; a judge from the primary alone.
for _, m := range members {
r := reach[m.ID]
scope.Modules = appendNew(scope.Modules, r.Moved...)
scope.Dependents = appendNew(scope.Dependents, r.Dependents...)
}
request, err := checkRequestFor(ctx, open, p, scope, entries)
if err != nil {
return "", err
}
request.Check.Group = group
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
if err != nil {
return "", err
}
for i, m := range members {
if i == primary {
continue
}
mp := m.pull()
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
source := inventory.Source{Seat: gitSeat, Repository: mp.Owner + "/" + mp.Repo}
for _, e := range entries {
if !e.Provided && sameRepository(e.Source.Repository, link.SourceMoved{Owner: mp.Owner, Repo: mp.Repo}) {
source = e.Source
break
}
}
url := source.Repository
if source.Seat != "" {
url, err = clonedFromSeat(world, source.Seat, source.Repository)
}
if err != nil {
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
}
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths})
}
seat := buildSeatHeld(ctx)
ask, err := askOverOn(seat)
if err != nil {
return "", err
}
defer ask.Close()
if err := ask.Ask(ctx, request); err != nil {
return "", err
}
fmt.Fprintf(os.Stderr, "group %s: asked %s to check %d head(s) composed together, as %s\n", group, seat,
len(members), request.ID)
return request.ID, nil
}
// appendNew appends what is not there yet.
func appendNew(to []string, items ...string) []string {
for _, i := range items {
if !slices.Contains(to, i) {
to = append(to, i)
}
}
return to
}
// shelfOf is the catalogue's manifests by name.
func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
shelf := map[string]catalogue.Manifest{}
for _, e := range entries {
shelf[e.Manifest.Module] = e.Manifest
}
return shelf
}
// sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
// comparison. Records arriving out of order are told apart by the plan's revision.
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
go publishPlanMoved(ctx, bus, p)
}
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
body, err := json.Marshal(p)
if err != nil {
return
}
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer stop()
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, body); err != nil {
fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err)
}
}
// sayingOnTheBus runs a command that keeps walks or says verdicts with the bus to say them on: the serving
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
// still runs; what it did is found by comparison.
func sayingOnTheBus(ctx context.Context, f func() error) error {
if checkEvents != nil {
return f()
}
ran := false
err := onTheBus(func(conn *nats.Conn) error {
js, err := conn.JetStream()
if err != nil {
return err
}
bus := link.OverNATS{Conn: conn, JS: js}
checkEvents = bus
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
ran = true
return f()
})
if !ran {
fmt.Fprintf(os.Stderr, "the bus cannot be reached (%v): what this does is not said, and is found by comparison\n", err)
return f()
}
return err
}
cmd/mesh-controller/delivery_conditions.go
-182
View File
@@ -1,182 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table
// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the
// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery.<id>.stalled`; healer
// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only
// the next observation says whether it worked (ADR 0231).
// probeDeliveriesID is the probe that reads the delivery's owner.
const probeDeliveriesID = "D14"
// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope.
const kindDeliveryStalled = "stalled"
// deliveryOwnerWithin is how long the owner is given to answer.
var deliveryOwnerWithin = 10 * time.Second
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
type stalledLine struct {
ID string `json:"id"`
State string `json:"state"`
For string `json:"for"`
Bound string `json:"bound"`
H2 string `json:"h2"`
Says string `json:"says"`
}
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted;
// a seam a test replaces. The holder's own refusal is an error naming it.
var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
granted := false
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
granted = granted || v.Verb == verb
}
if !granted {
return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb)
}
if conn == nil {
return nil, errors.New("this controller is not on the bus")
}
answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin)
if err != nil {
return nil, err
}
if answer.Error != "" {
return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error)
}
return unwrapToolResult(answer.Result), nil
}
// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the
// protocol's content list holding it as text.
func unwrapToolResult(raw json.RawMessage) json.RawMessage {
var wrapped struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
}
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) {
return json.RawMessage(wrapped.Content[0].Text)
}
return raw
}
// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record,
// or none answers (D3 says that one).
func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) {
if !held {
return nil, nil
}
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{})
if errors.Is(err, link.ErrNothingServes) {
return nil, nil // the holder not running is D3's holder-silent, said once there
}
if err != nil {
return nil, err
}
var lines []stalledLine
if err := json.Unmarshal(raw, &lines); err != nil {
return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err)
}
return lines, nil
}
// stalledObservations are the conditions of what is stalled.
func stalledObservations(lines []stalledLine) []conditions.Observation {
out := make([]conditions.Observation, 0, len(lines))
for _, l := range lines {
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
Severity: conditions.Warning,
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
if l.operatorsOnly() {
o.Resolver = conditions.ResolverOperator
}
out = append(out, o)
}
return out
}
// probeDeliveries is D14: every delivery held past its state's bound is said.
func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
entries, err := d.open.inventory.Catalogued(ctx)
if err != nil {
return nil, err
}
var conn *nats.Conn
if d.js != nil {
conn = d.js.Conn()
}
lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries))
if err != nil {
return nil, err
}
return stalledObservations(lines), nil
}
// --- H2, for a delivery ---------------------------------------------------------------------------------
// connOf is the bus a healer asks over.
func (h *healing) connOf() *nats.Conn {
if h.js == nil {
return nil
}
return h.js.Conn()
}
// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table
// lets H2 take; never one whose state is the operator's.
func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
lines, err := deliveriesStalled(ctx, h.connOf(), true)
if err != nil {
return "", false, "", err
}
for _, l := range lines {
if l.ID != c.Subject.ID {
continue
}
if l.operatorsOnly() {
return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil
}
return c.Key, true, "", nil
}
return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil
}
// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the
// transition its table names. A refusal is no repair, said; the next observation says whether it worked.
func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key})
if err != nil {
if errors.Is(err, link.ErrNothingServes) {
return "", "", err
}
return "closed nothing", oneLine(err.Error()), nil
}
var said string
if json.Unmarshal(raw, &said) != nil {
said = string(raw)
}
return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil
}
cmd/mesh-controller/delivery_conditions_test.go
-171
View File
@@ -1,171 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's
// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`.
// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records
// what close was asked.
func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string {
t.Helper()
var closed []string
was := askDeliveryOwner
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
switch verb {
case "stalled":
raw, _ := json.Marshal(lines)
return raw, nil
case "close":
closed = append(closed, args["id"].(string))
if closeErr != nil {
return nil, closeErr
}
return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil
}
t.Fatalf("asked the owner %s", verb)
return nil, nil
}
t.Cleanup(func() { askDeliveryOwner = was })
return &closed
}
func holdTheDeliverySeat(t *testing.T, open *stores) {
t.Helper()
m := catalogue.Manifest{Module: "mesh-delivery", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
}
var twoStalled = []stalledLine{
{ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s",
H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"},
{ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s",
H2: "none: the state is the operator's", Says: "it waits for the operator"},
}
func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
ownerAnswers(t, twoStalled, nil)
d := &doctor{open: open}
got, err := probeDeliveries(ctx, d)
if err != nil || len(got) != 0 {
t.Fatalf("with no holder on record D14 said %+v %v", got, err)
}
holdTheDeliverySeat(t, open)
got, err = probeDeliveries(ctx, d)
if err != nil || len(got) != 2 {
t.Fatalf("D14 said %+v %v", got, err)
}
if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning ||
got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") {
t.Fatalf("the first is %+v", got[0])
}
if got[1].Resolver != conditions.ResolverOperator {
t.Fatalf("a held delivery is not the operator's: %+v", got[1])
}
// The holder not running is D3's to say, not D14's.
was := askDeliveryOwner
askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) {
return nil, link.ErrNothingServes
}
defer func() { askDeliveryOwner = was }()
if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 {
t.Fatalf("an owner that is down was said by D14: %+v %v", got, err)
}
}
// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a
// refusal is no repair.
func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, _ := healingOn(t, open)
closed := ownerAnswers(t, twoStalled, nil)
for _, o := range stalledObservations(twoStalled) {
o.Source = probeDeliveriesID
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
t.Fatal(err)
}
}
h.tick(ctx)
if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) {
t.Fatalf("close was asked for %v", *closed)
}
acts := told.said()
if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" {
t.Fatalf("said %+v", acts)
}
// A refusal: closed nothing, said so.
c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled")
ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering"))
act, said, err := repairDelivery(ctx, h, c)
if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") {
t.Fatalf("a refused close reads %q %q %v", act, said, err)
}
}
// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else
// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read.
func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
for _, verb := range []string{"stalled", "close"} {
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
}
}
if _, err := askDeliveryOwner(t.Context(), nil, "stop", nil); err == nil || !strings.Contains(err.Error(), "grant") {
t.Fatalf("a verb the grant does not name was asked: %v", err)
}
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if _, err := deliveriesStalled(t.Context(), conn, true); err != nil {
t.Fatalf("an owner not running is D3's, not an error: %v", err)
}
lines, _ := json.Marshal(twoStalled)
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}})
sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) {
body, _ := json.Marshal(link.Answer{Result: wrapped})
_ = m.Respond(body)
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
got, err := deliveriesStalled(ctx, conn, true)
if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" {
t.Fatalf("over the bus: %+v %v", got, err)
}
}
cmd/mesh-controller/delivery_order_test.go
-145
View File
@@ -1,145 +0,0 @@
package main
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
type recordingDelivery struct {
did []string
grantErr error
declareErr error
}
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
for _, s := range sending {
r.did = append(r.did, "grant "+s.node)
}
return r.grantErr
}
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
if r.declareErr != nil {
return "", r.declareErr
}
r.did = append(r.did, "declare "+s.node)
return "digest-" + s.node, nil
}
func ready(names ...string) []readyNode {
var out []readyNode
for _, n := range names {
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
}
return out
}
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
// carries the controller's own right to issue them, and goes first.
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
d := &recordingDelivery{}
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
if err != nil {
t.Fatal(err)
}
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
if !reflect.DeepEqual(d.did, want) {
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
}
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
t.Fatalf("the digests sent were not answered: %v", digests)
}
held := &recordingDelivery{}
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
t.Fatal(err)
}
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
if !reflect.DeepEqual(held.did, want) {
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
}
}
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
// grant that would let the controller issue memberships is never held behind them.
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
// A failure naming no machine (the buckets): everything but the bus's machine.
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
!strings.Contains(err.Error(), "anchor, laptop not sent") {
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
}
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
}
// A membership that failed for one machine: that machine alone.
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
t.Fatalf("one machine's refused membership was not said: %v", err)
}
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
}
// The announced upgrade that hit it is asked again.
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
t.Fatal("an announcement whose send stopped at its grants is not asked again")
}
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
t.Fatal("any failure is asked again, not only a grant's")
}
// Nothing to send is nothing granted either.
none := &recordingDelivery{grantErr: errors.New("never asked")}
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
}
}
// Whether the bus's machine goes first is read from the user list alone, by its digest.
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
list := "users: [a, b]"
if userListBehind(list, digestOf([]byte(list))) {
t.Fatal("the list it was sent reads as behind")
}
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
t.Fatal("a changed or never-sent list reads as current")
}
if userListBehind("", "") {
t.Fatal("a machine sent no list reads as behind")
}
}
// The machine holding the bus goes first: its declaration carries the user list the new grants are
// checked against. Among the machines it is moved to the front; not among them it is added only
// when it is behind.
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
for _, c := range []struct {
what string
names []string
holder string
behind bool
want []string
}{
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
} {
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
}
}
}
cmd/mesh-controller/delivery_test.go
-318
View File
@@ -1,318 +0,0 @@
package main
import (
"encoding/json"
"reflect"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every
// reading — the build agent before what it builds, the controller before a manifest that needs it, the
// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered.
func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
members := []orderMember{
{ID: "cat", Repository: "novox/mesh-catalog"},
{ID: "ctl", Repository: "novox/mesh-controller"},
{ID: "host", Repository: "novox/mesh-host"},
{ID: "agent", Repository: "novox/build-agent"},
{ID: "app", Repository: "novox/app", After: []string{"lab"}},
{ID: "lab", Repository: "novox/lab"},
}
reach := map[string]orderReach{
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
"agent": {Moved: []string{"build-agent"}},
"app": {Moved: []string{"app"}},
"lab": {Moved: []string{"lab"}},
}
edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
got := orderOf(members, reach, edges)
if len(got.Cycle) > 0 {
t.Fatalf("a cycle where there is none: %v", got.Cycle)
}
// By repository among those with nothing before them: the build agent, the lab, then what waited on both.
want := []string{"agent", "lab", "app", "host", "ctl", "cat"}
if !reflect.DeepEqual(got.Order, want) {
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
}
why := map[string]string{}
for _, p := range got.Pairs {
why[p.Before+">"+p.After] = p.Why
}
for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew,
"ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} {
if why[pair] != reason {
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
}
}
// The same members in another order read the same.
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) {
t.Fatalf("another reading ordered %v", again.Order)
}
// A declared order against an inferred one is a cycle: named, and nothing ordered.
members[1].After = []string{"mesh-catalog"}
got = orderOf(members, reach, edges)
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) {
t.Fatalf("the cycle is %v, ordered %v", got.Cycle, got.Order)
}
}
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
// record, and never one that moves a module on the controller's own path.
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
unassigned := holder
unassigned.On = nil
for _, c := range []struct {
entries []inventory.Entry
moved []string
waits bool
}{
{nil, []string{"gitea"}, false},
{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
{[]inventory.Entry{holder}, []string{"node-tools"}, false},
{[]inventory.Entry{holder}, []string{"nats"}, false},
{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
} {
d := awaitsFor(c.entries, c.moved)
if (d != nil) != c.waits {
t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
c.moved, d != nil, c.waits)
}
if d != nil && d.Awaits != catalogue.DeliverySeat {
t.Errorf("waits for %q", d.Awaits)
}
}
}
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
// hand when the owner is down, and the owner's stop ends one as stopped.
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := asksRecorded(t)
withConditionsInMemory(t)
for _, name := range []string{"app", "mesh-delivery"} {
m := catalogue.Manifest{Module: name, Version: "1"}
if name == "mesh-delivery" {
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
}
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
}
merge := func(commit string, paths ...string) inventory.Plan {
t.Helper()
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
recent, err := inv.RecentPlans(ctx, 1)
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
}
return recent[0]
}
// finish ends a walk as done, so the next merge has nothing of it to take over.
finish := func(id string) {
t.Helper()
w, err := inv.PlanByID(ctx, id)
if err != nil {
t.Fatal(err)
}
w.State = inventory.PlanDone
if err := inv.SavePlan(ctx, &w); err != nil {
t.Fatal(err)
}
}
// No holder on record: the merge starts its walk, as before.
p := merge("c1aaaaaaaa", "modules/app/index.ts")
if p.Waiting() || len(*asked) != 1 {
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
}
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either.
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
p = merge("c2bbbbbbbb", "modules/app/index.ts")
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
}
advanceHeld(ctx, open)
if len(*asked) != 1 {
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
}
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
t.Errorf("a waiting walk reads %q", line)
}
// The delivery's word starts it.
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
t.Fatal(err)
}
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
t.Fatal("a walk was let go twice")
}
advanceHeld(ctx, open)
if len(*asked) != 2 {
t.Fatalf("the word did not start the walk: %v", *asked)
}
got, err := inv.PlanByID(ctx, p.ID)
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
}
// The delivery's owner's own merge never waits for it — and takes over what the older walk had not
// built (app, folded in: ADR 0218), which goes with it on the controller's own path.
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
if p.Waiting() || len(*asked) != 4 {
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
}
// The owner down: a person starts a waiting walk, with why.
finish(p.ID)
p = merge("c4dddddddd", "modules/app/index.ts")
if !p.Waiting() {
t.Fatal("the walk did not wait")
}
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
t.Fatalf("a walk was started by hand without why: %v", err)
}
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
t.Fatal(err)
}
advanceHeld(ctx, open)
got, _ = inv.PlanByID(ctx, p.ID)
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 5 {
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
}
// The owner's stop: failed, said as stopped by it.
finish(p.ID)
p = merge("c5eeeeeeee", "modules/app/index.ts")
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
"mesh-delivery for jochen"}); err != nil {
t.Fatal(err)
}
got, _ = inv.PlanByID(ctx, p.ID)
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
!strings.Contains(got.Note, "the operator stopped it") {
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
}
// Unassigned: the mesh is back on the controller's own path.
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
asks := len(*asked)
if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
}
}
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want []string
}{
{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
{"delivery-check", map[string]any{"group": "feat/x", "members": "[]"},
[]string{"delivery", "check", "--group", "feat/x", "--members", "[]"}},
{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"},
[]string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}},
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
} {
got, err := argvFor(c.verb, c.args)
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
}
}
if _, err := argvFor("deliver", map[string]any{}); err == nil {
t.Error("deliver without a walk was composed")
}
if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
t.Error("a stop without why was composed")
}
}
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
url := testbus.URL(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
heard := make(chan *nats.Msg, 4)
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {}},
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
t.Fatal(err)
}
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
var said inventory.Plan
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
said.Delivery.Go == nil {
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
}
case <-time.After(5 * time.Second):
t.Fatal("a walk let go by a verb was not said")
}
if checkEvents != nil || inventory.PlanSaved != nil {
t.Fatal("the command's own bus was left in place")
}
}
cmd/mesh-controller/doctor.go
-600
View File
@@ -1,600 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
//
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
// design — every machine's declaration composes and validates, every resolver answers, every seat's
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
// condition it raises when the invariant does not hold. The serving controller runs the registry every
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
// machine: a controller that stops checking is itself said, through a channel that does not pass
// through it.
//
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
// The self-check's clocks.
var (
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
// starts the first run waits, so what the controller hears at its start has arrived.
doctorEvery = 5 * time.Minute
doctorFirstAfter = time.Minute
// probeWithin is each probe's bound.
probeWithin = 30 * time.Second
)
// The verdicts a probe can have.
const (
verdictPass = "pass"
verdictFail = "fail"
verdictFailedToRun = "failed-to-run"
verdictDeferred = "deferred"
)
// probe is one live invariant.
type probe struct {
ID string
Asserts string
From string
// Kind is the condition kind raised when the invariant does not hold.
Kind string
// Raises are the other kinds its findings carry, each its own (a consumer missing, one far behind):
// what a healer may be registered against (healers.go).
Raises []string
Phase int
// Deferred says why it is not run yet; empty for one that is.
Deferred string
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
// controller's grant names every one (a test over this registry): a probe whose question the bus
// refuses checks nothing (D8, 2026-10-06).
Asks []broker.SeatVerb
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
}
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
// probe added to a design is a row added here.
var probeRegistry = []probe{
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
run: probeDeclarations},
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
Kind: "archives-unheld", Phase: 1, run: probeArchives},
{ID: "D5", Asserts: "exactly one lease holder — the key names this controller at its epoch, and the record " +
"holds no other epoch open; no message from a stale epoch refused in the last interval",
From: "issue 204", Kind: "lease-split", Phase: 2, run: probeLease},
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Raises: []string{"consumer-lost", kindConsumerBehind},
Phase: 1, run: probeConsumers},
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding " +
"its cleanup", From: "ADR 0230", Kind: kindCleanupWaiting, Phase: 2, run: probeRetired},
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
{ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " +
"written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " +
"empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " +
"valuable is retired, not forgotten", From: "issue 273, ADR 0233",
Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale,
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
Phase: 2, run: probeData,
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
// The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by
// the controller, and H2 works it through the owner's `close`.
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
// judged by on its first machine, run against every machine between upgrades too.
{ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " +
"ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8",
Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth},
{ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " +
"build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth},
{ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus",
From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth},
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " +
"it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4,
run: probeBusHealth},
// The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its
// condition until a newer build passes; each rollback a witness stands by is said.
{ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " +
"a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack,
Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates},
// The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus.
{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
}
// probeVerdict is one probe's outcome in a run.
type probeVerdict struct {
ID string `json:"id"`
Verdict string `json:"verdict"`
Found []string `json:"found,omitempty"`
// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before:
// raised if the next run sees them too (confirm.go). Not a pass, and not yet a condition.
Unconfirmed []string `json:"unconfirmed,omitempty"`
Error string `json:"error,omitempty"`
Took string `json:"took,omitempty"`
}
// doctorCounts are a run's verdicts, counted.
type doctorCounts struct {
Passed int `json:"passed"`
Failed int `json:"failed"`
FailedToRun int `json:"failed-to-run"`
Deferred int `json:"deferred"`
}
// doctorRun is one run of the registry, and the body of its heartbeat.
type doctorRun struct {
Run string `json:"run"`
At time.Time `json:"at"`
Started time.Time `json:"started"`
Took string `json:"took"`
IntervalSeconds int `json:"interval-seconds"`
Counts doctorCounts `json:"counts"`
Probes []probeVerdict `json:"probes"`
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
Controller string `json:"controller"`
Why string `json:"why"`
// Unsaid is how many condition transitions this controller could not say, since it started.
Unsaid int `json:"unsaid,omitempty"`
}
// doctor is the registry and what its probes need.
type doctor struct {
open *stores
js *broker.JetStream
keeper *conditions.Keeper
teller conditions.Teller
watchdogs *watchdogs
host string
// confirm holds back what one run alone saw of a finding a single look can be wrong about.
confirm confirming
running sync.Mutex
mu sync.Mutex
last *doctorRun
ended time.Time
}
// lastRunEnded is when the last run ended; zero before the first.
func (d *doctor) lastRunEnded() time.Time {
d.mu.Lock()
defer d.mu.Unlock()
return d.ended
}
// lastRun is the last run's verdict; nil before the first.
func (d *doctor) lastRun() *doctorRun {
d.mu.Lock()
defer d.mu.Unlock()
return d.last
}
// keep runs the registry on its schedule until ctx ends.
func (d *doctor) keep(ctx context.Context) {
select {
case <-ctx.Done():
return
case <-time.After(doctorFirstAfter):
}
tick := time.NewTicker(doctorEvery)
defer tick.Stop()
for {
// Only the controller acting checks the mesh on a schedule: one standing by would say a
// heartbeat for a self-check that is not the mesh's.
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
d.runOnce(ctx, "the schedule")
}
select {
case <-ctx.Done():
return
case <-tick.C:
}
}
}
var doctorRuns struct {
sync.Mutex
n uint64
}
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
// at a time: a person's `doctor run` during a scheduled one waits for it.
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
d.running.Lock()
defer d.running.Unlock()
doctorRuns.Lock()
doctorRuns.n++
n := doctorRuns.n
doctorRuns.Unlock()
started := time.Now()
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
type result struct {
obs []conditions.Observation
err error
took time.Duration
}
results := make([]result, len(probeRegistry))
var wg sync.WaitGroup
for i, p := range probeRegistry {
if p.run == nil {
continue
}
wg.Add(1)
go func(i int, p probe) {
defer wg.Done()
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
defer cancel()
began := time.Now()
done := make(chan result, 1)
go func() {
defer func() {
if r := recover(); r != nil {
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
}
}()
obs, err := p.run(probing, d)
done <- result{obs: obs, err: err}
}()
select {
case r := <-done:
r.took = time.Since(began)
results[i] = r
case <-probing.Done():
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
}
}(i, p)
}
wg.Wait()
var blind []conditions.Observation
for i, p := range probeRegistry {
v := probeVerdict{ID: p.ID}
r := results[i]
switch {
case p.run == nil:
v.Verdict = verdictDeferred
run.Counts.Deferred++
case r.err != nil:
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
run.Counts.FailedToRun++
// A probe that could not run once — a question timed out on a loaded machine — is said in
// the verdict at once, and raised as a condition when the next run cannot run it either.
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
Token: "failed", Severity: conditions.Warning, Confirm: true,
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
Said: firstLine(r.err.Error())})
default:
raise, held := d.confirm.pass(ctx, d.keeper, p.ID, kindedAs(r.obs, p.Kind))
if err := d.keeper.Reconcile(ctx, p.ID, raise); err != nil {
v.Error = "what it found could not be kept: " + err.Error()
}
for _, o := range held {
v.Unconfirmed = append(v.Unconfirmed, o.Summary)
}
if len(raise) == 0 {
v.Verdict = verdictPass
run.Counts.Passed++
} else {
v.Verdict = verdictFail
run.Counts.Failed++
for _, o := range raise {
v.Found = append(v.Found, o.Summary)
}
}
}
if p.run != nil {
v.Took = r.took.Round(time.Millisecond).String()
}
run.Probes = append(run.Probes, v)
}
blind, _ = d.confirm.pass(ctx, d.keeper, sourceDoctor, blind)
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
}
ended := time.Now()
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
d.mu.Lock()
d.last, d.ended = &run, ended
d.mu.Unlock()
d.sayHeartbeat(ctx, run)
return run
}
// sourceDoctor is what raises a probe's own failure to run.
const sourceDoctor = "doctor"
// kindedAs gives each observation of a probe the probe's kind where it named none.
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
out := make([]conditions.Observation, 0, len(obs))
for _, o := range obs {
if o.Kind == "" {
o.Kind = kind
}
out = append(out, o)
}
return out
}
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
// says it outward: the watcher hears nothing.
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
if d.teller == nil {
return
}
body, err := json.Marshal(run)
if err != nil {
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
return
}
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
"will say the self-check is silent: %v\n", run.Run, err)
}
}
// doctorFrom is the serving controller's self-check; nil in any other process.
var doctorFrom *doctor
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
func doctorCommand(ctx context.Context, args []string) error {
sub := ""
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("doctor [run|probes|signals] [--json]")
}
answer, err := doctorAnswer(ctx, sub)
if err != nil {
return err
}
if *asJSON {
return printJSON(answer)
}
fmt.Print(doctorText(answer))
return nil
}
// doctorAnswer is what the verb answers, as data.
func doctorAnswer(ctx context.Context, sub string) (any, error) {
switch sub {
case "":
if doctorFrom != nil {
if run := doctorFrom.lastRun(); run != nil {
return verdictAnswer(*run, time.Now()), nil
}
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
}
run, err := lastHeartbeat(ctx)
if err != nil {
return nil, err
}
return verdictAnswer(run, time.Now()), nil
case "run":
d := doctorFrom
if d == nil {
local, closeIt, err := localDoctor(ctx)
if err != nil {
return nil, err
}
defer closeIt()
d = local
}
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
case "probes":
return probesAnswer(), nil
case "signals":
if doctorFrom == nil || doctorFrom.watchdogs == nil {
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
"hears them: ask it through the mesh-controller seat's doctor verb")
}
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
}
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
}
// verdictAnswer is a run as the verb answers it, with its age.
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it, " +
"and one a single look can be wrong about — an unanswered question, a slow answer — is raised when two " +
"runs in a row see it"}
}
// probesAnswer is the registry.
func probesAnswer() map[string]any {
var out []map[string]any
for _, p := range probeRegistry {
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
if len(p.Raises) > 0 {
row["raises"] = p.Raises
}
if p.Deferred != "" {
row["deferred"] = p.Deferred
}
out = append(out, row)
}
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
}
// signalsAnswer is every row of the signals table with the age of its newest signal.
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
var rows []map[string]any
for _, r := range signalsTable {
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
switch {
case r.Deferred != "":
row["deferred"] = r.Deferred
case f == nil:
row["newest"] = "not yet looked at"
default:
if err := r.needs(f); err != nil {
row["blind"] = err.Error()
} else if newest := r.newest(f); newest.IsZero() {
row["newest"] = "none heard"
} else {
row["newest"] = newest.UTC().Format(time.RFC3339)
row["age"] = f.now.Sub(newest).Round(time.Second).String()
}
}
rows = append(rows, row)
}
out := map[string]any{"signals": rows, "every": watchEvery.String()}
if !ticked.IsZero() {
out["looked"] = ticked.UTC().Format(time.RFC3339)
}
return out
}
// doctorText is an answer as a person reads it.
func doctorText(answer any) string {
body, _ := json.Marshal(answer)
var b strings.Builder
var verdict struct {
Run doctorRun `json:"run"`
Age string `json:"age"`
}
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
r := verdict.Run
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
for _, p := range r.Probes {
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
for _, f := range p.Found {
fmt.Fprintf(&b, " %s\n", f)
}
for _, f := range p.Unconfirmed {
fmt.Fprintf(&b, " unconfirmed, raised if the next run sees it too: %s\n", f)
}
if p.Error != "" {
fmt.Fprintf(&b, " %s\n", p.Error)
}
}
return b.String()
}
pretty, _ := json.MarshalIndent(answer, "", " ")
return string(pretty) + "\n"
}
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
// the serving controller answers `doctor` from.
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
var run doctorRun
err := onTheBus(func(conn *nats.Conn) error {
js, err := conn.JetStream(nats.Context(ctx))
if err != nil {
return err
}
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
if errors.Is(err, nats.ErrMsgNotFound) {
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
}
if err != nil {
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
}
return json.Unmarshal(msg.Data, &run)
})
return run, err
}
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
// its own connection, and its own keeper, closed after.
func localDoctor(ctx context.Context) (*doctor, func(), error) {
open, err := openStores(ctx)
if err != nil {
return nil, nil, err
}
js, err := dialTheBus()
if err != nil {
open.Close()
return nil, nil, err
}
k, err := keeperOn(ctx, js.Conn())
if err != nil {
js.Close()
open.Close()
return nil, nil, err
}
jsCtx := js.Context()
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
host: controlHost(ctx, open.inventory)}
return d, func() {
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
k.Close(flushing)
js.Close()
open.Close()
}, nil
}
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
func sortedFound(obs []conditions.Observation) []conditions.Observation {
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
return obs
}
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
type probeAsksKey struct{}
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
p, ok := ctx.Value(probeAsksKey{}).(probe)
if !ok {
return "a caller outside the self-check", false
}
for _, v := range p.Asks {
if v.Seat == seat && v.Verb == verb {
return p.ID, true
}
}
return p.ID, false
}
cmd/mesh-controller/doctor_test.go
-430
View File
@@ -1,430 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"net"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/net/dns/dnsmessage"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
// withProbes runs the test with a registry of its own.
func withProbes(t *testing.T, probes ...probe) {
t.Helper()
before := probeRegistry
probeRegistry = probes
t.Cleanup(func() { probeRegistry = before })
}
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
var broken atomic.Bool
broken.Store(true)
withProbes(t,
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
if broken.Load() {
return []conditions.Observation{failing}, nil
}
return nil, nil
}},
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
if broken.Load() {
return nil, errors.New("the store is away")
}
return nil, nil
}},
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
if broken.Load() {
<-ctx.Done()
time.Sleep(50 * time.Millisecond)
}
return nil, nil
}},
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
)
before := probeWithin
probeWithin = 200 * time.Millisecond
t.Cleanup(func() { probeWithin = before })
store := conditions.NewInMemory()
told := &conditions.Told{}
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
d := &doctor{keeper: k, teller: told, host: "anchor"}
// A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot
// run it either (novox/hq issue 277): one timed-out question is not a probe gone blind.
first := d.runOnce(t.Context(), "a test")
if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
t.Fatalf("counted %+v", first.Counts)
}
if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" {
t.Fatalf("after one run, open: %+v", open)
}
run := d.runOnce(t.Context(), "a test")
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
t.Fatalf("counted %+v", run.Counts)
}
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
var keys []string
for _, c := range open {
keys = append(keys, c.Key+"="+c.Kind)
}
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
"probe.P4.failed=probe-failed"} {
if !slices.Contains(keys, want) {
t.Errorf("%s is not open: %v", want, keys)
}
}
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent {
t.Fatalf("said %v", told.Names)
}
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
t.Fatal("the run is not the last verdict")
}
body, _ := json.Marshal(run)
var shape map[string]any
_ = json.Unmarshal(body, &shape)
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
if _, ok := shape[field]; !ok {
t.Errorf("the heartbeat carries no %q: %s", field, body)
}
}
// Mended: the next run clears every one of them.
broken.Store(false)
d.runOnce(t.Context(), "a test")
if open, _ := k.Open(t.Context()); len(open) != 0 {
t.Fatalf("a passing run left open %+v", open)
}
}
// **The registry says what each probe asserts**, and a probe not built says why and when.
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
seen := map[string]bool{}
for _, p := range probeRegistry {
if seen[p.ID] {
t.Errorf("%s twice", p.ID)
}
seen[p.ID] = true
if p.Asserts == "" || p.From == "" || p.Kind == "" {
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
}
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
}
}
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
if !seen[id] {
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
}
}
}
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
// stopped is S10 (signals_test.go).
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
w := &watchdogs{started: time.Now().Add(-time.Hour)}
d := &doctor{watchdogs: w, host: "anchor"}
got, err := probeWatchdogs(t.Context(), d)
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
t.Fatalf("%+v %v", got, err)
}
w.ticked = time.Now()
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
t.Fatalf("%+v", got)
}
}
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
open := aMesh(t)
got, err := probeDeclarations(t.Context(), &doctor{open: open})
if err != nil {
t.Fatal(err)
}
if len(got) != 0 {
t.Fatalf("a healthy mesh failed D1: %+v", got)
}
}
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
one, two := rivals()
register(t, open, one)
register(t, open, two)
for _, m := range []string{"rival-one", "rival-two"} {
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
t.Fatal(err)
}
}
got, err := probeDeclarations(ctx, &doctor{open: open})
linted(got)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
t.Fatalf("%+v", got)
}
}
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
answerAs := func(rcode dnsmessage.RCode) string {
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = conn.Close() })
go func() {
buf := make([]byte, 1500)
for {
n, from, err := conn.ReadFrom(buf)
if err != nil {
return
}
var q dnsmessage.Message
if q.Unpack(buf[:n]) != nil {
continue
}
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
if q.Questions[0].Type == dnsmessage.TypeA {
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
} else {
reply.RCode = rcode
}
packed, _ := reply.Pack()
_, _ = conn.WriteTo(packed, from)
}
}()
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
return port
}
before := resolverPort
t.Cleanup(func() { resolverPort = before })
resolverPort = answerAs(dnsmessage.RCodeSuccess)
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
t.Fatalf("%v %v %v", v4, rcode, err)
}
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
}
resolverPort = answerAs(dnsmessage.RCodeNameError)
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
t.Fatalf("NXDOMAIN read as %v", rcode)
}
}
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
url := testbus.URL(t)
open := aMesh(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
_ = js.Context().DeleteStream(s)
}
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
t.Fatal(err)
}
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
d := &doctor{open: open, js: js}
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
got, err := p(t.Context(), d)
if err != nil || len(got) != 0 {
t.Fatalf("a bus just raised fails: %+v %v", got, err)
}
}
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
t.Fatal(err)
}
info, err := js.Context().StreamInfo("EVENTS")
if err != nil {
t.Fatal(err)
}
cfg := info.Config
cfg.MaxMsgsPerSubject = 3
if _, err := js.Context().UpdateStream(&cfg); err != nil {
t.Fatal(err)
}
consumers, err := probeConsumers(t.Context(), d)
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
}
streams, err := probeStreams(t.Context(), d)
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
t.Fatalf("the redefined stream: %+v %v", streams, err)
}
}
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
heard := make(chan *nats.Msg, 16)
for _, subject := range broker.BusAdvisories {
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
t.Fatal(err)
}
}
api, _ := jetstream.New(conn)
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
if err != nil {
t.Fatal(err)
}
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
if err != nil {
t.Fatal(err)
}
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
t.Fatal(err)
}
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
t.Fatal(err)
}
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
time.Sleep(300 * time.Millisecond)
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
t.Fatal(err)
}
kinds := map[string]string{}
deadline := time.After(5 * time.Second)
for len(kinds) < 2 {
select {
case m := <-heard:
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
kinds[a.Kind] = a.Said
}
case <-deadline:
t.Fatalf("the bus said only %v", kinds)
}
}
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
t.Fatalf("%v", kinds)
}
}
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
// machine read as behind right after a push sent it the current build — it says the digest-named
// directory it runs from, and the mesh holds a commit).
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
}}
delivered := deliveredVersions(m)
if !slices.Equal(delivered, []string{"31045596c83a"}) {
t.Fatalf("%v", delivered)
}
commit := "1545b00a9f0c"
for _, c := range []struct {
reported string
behind bool
}{
{"31045596c83a", false}, // the live case: current, and was called behind
{"0123456789ab", true}, // another delivery
{"1545b00a", false}, // placed by hand, stamped with the commit
{"", false}, // not said
} {
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
}
}
if engineBehind("31045596c83a", nil, commit) {
t.Error("behind a mesh that holds no delivered build")
}
}
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
asked := 0
for _, p := range probeRegistry {
for _, v := range p.Asks {
asked++
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
}
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
}
}
}
if asked == 0 {
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
}
// And a probe asking what it did not declare is refused before anything is sent.
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
!strings.Contains(err.Error(), "does not declare") {
t.Fatalf("an undeclared question was asked: %v", err)
}
}
// subjectMatches is the bus's matching of a permission pattern against a subject.
func subjectMatches(pattern, subject string) bool {
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
for i, tok := range p {
if tok == ">" {
return len(s) > i
}
if i >= len(s) || (tok != "*" && tok != s[i]) {
return false
}
}
return len(p) == len(s)
}
cmd/mesh-controller/dry_run_test.go
-38
View File
@@ -1,38 +0,0 @@
package main
import (
"encoding/json"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
// store at all, so anything that tried to record or register would fail rather than pass quietly.
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
err := builds{}.Built(t.Context(), link.BuildResult{
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
})
if err != nil {
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
}
}
// The mark survives the wire both ways: asked as a dry run, answered as one.
func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
raw, _ := json.Marshal(link.BuildRequest{ID: "build-1", Repository: "r", DryRun: true})
var asked link.BuildRequest
if err := json.Unmarshal(raw, &asked); err != nil || !asked.DryRun {
t.Fatalf("the request lost its dry-run mark: %s", raw)
}
raw, _ = json.Marshal(link.BuildResult{ID: "build-1", DryRun: true})
var answered link.BuildResult
if err := json.Unmarshal(raw, &answered); err != nil || !answered.DryRun {
t.Fatalf("the outcome lost its dry-run mark: %s", raw)
}
raw, _ = json.Marshal(link.BuildResult{ID: "build-2"})
if string(raw) != `{"id":"build-2","repository":"","on":""}` {
t.Fatalf("an ordinary outcome carries a dry-run mark: %s", raw)
}
}
cmd/mesh-controller/durations.go
-170
View File
@@ -1,170 +0,0 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
//
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
// build's outcome — and summarised here per machine, repository or module.
// recordBuildDuration measures one build from its ask to its outcome heard.
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
if asked.IsZero() || result.ID == "" {
return
}
subject := result.Module
if subject == "" {
subject = result.Repository
}
detail := "built"
if result.Failed != "" {
detail = "failed: " + firstLine(result.Failed)
}
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
}
}
// durationSummary is one subject's measurements of one kind.
type durationSummary struct {
Kind string `json:"kind"`
Subject string `json:"subject"`
Count int `json:"count"`
Median string `json:"median"`
P90 string `json:"p90"`
Max string `json:"max"`
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
// time: three times the slowest apply (S2), three times the median word interval (S1).
Suggests string `json:"suggests,omitempty"`
}
func summarise(ds []inventory.Duration) []durationSummary {
type key struct{ kind, subject string }
by := map[key][]time.Duration{}
for _, d := range ds {
k := key{d.Kind, d.Subject}
by[k] = append(by[k], d.Took)
}
var out []durationSummary
for k, took := range by {
slices.Sort(took)
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
switch k.kind {
case inventory.DurationApply:
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
case inventory.DurationHeartbeatGap:
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
}
out = append(out, s)
}
sort.Slice(out, func(i, j int) bool {
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
if ki != kj {
return ki < kj
}
return out[i].Subject < out[j].Subject
})
return out
}
func round(d time.Duration) string {
switch {
case d < time.Second:
return d.Round(time.Millisecond).String()
case d < time.Minute:
return d.Round(100 * time.Millisecond).String()
default:
return d.Round(time.Second).String()
}
}
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
func durationsCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("durations", flag.ContinueOnError)
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
days := set.Int("days", 14, "how many days back")
asJSON := set.Bool("json", false, "the summary as data")
all := set.Bool("all", false, "every measurement rather than the summary")
if _, err := parseAround(set, args); err != nil {
return err
}
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
if err != nil {
return err
}
if *all {
body, err := json.MarshalIndent(ds, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
summary := summarise(ds)
if *asJSON {
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
if len(summary) == 0 {
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
"plan-tier and build durations as it hears them\n", *days)
return nil
}
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
for _, s := range summary {
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
if s.Suggests != "" {
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
}
}
return nil
}
// forgettingOldDurations removes what is older than a month, at start and daily after.
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
for {
if n, err := inv.ForgetOldDurations(ctx); err != nil {
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
} else if n > 0 {
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
}
select {
case <-ctx.Done():
return
case <-time.After(24 * time.Hour):
}
}
}
cmd/mesh-controller/epoch_send_test.go
-115
View File
@@ -1,115 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said
// it reads one, and to no other: an older node-engine refuses a key it does not know, whole.
// bodiesDelivery records each send as the mesh does, and keeps the bodies.
type bodiesDelivery struct {
recordedDelivery
bodies map[string][]byte
}
func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
b.bodies[s.node] = body
return b.recordedDelivery.declare(ctx, s, body)
}
func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
epoch := uint64(57)
was := epochForActs
epochForActs = func(context.Context) (uint64, error) { return epoch, nil }
t.Cleanup(func() { epochForActs = was })
anchor, err := inv.NodeByName(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil {
t.Fatal(err)
}
carried := func(node string) (epoch float64, has bool) {
var envelope map[string]any
if err := json.Unmarshal(d.bodies[node], &envelope); err != nil {
t.Fatal(err)
}
epoch, has = envelope["epoch"].(float64)
return epoch, has
}
if e, has := carried("anchor"); !has || e != 57 {
t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"])
}
if _, has := carried("laptop"); has {
t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"])
}
// A new holder of the lease is not a change of the machine: neither reads as behind.
epoch = 58
would, err := wouldSend(ctx, open, mustNodes(t, open))
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
sent, err := inv.Outstanding(ctx, node)
if err != nil {
t.Fatal(err)
}
if would[node] != sent {
t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node)
}
}
}
// A process that may not act composes nothing and sends nothing: its number is not taken.
func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
was := epochForActs
epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }
t.Cleanup(func() { epochForActs = was })
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "")
if err != nil {
t.Fatal(err)
}
if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") {
t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused)
}
anchor, _ := open.inventory.NodeByName(ctx, "anchor")
if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 {
t.Fatalf("a controller without the lease took sequence %d", seq)
}
// And at the send itself: the gate every declaration passes.
gate := link.ActingGate
link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") }
t.Cleanup(func() { link.ActingGate = gate })
if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil ||
!strings.Contains(err.Error(), "lost the lease") {
t.Fatalf("a declaration was let through the gate: %v", err)
}
}
cmd/mesh-controller/facts.go
-645
View File
@@ -1,645 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/validate"
"github.com/novox/mesh-controller/internal/artifacts"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The facts snapshot (novox/hq to-be 45 §9, ADR 0227 rule 9): what a merge check needs to judge a change
// against the mesh that runs, written by the controller to the artifact store, where the build seat reads
// it. internal/facts says what it holds and what it never holds; this composes it from the store and
// keeps it current.
// factsEvery is how often the snapshot is composed. It is kept when it moved — a machine, an
// assignment, a seat, a setting, a build — or once a day when nothing did, so its age says the
// controller is still writing it (S14).
var factsEvery = 10 * time.Minute
// factsDaily is how old a snapshot of an unchanged mesh may grow before it is written again.
const factsDaily = 24 * time.Hour
// factsStaleAfter is S14's bound: a snapshot older than this is one no check should be fed.
const factsStaleAfter = 48 * time.Hour
// factsExport is what this controller knows of the snapshot it keeps: when the newest was taken, its
// content, and the last attempt's error.
type factsExport struct {
mu sync.Mutex
taken time.Time
content string
digest string
err error
began time.Time
}
// exportedFacts is this process's export, read by S14.
var exportedFacts = &factsExport{}
func (e *factsExport) last() (taken time.Time, digest string, began time.Time, err error) {
e.mu.Lock()
defer e.mu.Unlock()
return e.taken, e.digest, e.began, e.err
}
func (e *factsExport) kept(f snapshot.Facts, content, digest string) {
e.mu.Lock()
defer e.mu.Unlock()
e.taken, e.content, e.digest, e.err = f.Taken, content, digest, nil
}
func (e *factsExport) failed(err error) {
e.mu.Lock()
defer e.mu.Unlock()
e.err = err
}
// exportingFacts keeps the snapshot current for as long as this controller holds the lease: ctx ends
// when it stops acting.
func exportingFacts(ctx context.Context, open *stores, busVersion func() string) {
exportedFacts.mu.Lock()
exportedFacts.began = time.Now()
exportedFacts.mu.Unlock()
// What the store holds already, so a restarted controller neither writes an unchanged snapshot again
// nor reads its age as zero.
if address, err := factsStore(ctx, open); err == nil {
if body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag); err == nil {
if f, err := snapshot.Decode(body); err == nil {
content, _ := f.Content()
exportedFacts.kept(f, content, digest)
}
}
}
failing := ""
first := time.NewTimer(time.Minute)
defer first.Stop()
tick := time.NewTicker(factsEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-first.C:
case <-tick.C:
}
wrote, err := exportFacts(ctx, open, busVersion(), false)
why := ""
if err != nil {
why = err.Error()
exportedFacts.failed(err)
}
if why != failing {
if why != "" {
fmt.Printf("the facts snapshot cannot be kept: %s\n", why)
} else {
fmt.Println("the facts snapshot is kept again")
}
failing = why
}
if wrote != "" {
fmt.Printf("the facts snapshot moved and is kept as %s\n", short(strings.TrimPrefix(wrote, "sha256:")))
}
}
}
// exportFacts composes the snapshot and keeps it when it moved, or when the one kept is a day old, or
// when told to. Answers the digest it kept, empty when it kept nothing.
func exportFacts(ctx context.Context, open *stores, busVersion string, force bool) (string, error) {
f, err := gatherFacts(ctx, open, busVersion)
if err != nil {
return "", err
}
content, err := f.Content()
if err != nil {
return "", err
}
exportedFacts.mu.Lock()
unchanged := content == exportedFacts.content && time.Since(exportedFacts.taken) < factsDaily
exportedFacts.mu.Unlock()
if unchanged && !force {
return "", nil
}
body, err := f.Encode()
if err != nil {
return "", err
}
address, err := factsStore(ctx, open)
if err != nil {
return "", err
}
digest, err := (artifacts.Store{Address: address}).PutTagged(ctx, snapshot.Repository, snapshot.Tag, snapshot.MediaType, body)
if err != nil && digest == "" {
return "", err
}
exportedFacts.kept(f, content, digest)
return digest, err
}
// factsStore is the artifact store as this controller reaches it.
func factsStore(ctx context.Context, open *stores) (string, error) {
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
}
address, err := artifactStoreAddress(ctx, open.inventory, shelf, "")
if err != nil {
return "", err
}
if address == "" {
return "", errors.New("the artifact store is not on the private network, so there is nowhere to keep the facts")
}
return address, nil
}
// gatherFacts composes one snapshot from the store: read only, nothing made, nothing sent.
func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot.Facts, error) {
inv := open.inventory
f := snapshot.Facts{Format: snapshot.Format, Taken: time.Now().UTC(), Controller: snapshot.Build{Version: version}}
f.Versions.Bus = busVersion
storeVersion, err := inv.ServerVersion(ctx)
if err != nil {
return snapshot.Facts{}, fmt.Errorf("the store will not say its version: %w", err)
}
f.Versions.Store = storeVersion
nodes, err := inv.Nodes(ctx)
if err != nil {
return snapshot.Facts{}, err
}
overlays, err := inv.Overlays(ctx)
if err != nil {
return snapshot.Facts{}, err
}
place := map[string]inventory.Overlay{}
for _, o := range overlays {
place[o.Name] = o
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return snapshot.Facts{}, err
}
shelf := map[string]catalogue.Manifest{}
for _, e := range entries {
shelf[e.Manifest.Module] = e.Manifest
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
return snapshot.Facts{}, err
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return snapshot.Facts{}, err
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return snapshot.Facts{}, err
}
holdings, err := inv.Holdings(ctx)
if err != nil {
return snapshot.Facts{}, err
}
// **Every name first**, so text read afterwards — a setting naming a machine, a problem naming a
// site — has it replaced wherever it appears.
scrub := snapshot.NewScrubber()
domains := map[string]string{}
for _, n := range nodes {
scrub.Machine(n.Name)
if n.Account != "" && n.Account != "root" {
scrub.Account(n.Account)
}
d, err := inv.PublicDomainOf(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
domains[n.Name] = scrub.Domain(d)
}
for _, o := range overlays {
scrub.Site(o.Site)
}
// What a merge check runs in and reads beside it, as the build seat would be asked for it.
if held, err := inv.Held(ctx); err == nil {
for language, reference := range builder.ToolchainsOf(held) {
if f.Versions.Toolchains == nil {
f.Versions.Toolchains = map[string]string{}
}
f.Versions.Toolchains[language] = catalogue.Recorded(reference)
}
} else {
return snapshot.Facts{}, err
}
for _, e := range entries {
if dir, core := coreModules[e.Manifest.Module]; core && !e.Provided && e.Source.Repository != "" {
for d, ref := range besideRefs(dir, current[e.Manifest.Module].Commit) {
if f.Beside == nil {
f.Beside = map[string]string{}
}
f.Beside[d] = ref
}
}
}
if c, ok := current["mesh-controller"]; ok {
f.Controller.Commit = c.Commit
}
gens, gensErr := generators(ctx, open)
hostShelf := shelf[hostModule]
meshWide := map[string]bool{}
engines := map[string]bool{}
for _, n := range nodes {
m := snapshot.Machine{Name: scrub.Machine(n.Name), Length: len(n.Name), Adopted: n.Adopted,
AccountHome: scrub.Text(n.AccountHome), NodeEngine: n.HostVersion, PublicDomain: domains[n.Name]}
switch n.Account {
case "", "root":
m.Account = n.Account
default:
m.Account = scrub.Account(n.Account)
}
if n.HostVersion != "" {
engines[n.HostVersion] = true
}
m.System = systemOf(hostShelf, n.HostVersion)
m.Libc = libcOf(m.System)
reported, err := inv.DescribedOf(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
outward, err := inv.OutwardLinksOf(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
for _, l := range outward {
m.OutwardLinks = append(m.OutwardLinks, scrub.Text(l))
}
capabilities, err := inv.Profile(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
for _, c := range capabilities {
kept := snapshot.Capability{Name: c.Name, Present: c.Present}
// The detail only where it is a version: everything else a detector says — a ruleset, a
// device, a path — is the machine's own business and no check reads it.
if c.Present && (c.Name == "container-runtime" || c.Name == "package-manager") {
kept.Detail = scrub.Text(c.Detail)
}
m.Capabilities = append(m.Capabilities, kept)
}
if o, ok := place[n.Name]; ok {
m.Site, m.Hub, m.Public, m.OnNetwork = scrub.Site(o.Site), o.Hub, o.Endpoint != "", o.Address != ""
}
assigned, err := inv.Assigned(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
m.Assigned = assigned
sent, known, err := inv.SentBuilds(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
if known && slices.Contains(assigned, broker.RuntimeModule) {
m.NodeTools = sent[broker.RuntimeModule]
}
pins, err := inv.PinsFor(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
for provision, c := range pins {
m.Pins = append(m.Pins, snapshot.Pin{Provision: provision, Machine: scrub.Machine(c.Node), Module: c.Module})
}
for _, module := range assigned {
held, err := inv.SecretsOf(ctx, n.Name, module)
if err != nil {
return snapshot.Facts{}, err
}
for _, h := range held {
if h.Origin == inventory.OriginAccepted {
m.Accepted = append(m.Accepted, snapshot.Accepted{Module: module, Name: h.Name,
Provider: scrub.Machine(h.Provider), Local: h.Local})
}
}
layers, err := inv.SettingsFor(ctx, n.Name, module)
if err != nil {
return snapshot.Facts{}, err
}
for _, layer := range layers {
if layer.From == catalogue.MeshWideLayer {
if !meshWide[module] {
meshWide[module] = true
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
}
continue
}
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
}
}
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
if ctx.Err() != nil {
return snapshot.Facts{}, ctx.Err()
}
f.Machines = append(f.Machines, m)
}
for e := range engines {
f.Versions.NodeEngines = append(f.Versions.NodeEngines, e)
}
// Seats and their holders, and from them the roles a machine is named by.
roles := map[string][]string{}
seats := map[string]*snapshot.Seat{}
for _, h := range holdings {
key := h.Claim + "\x00" + h.Scope
s, ok := seats[key]
if !ok {
s = &snapshot.Seat{Name: h.Claim, Scope: h.Scope}
seats[key] = s
}
s.Holders = append(s.Holders, snapshot.Holder{Machine: scrub.Machine(h.Node), Module: h.Module})
if h.Scope == catalogue.ScopeMesh {
role := "holds " + h.Claim
if h.Claim == catalogue.ControllerSeatName {
role = "the control node"
}
roles[h.Node] = append(roles[h.Node], role)
}
}
for _, s := range seats {
f.Seats = append(f.Seats, *s)
}
for i := range f.Machines {
for _, n := range nodes {
if scrub.Machine(n.Name) != f.Machines[i].Name {
continue
}
f.Machines[i].Roles = roles[n.Name]
if f.Machines[i].Hub {
f.Machines[i].Roles = append(f.Machines[i].Roles, "the hub")
}
}
}
// Every module, as the mesh holds it, and where it is built from.
newest := map[string]inventory.Source{}
count := map[string]int{}
for _, e := range entries {
raw, err := json.Marshal(e.Manifest)
if err != nil {
return snapshot.Facts{}, err
}
mod := snapshot.Module{Name: e.Manifest.Module, Repository: e.Source.Repository, Path: e.Source.Path,
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
Manifest: raw}
for _, r := range read[e.Manifest.Module] {
mod.Reads = append(mod.Reads, r.Repository)
}
f.Modules = append(f.Modules, mod)
if e.Provided || e.Source.Repository == "" {
continue
}
count[e.Source.Repository]++
if was, ok := newest[e.Source.Repository]; !ok || e.Source.Seen.After(was.Seen) {
newest[e.Source.Repository] = e.Source
}
}
for repository, s := range newest {
commit := s.Head
if commit == "" {
commit = s.BuiltFrom
}
f.Sources = append(f.Sources, snapshot.Source{Repository: repository, Commit: commit, Modules: count[repository]})
}
for _, e := range edges {
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
}
f.Sorted()
return f, nil
}
// declarationFacts is how one machine's declaration composes now, as the next push would compose it and
// without making anything (D1's composition), and whether the node-engine's validator takes it.
func declarationFacts(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
gensErr error, scrub *snapshot.Scrubber) snapshot.Declaration {
var d snapshot.Declaration
if gensErr != nil {
d.Problems = []string{scrub.Text("the private network cannot be computed: " + oneLine(gensErr.Error()))}
return d
}
declared, problems, err := composedAndValidated(ctx, open, node, gens, Foreseeing)
if err != nil {
d.Problems = []string{scrub.Text(oneLine(err.Error()))}
return d
}
for _, p := range problems {
d.Problems = append(d.Problems, scrub.Text(p))
}
d.Composes = len(problems) == 0
if body, err := declared.Body(); err == nil {
d.Digest = fmt.Sprintf("sha256:%x", sha256.Sum256(body))
}
// Scrubbed like every other word: a resource is named after the machine a grant is for.
for _, r := range resourceNames(declared.Resources) {
d.Resources = append(d.Resources, scrub.Text(r))
}
for module, why := range declared.leftOutWhy {
if d.LeftOut == nil {
d.LeftOut = map[string]string{}
}
d.LeftOut[module] = scrub.Text(why)
}
for _, o := range declared.withheld {
d.Withheld = append(d.Withheld, scrub.Text(o.String()))
}
for _, u := range declared.unbound {
d.Unbound = append(d.Unbound, scrub.Text(u.String()))
}
sort.Strings(d.Withheld)
sort.Strings(d.Unbound)
return d
}
// composedAndValidated composes one machine's declaration — as a push would (Allocating) or as the next
// push will without making anything (Foreseeing) — with the order it was last sent, and runs the
// node-engine's own validator over the body. An error is that it did not compose; problems are what the
// validator refuses.
func composedAndValidated(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
choosing Choosing) (sendable, []string, error) {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
return sendable{}, nil, err
}
declared, err := declarationWith(ctx, open, node, plan, settings, gens, choosing)
if err != nil {
return sendable{}, nil, err
}
record, err := open.inventory.NodeByName(ctx, node)
if err != nil {
return sendable{}, nil, err
}
if declared.Sequence, err = open.inventory.Sequence(ctx, record.ID); err != nil {
return sendable{}, nil, err
}
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
return sendable{}, nil, err
}
body, err := declared.Body()
if err != nil {
return sendable{}, nil, err
}
return declared, validate.Declaration(body), nil
}
// resourceNames are a declaration's resources as `type:id`, sorted.
func resourceNames(resources []map[string]any) []string {
out := make([]string, 0, len(resources))
for _, r := range resources {
kind, _ := r["type"].(string)
id, _ := r["id"].(string)
out = append(out, kind+":"+id)
}
sort.Strings(out)
return out
}
// systemOf is the system a node-engine of that version was built for, read from the build the mesh
// holds: the artifact a resource delivered into `versions/<version>` came from is named for its system
// (`host-arch`). Empty when the version is not a delivered one — an engine placed by hand.
func systemOf(host catalogue.Manifest, version string) string {
if version == "" {
return ""
}
for _, r := range host.Resources {
path, _ := r["path"].(string)
if !strings.HasSuffix(path, "/versions/"+version) {
continue
}
source, _ := r["source"].(string)
for _, part := range strings.Split(source, "/") {
if system, ok := strings.CutPrefix(part, "host-"); ok && system != "" {
return system
}
}
}
return ""
}
// libcOf is the C library of a system the node-engine is built for.
func libcOf(system string) string {
switch system {
case "arch":
return "glibc"
case "alpine":
return "musl"
case "android":
return "bionic"
}
return ""
}
// factsCommand is `facts`: what the controller keeps, and keeping it now.
//
// facts the snapshot the artifact store holds: when, which, how many machines
// facts show the same, whole, as JSON
// facts export compose and keep one now
// facts compose compose one and print it, keeping nothing
func factsCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("facts", flag.ContinueOnError)
rest, err := parseAround(set, args)
if err != nil {
return err
}
what := ""
if len(rest) > 0 {
what = rest[0]
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
switch what {
case "", "show":
address, err := factsStore(ctx, open)
if err != nil {
return err
}
body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag)
if err != nil {
return err
}
if what == "show" {
_, err := os.Stdout.Write(body)
return err
}
f, err := snapshot.Decode(body)
if err != nil {
return err
}
fmt.Printf("the facts snapshot kept as %s:%s is %s, taken %s (%s ago) by controller %s\n",
snapshot.Repository, snapshot.Tag, short(strings.TrimPrefix(digest, "sha256:")),
f.Taken.Format(time.RFC3339), ago(time.Since(f.Taken)), orNone(f.Controller.Commit))
fmt.Printf(" %d machine(s), %d module(s), %d seat(s); the longest machine name is %d characters\n",
len(f.Machines), len(f.Modules), len(f.Seats), f.Longest())
fmt.Printf(" the bus runs %s, the store %s\n", orNone(f.Versions.Bus), orNone(f.Versions.Store))
for _, m := range f.Machines {
state := "composes"
if !m.Declaration.Composes {
state = "does NOT compose: " + strings.Join(m.Declaration.Problems, "; ")
}
fmt.Printf(" %-12s %s; %d module(s); %s\n", m.Name, m.Described(), len(m.Assigned), state)
}
return nil
case "export", "compose":
busVersion := ""
if server, err := connectLink(ctx, nil, nil, nil); err == nil {
busVersion = busVersionOf(server)
server.Close()
}
if what == "compose" {
f, err := gatherFacts(ctx, open, busVersion)
if err != nil {
return err
}
body, err := f.Encode()
if err != nil {
return err
}
_, err = os.Stdout.Write(append(body, '\n'))
return err
}
digest, err := exportFacts(ctx, open, busVersion, true)
if err != nil {
return err
}
fmt.Printf("the facts snapshot is kept as %s:%s, %s\n", snapshot.Repository, snapshot.Tag, digest)
return nil
}
return fmt.Errorf("facts [show|export|compose], not %q", what)
}
// busVersionOf is the bus server's release, as it told this connection.
func busVersionOf(server *link.Server) string {
if bus, ok := server.Bus().(link.OverNATS); ok && bus.Conn != nil {
return bus.Conn.ConnectedServerVersion()
}
return ""
}
cmd/mesh-controller/facts_test.go
-137
View File
@@ -1,137 +0,0 @@
package main
import (
"regexp"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
)
// The facts snapshot (novox/hq to-be 45 §9): composed from the store, every machine under a pseudonym
// of its name's length, and nothing of the installation in it — no secret, no address, no name.
// aMeshWithSecrets is aMesh with a provider and its consumers, a value given by hand, settings carrying
// a password, an address and a machine's name, and a push's worth of credentials made.
func aMeshWithSecrets(t *testing.T) (*stores, []string) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"},
Resources: []map[string]any{{"id": "config", "type": "file", "path": "/etc/files/config.json",
"mode": "0600", "content": "{}", "merge": "json"}}})
for _, a := range [][2]string{{"anchor", "objects"}, {"laptop", "files"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
secrets := []string{"Hunter2-Is-Not-A-Password-9f8e7d", "0123456789abcdefABCDEF0123456789zz"}
if err := open.inventory.SetSettings(ctx, "", "files", map[string]any{
"admin_password": secrets[0], "upstream": "10.77.0.9", "hub": "anchor"}); err != nil {
t.Fatal(err)
}
if err := open.inventory.SetSettings(ctx, "laptop", "files", map[string]any{
"note": "reach me at 192.168.1.135, token " + secrets[1]}); err != nil {
t.Fatal(err)
}
// A push's worth of composition, which makes the pair credential the consumer is sent.
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"laptop", "anchor"} {
if _, _, err := composedAndValidated(ctx, open, node, gens, Allocating); err != nil {
t.Fatalf("%s does not compose: %v", node, err)
}
}
issued, err := open.inventory.SecretsFrom(ctx, "anchor")
if err != nil || len(issued) == 0 {
t.Fatalf("no credential was made for the consumer: %v", err)
}
for _, s := range issued {
// Sealed, never kept plain (ADR 0004): the sealed blobs are what the store holds, and none may leave.
secrets = append(secrets, s.ForConsumer, s.ForProvider)
}
return open, secrets
}
func TestTheFactsCarryNoSecretNoAddressAndNoName(t *testing.T) {
open, secrets := aMeshWithSecrets(t)
f, err := gatherFacts(t.Context(), open, "2.11.17")
if err != nil {
t.Fatal(err)
}
body, err := f.Encode()
if err != nil {
t.Fatal(err)
}
text := string(body)
for _, s := range secrets {
if s != "" && strings.Contains(text, s) {
t.Errorf("a secret is in the snapshot: %q", s)
}
}
for _, leaked := range []string{"anchor", "laptop", "10.77.0.", "192.168.1.135", ".example:51820"} {
if strings.Contains(text, leaked) {
t.Errorf("%q is in the snapshot", leaked)
}
}
// Every address it carries is a documentation address.
for _, a := range regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b`).FindAllString(text, -1) {
if !strings.HasPrefix(a, "192.0.2.") && !strings.HasPrefix(a, "198.51.100.") && !strings.HasPrefix(a, "203.0.113.") {
t.Errorf("%s is an address outside the documentation ranges", a)
}
}
// What a check needs is there: every machine, its length, its modules, its declaration composing.
if len(f.Machines) != 2 || f.Longest() != len("laptop") {
t.Fatalf("machines %+v, longest %d", f.Machines, f.Longest())
}
anchor := snapshot.Pseudonym("machine", "anchor")
m, ok := f.Machine(anchor)
if !ok || !m.Hub || !strings.Contains(m.Described(), "the hub") || len(m.Name) != len("anchor") {
t.Fatalf("the anchor reads as %+v", m)
}
if !m.Declaration.Composes || m.Declaration.Digest == "" || len(m.Declaration.Resources) == 0 {
t.Errorf("the anchor's declaration reads as %+v", m.Declaration)
}
laptop, _ := f.Machine(snapshot.Pseudonym("machine", "laptop"))
if strings.Join(laptop.Assigned, ",") != "files,mesh-wireguard" && !strings.Contains(strings.Join(laptop.Assigned, ","), "files") {
t.Errorf("the laptop's assignments read as %v", laptop.Assigned)
}
var meshWide map[string]any
for _, s := range f.Settings {
if s.Module == "files" {
meshWide = s.Values
}
}
if meshWide["admin_password"] != snapshot.Withheld || meshWide["hub"] != anchor {
t.Errorf("the mesh-wide settings read as %v", meshWide)
}
if f.Versions.Bus != "2.11.17" || f.Versions.Store == "" {
t.Errorf("versions read as %+v", f.Versions)
}
var objects bool
for _, mod := range f.Modules {
objects = objects || mod.Name == "objects" && len(mod.Manifest) > 0
}
if !objects {
t.Error("the modules the mesh holds are not in the snapshot")
}
// And an unchanged mesh is the same content a moment later.
again, err := gatherFacts(t.Context(), open, "2.11.17")
if err != nil {
t.Fatal(err)
}
a, _ := f.Content()
b, _ := again.Content()
if a != b {
t.Error("two snapshots of an unchanged mesh differ, so it would be written again every ten minutes")
}
}
cmd/mesh-controller/gate.go
-942
View File
@@ -1,942 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/micro"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0236, to-be 45
// §8, ADR 0227 rule 8).
//
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
// the component's health — the core's health definitions, or a module's own — passing three times over
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
//
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
// build goes back to the one the first machine ran before, and that machine is sent it: the ordinary
// path, again. Once per build: the verdict is written before the send, and a verdict once written is
// not written over. Said as a condition, urgent for the core and when it could not be put back, and as
// the event `rolled-back`.
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
var (
// gateSettle is how long the first machine must stay healthy, at the least.
gateSettle = 2 * time.Minute
// gateBound is how long after the apply the build may take to become healthy.
gateBound = 10 * time.Minute
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
gatePasses = 3
gateEvery = 40 * time.Second
)
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
const gateProbe = "DG"
// The kinds a gate raises.
const (
kindRolledBack = "rolled-back"
kindRollbackFailed = "rollback-failed"
)
// coreComponent is the core component a module is, as a witness names it — controller, node-engine,
// node-tools — or empty: the core is judged by its health definitions, anything else by its own health.
func coreComponent(module string) string {
switch module {
case catalogue.ControllerSeatName:
return lease.ComponentController
case hostModule:
return lease.ComponentEngine
case broker.RuntimeModule:
return lease.ComponentNodeTools
}
return ""
}
// health is a judging's word on one machine.
type health int
const (
healthGood health = iota
healthNotYet
healthBroken
)
// served is what one machine's node tools answered the bus's discovery with.
type served struct {
runtime bool
tools map[string]bool
}
// gateFacts is what one judging reads, gathered once for every machine it judges.
type gateFacts struct {
now time.Time
reports map[string]inventory.Reported
// engines is each machine's node-engine build as it last reported it.
engines map[string]string
// served is what the bus's discovery answered; servedErr why it could not be asked.
served map[string]served
servedErr error
// open are the open conditions; openErr why they could not be read (nil keeper: not judged).
open []conditions.Condition
openErr error
judged bool
// rolledBack is what each machine's witnesses say they decided.
rolledBack map[string][]lease.Rollback
// holder is who holds the controller lease, for judging the controller.
holder *lease.Holder
holderErr error
}
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
// variable so a test can hand a judging its facts.
var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
inv := open.inventory
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: witnessed.all()}
reports, err := inv.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
f.reports[r.Node] = r
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return f, err
}
for _, n := range nodes {
f.engines[n.Name] = n.HostVersion
}
if d := doctorFrom; d != nil {
if d.keeper != nil {
f.judged = true
f.open, f.openErr = d.keeper.Open(ctx)
}
if d.js != nil {
f.served, f.servedErr = servedOnTheBus(ctx, d.js.Conn())
} else {
f.servedErr = errors.New("this controller has no bus to ask")
}
} else {
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
}
if theLease != nil {
h, found, err := theLease.holder(ctx)
switch {
case err != nil:
f.holderErr = err
case found:
f.holder = &h
}
if component != lease.ComponentController && f.holderErr != nil {
f.holderErr = nil // read only for the controller's own judging
}
}
return f, nil
}
// judgeHealth is one machine's health for a module's new build, from what one judging read: healthy,
// not yet (with what is wanting), or healthBroken — a witness put it back, or the machine refused or failed
// what it was sent. Pure.
func judgeHealth(module, component string, m catalogue.Manifest, machine string, since time.Time, f gateFacts) (health, string) {
// A witness's verdict made since the build was sent: the build failed its health there. One that
// could not judge at all (unwitnessed) is not a verdict on the build.
for _, r := range f.rolledBack[machine] {
if component == "" || r.Component != component || r.Outcome == lease.OutcomeUnwitnessed || r.At.Before(since) {
continue
}
return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component,
short(r.From), r.Outcome, r.Why)
}
r, said := f.reports[machine]
switch {
case !said || r.At == nil || !r.Current:
return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine)
case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused:
return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome)
case r.Outcome != inventory.OutcomeApplied:
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
}
// **No new condition about it**: about the machine itself, or naming the module on that machine,
// raised since the judging began. The gate's own are not evidence about the build.
if f.judged {
if f.openErr != nil {
return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " +
firstLine(f.openErr.Error())
}
for _, c := range f.open {
if c.Source == gateProbe || c.Raised.Before(since) {
continue
}
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
(c.Subject.Scope == conditions.ScopeMachine && c.Subject.ID == machine)
if !onIt {
continue
}
if c.Subject.Scope == conditions.ScopeMachine || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
}
}
}
switch component {
case lease.ComponentEngine:
// The node-engine has reported its current declaration under its own build.
want := deliveredVersions(m)
if len(want) > 0 && !slices.Contains(want, f.engines[machine]) {
return healthNotYet, fmt.Sprintf("%s's node-engine reports build %s, not the new %s", machine,
orNotKnown(f.engines[machine]), strings.Join(want, " or "))
}
case lease.ComponentNodeTools:
// The node tools are announced and answer.
if f.servedErr != nil {
return healthNotYet, "whether the node tools answer cannot be asked: " + firstLine(f.servedErr.Error())
}
if !f.served[machine].runtime {
return healthNotYet, fmt.Sprintf("the node tools on %s do not answer the bus", machine)
}
case lease.ComponentController:
// The new controller holds the lease and says it is ready.
switch {
case f.holderErr != nil:
return healthNotYet, "who holds the controller lease cannot be read: " + firstLine(f.holderErr.Error())
case f.holder == nil:
return healthNotYet, "no controller holds the lease"
case f.holder.Taken.Before(since.Add(-time.Minute)):
return healthNotYet, fmt.Sprintf("the lease is held since %s, by a controller older than the new build",
f.holder.Taken.UTC().Format(time.RFC3339))
case f.holder.Health == nil || !f.holder.Health.Ready:
why := "the controller holding the lease does not say it is ready"
if f.holder.Health != nil && f.holder.Health.Why != "" {
why += ": " + f.holder.Health.Why
}
return healthNotYet, why
}
default:
// A module's tools answer, where it has any and the machine runs the node tools that serve them.
if len(m.Tools) > 0 {
if f.servedErr != nil {
return healthNotYet, "whether its tools are served cannot be asked: " + firstLine(f.servedErr.Error())
}
if s := f.served[machine]; s.runtime && !s.tools[module] {
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
}
}
}
return healthGood, ""
}
// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is
// the judging's facts without those conditions, on what each names among the modules the send moved,
// and whole what holds the machine back as a whole.
type machineWord struct {
facts gateFacts
on map[string]string
whole string
}
// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
// yet reported applying the node tools it was last sent.
const kindCoreBehind = "core-behind"
// aboutTheMachine sorts the conditions raised about a machine itself since a send was made there
// (novox/hq issue 281). The gate read every one of them as the module's it was kept on: a machine-level
// condition caused by anything else in the send — or by a tier sent one module at a time — failed that
// module, at the bound, with a reason that was never about it.
//
// - one naming a module the send moved is that module's;
// - the core being behind (D10) is the core's: of the node-engine or the node tools when the send
// moved them — inside their settle window, which is the gate's bound — and otherwise no evidence about
// what was sent: a send not yet applied the machine's reports already say, and a newer core build
// that no plan sends is not this send's;
// - anything else holds the machine back as a whole: everything the send moved there waits on it, and
// fails with it, together, at the bound, with that reason.
//
// Pure.
func aboutTheMachine(machine string, moved []string, since time.Time, f gateFacts) machineWord {
w := machineWord{facts: f, on: map[string]string{}}
if !f.judged || f.openErr != nil {
return w
}
var core []string
for _, m := range moved {
if c := coreComponent(m); c == lease.ComponentEngine || c == lease.ComponentNodeTools {
core = append(core, m)
}
}
kept := make([]conditions.Condition, 0, len(f.open))
for _, c := range f.open {
aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine ||
slices.Contains(c.Subject.Also, machine))
if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) {
kept = append(kept, c)
continue
}
said := fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
parts := strings.Split(c.Subject.ID, ".")
var named []string
for _, m := range moved {
if slices.Contains(parts, m) {
named = append(named, m)
}
}
coreBehind := c.Kind == kindCoreBehind || strings.HasSuffix(c.Key, "."+kindCoreBehind)
switch {
case len(named) > 0:
for _, m := range named {
if _, already := w.on[m]; !already {
w.on[m] = said
}
}
case coreBehind && len(core) > 0:
for _, m := range core {
if _, already := w.on[m]; !already {
w.on[m] = said + " (its settle window runs to the gate's bound)"
}
}
case coreBehind:
// Not what the send moved: said nowhere against it.
default:
if w.whole == "" {
w.whole = machine + " as a whole: " + said
}
}
}
w.facts.open = kept
return w
}
// judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass
// counted, a pass missed (and why), or the verdict. Answers the verdict once there is one.
func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
running []string, now time.Time) (string, error) {
g := state.Gate
if g == nil {
// Judged from the send: a witness's verdict, a condition, the bound — all counted from when the
// first machine was sent the build.
start := now
if state.FirstAt != nil {
start = *state.FirstAt
}
var machines []string
for _, n := range state.First {
if slices.Contains(running, n) {
machines = append(machines, n)
}
}
g = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, From: state.Previous,
To: state.Commit, Since: &start}
state.Gate = g
}
pairs := []judged{}
for _, n := range g.Machines {
pairs = append(pairs, judged{module: module, node: n})
}
return judgeMoves(ctx, open, g, pairs, now)
}
// judged is one module on one machine, as a gate judges it.
type judged struct{ module, node string }
// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and
// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is
// wanting, and which modules), or the verdict. Answers the verdict once there is one.
func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) {
if g.Verdict != "" {
return g.Verdict, nil
}
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
return "", nil
}
for _, c := range g.Carried {
if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) {
pairs = append(pairs, judged{module: c.Module, node: c.Node})
}
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
}
facts, err := gatherGateFacts(ctx, open, g.Component)
if err != nil {
return "", err
}
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
// gate happens to be kept on.
byMachine := map[string][]string{}
for _, j := range pairs {
byMachine[j.node] = append(byMachine[j.node], j.module)
}
words := map[string]machineWord{}
for node, moved := range byMachine {
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
}
worst, why := healthGood, ""
var failing []string
broken := map[string]bool{}
for _, j := range pairs {
w := words[j.node]
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
if h == healthGood {
if on, named := w.on[j.module]; named {
h, said = healthNotYet, on
} else if w.whole != "" {
h, said = healthNotYet, w.whole
}
}
if h != healthGood && !slices.Contains(failing, j.module) {
failing = append(failing, j.module)
}
if h == healthBroken {
broken[j.module] = true
}
if h > worst {
worst, why = h, said
} else if h == worst && h != healthGood && why == "" {
why = said
}
}
switch {
case worst == healthBroken:
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
g.Failing = failing
decide(g, inventory.GateFailed, why, now)
case worst == healthNotYet:
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
if now.Sub(*g.Since) > gateBound {
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
}
default:
g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second)), now)
}
}
return g.Verdict, nil
}
// decide sets a gate's verdict.
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
g.Took = now.Sub(*g.Since).Round(time.Second).String()
}
// gatePassed keeps a passing build's verdict, so `plans` and the gate's probe can read it.
func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule) {
g := state.Gate
err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
if err != nil && state.Build != "" {
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
}
passCarried(ctx, open, p, g, module)
fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module,
strings.Join(g.Machines, ", "), g.Why)
if module == catalogue.ControllerSeatName {
carryUserList(ctx, open, p)
}
}
// carryUserList sends the machine holding the bus the user list a new controller composes, once that
// controller passed its gate (ADR 0236). The controller's own grants travel in that list, and the old
// controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new
// controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR
// 0221): then it is said, as a push would say it.
func carryUserList(ctx context.Context, open *stores, p *inventory.Plan) {
holder, behind, err := brokerBehind(ctx, open, nil)
if err != nil || holder == "" || !behind {
if err != nil {
fmt.Printf("%s: whether the bus's user list is behind the new controller cannot be read: %v\n", p.ID, err)
}
return
}
held, err := heldMachines(ctx, open, []string{holder})
if err != nil {
fmt.Printf("%s: whether %s may be sent the new user list cannot be read: %v\n", p.ID, holder, err)
return
}
if why, isHeld := held[holder]; isHeld {
fmt.Printf("%s: the new controller's user list is not carried to %s, which holds the bus: %s — `push %s` "+
"carries it\n", p.ID, holder, strings.Join(why, "; "), holder)
return
}
if _, err := sendRollout(ctx, open, []string{holder}); err != nil {
fmt.Printf("%s: the new controller's user list could not be carried to %s: %v\n", p.ID, holder, err)
return
}
fmt.Printf("%s: carried the new controller's user list to %s, which holds the bus\n", p.ID, holder)
}
// gateFailed stops the plan at a build that failed its gate and puts the previous build back on the
// machines it was judged on — once per build, said as a condition and an event. The plan is saved
// before the send: a controller that is itself the build being put back does not outlive it.
func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
machines []string, why string) {
inv := open.inventory
now := time.Now().UTC()
if state.Gate == nil {
state.Gate = &inventory.PlanGate{Component: coreComponent(module), Machines: machines,
From: state.Previous, To: state.Commit, Since: state.FirstAt}
}
g := state.Gate
if g.Verdict == "" {
if g.Since == nil {
g.Since = &now
}
decide(g, inventory.GateFailed, why, now)
}
if len(g.Machines) == 0 {
g.Machines = machines
}
state.Why = "failed its gate: " + g.Why
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s failed its gate on %s in tier %d: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why)
verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous,
Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why,
Component: g.Component, JudgingFrom: g.Since}
// **A send that changed nothing of the module there is no verdict on its build** (novox/hq issue
// 280). The machine already ran this build, or one that made the same artifacts from the same
// manifest: whatever the gate found wanting, this build did not bring it, and there is nothing to
// put back. On 2026-10-06 such a module was marked failed, and the rollback looked for an earlier
// build of the very commit it had failed — "no build kept" — while the build it had run before was
// kept all along. Said, left as it is, never marked.
if unchangedBy(ctx, inv, module, state.Previous, state.Commit) {
g.Rollback = gateUnchanged
state.Why = "stopped with its send; the send changed nothing of it: " + g.Why
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was left as it was — %s already ran "+
"%s %s, or a build identical to it, before the send, so nothing of it moved and nothing is put back",
strings.Join(g.Machines, ", "), p.Tier, g.Why, module, strings.Join(g.Machines, ", "), module, short(state.Commit))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return
}
if state.Build == "" {
// A plan from before builds were asked by id: nothing to mark, so nothing is put back by the
// mesh — said, for a person.
g.Rollback = inventory.NotRolledBack
p.Note += "; not put back: the plan does not know which build it sent"
sayRollback(ctx, open, module, g, "")
return
}
if err := inv.RecordGate(ctx, verdict); err != nil {
if errors.Is(err, inventory.ErrGateKept) {
// Already judged and acted on, by this controller before a restart or by another: never twice.
if kept, found, _ := inv.GateOf(ctx, state.Build); found {
g.Rollback = kept.Rollback
}
p.Note += "; its rollback was already made once and is not made again"
return
}
g.Rollback = inventory.NotRolledBack
p.Note += "; not put back: its verdict could not be kept, and a rollback that cannot be counted is not made — " + err.Error()
sayRollback(ctx, open, module, g, "")
return
}
// The previous build: the one the first machine ran, from the build records.
notBack := func(why string) {
g.Rollback = inventory.NotRolledBack
p.Note += "; NOT put back: " + why
if err := inv.SetRollback(ctx, state.Build, inventory.NotRolledBack, g.Why+"; not put back: "+why); err != nil {
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
}
sayRollback(ctx, open, module, g, why)
}
if state.Previous == "" {
// A first build there: nothing ran before it, so nothing can be put back, and the machine is left
// with it — said as that, not as a build the mesh lost.
notBack(fmt.Sprintf("this is the first build of %s that %s was sent, or what it was sent before is not known: "+
"there is no earlier build there to put back, so it is left with this one — `unassign` takes it off, "+
"a newer merge replaces it", module, strings.Join(g.Machines, ", ")))
return
}
failed, _, err := inv.BuildByID(ctx, state.Build)
if err != nil {
notBack("the failed build's record cannot be read: " + err.Error())
return
}
previous, found, err := inv.PreviousBuild(ctx, module, state.Previous, failed)
if err != nil {
notBack("the build records cannot be read: " + err.Error())
return
}
if !found {
notBack(fmt.Sprintf("no build of %s from %s, asked before the failed one, is among the %d newest kept to put "+
"back", module, short(state.Previous), inventory.KeptBuilds))
return
}
if err := inv.RestoreModule(ctx, previous); err != nil {
notBack(err.Error())
return
}
g.Rollback = inventory.RollingBack
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err)
}
// Put back with the rest of its send, in one send per machine (issue 281).
if b, batched := ctx.Value(rollbacksKey{}).(*rollbacks); batched {
b.pending = append(b.pending, pendingRollback{module: module, state: state, g: g, previous: previous})
b.modules[module] = true
for _, n := range g.Machines {
if !slices.Contains(b.machines, n) {
b.machines = append(b.machines, n)
}
}
return
}
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
if err != nil {
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
return
}
g.Rollback = inventory.RolledBack
p.Note += fmt.Sprintf("; put back to %s on %s", short(previous.Commit), strings.Join(sent, ", "))
if err := inv.SetRollback(ctx, state.Build, inventory.RolledBack, g.Why); err != nil {
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
}
sayRollback(ctx, open, module, g, "")
}
// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one
// send judges what it moved as one, and what it found wanting goes back in one send per machine — not
// in a send for each module, which is the churn that failed the gate in the first place.
type rollbacks struct {
modules map[string]bool
machines []string
pending []pendingRollback
}
type pendingRollback struct {
module string
state *inventory.PlanModule
g *inventory.PlanGate
previous inventory.Build
}
type rollbacksKey struct{}
// batchingRollbacks is a context under which gateFailed registers what it puts back and leaves the send
// to sendRollbacks.
func batchingRollbacks(ctx context.Context) (context.Context, *rollbacks) {
b := &rollbacks{modules: map[string]bool{}}
return context.WithValue(ctx, rollbacksKey{}, b), b
}
// sendRollbacks sends what a failed send put back, once to each machine, and says each module's rollback.
func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *rollbacks) {
if len(b.pending) == 0 {
return
}
inv := open.inventory
sort.Strings(b.machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
var back []string
for _, r := range b.pending {
if err != nil {
why := fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
"sends it", short(r.previous.Commit), strings.Join(r.g.Machines, ", "), err, firstOf(r.g.Machines))
r.g.Rollback = inventory.NotRolledBack
if err := inv.SetRollback(ctx, r.state.Build, inventory.NotRolledBack, r.g.Why+"; not put back: "+why); err != nil {
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
}
sayRollback(ctx, open, r.module, r.g, why)
continue
}
r.g.Rollback = inventory.RolledBack
back = append(back, r.module+" to "+short(r.previous.Commit))
if err := inv.SetRollback(ctx, r.state.Build, inventory.RolledBack, r.g.Why); err != nil {
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
}
sayRollback(ctx, open, r.module, r.g, "")
}
if err != nil {
p.Note += fmt.Sprintf("; NOT put back: sending %s was refused: %v", strings.Join(b.machines, ", "), err)
return
}
p.Note += fmt.Sprintf("; put back %s on %s, in one send", strings.Join(back, ", "), strings.Join(sent, ", "))
}
// gateUnchanged is a failed gate's word on a module its send changed nothing of (novox/hq issue 281):
// left as it was, never marked failed, and so no verdict on its build — `plans retry` asks it again.
const gateUnchanged = "unchanged"
// unchangedBy says whether a send moving a module from one build to another changed nothing of it: the
// same commit, builds made from the same source, or builds that made the same artifacts from the same
// manifest. Not known is changed.
func unchangedBy(ctx context.Context, inv *inventory.Inventory, module, from, to string) bool {
if from == "" || to == "" {
return false
}
if sameCommit(from, to) {
return true
}
// Made from the same source (issue 280), or the same artifacts from the same manifest.
f := moveFacts{}
var err error
if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
return false
}
if f.fps, err = inv.Fingerprints(ctx); err != nil {
return false
}
return f.identical(module, from, to)
}
// rolledBackEvent is the body of `rolled-back` (ADR 0236): a contract, like a condition's events.
type rolledBackEvent struct {
Event string `json:"event"`
At time.Time `json:"at"`
Module string `json:"module"`
Component string `json:"component,omitempty"`
Machines []string `json:"machines"`
From string `json:"from,omitempty"`
To string `json:"to,omitempty"`
Why string `json:"why"`
// Rollback is rolled-back, or not-rolled-back with NotWhy.
Rollback string `json:"rollback"`
NotWhy string `json:"not_why,omitempty"`
Show string `json:"show"`
}
// sayRollback raises the gate's condition at once — the probe keeps it from then — and says the event.
func sayRollback(ctx context.Context, open *stores, module string, g *inventory.PlanGate, notWhy string) {
o := gateObservation(module, g.Component, g.Machines, g.Rollback, g.Why, notWhy, g.To, g.From)
fmt.Println(o.Summary)
d := doctorFrom
if d == nil {
return
}
if d.keeper != nil {
o.Source = gateProbe
if _, err := d.keeper.Observe(ctx, o); err != nil {
fmt.Printf("the gate's condition %s could not be kept: %v\n", o.Key(), err)
}
}
if d.teller == nil {
return
}
body, err := json.Marshal(rolledBackEvent{Event: link.KeyRolledBack, At: time.Now().UTC(), Module: module,
Component: g.Component, Machines: g.Machines, From: g.To, To: g.From, Why: g.Why, Rollback: g.Rollback,
NotWhy: notWhy, Show: conditions.Condition{Key: o.Key()}.Show()})
if err != nil {
return
}
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyRolledBack, body); err != nil {
fmt.Printf("%s's rollback could NOT be said on the bus: %v\n", module, err)
}
}
// gateObservation is a failed gate as a condition: `core.<component>.<machine>.rolled-back` for the
// core (to-be 45 §2), `build.<module>.<machine>.rolled-back` for any other module; `rollback-failed`
// when it could not be put back. Urgent for the core and for a build left in place; a warning for a
// module put back, which runs what it ran before.
func gateObservation(module, component string, machines []string, rollback, why, notWhy, failed, previous string) conditions.Observation {
machine := strings.Join(machines, ",")
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: module + "." + machine, Kind: kindRolledBack,
Token: kindRolledBack, Machine: firstOf(machines), Severity: conditions.Warning, Source: gateProbe,
Summary: fmt.Sprintf("%s's build %s failed its gate on %s and was put back to %s: %s", module, short(failed),
machine, short(previous), why)}
if component != "" {
o.Scope, o.ID, o.Severity = conditions.ScopeCore, component+"."+machine, conditions.Urgent
}
if len(machines) > 1 {
o.Also = machines[1:]
}
if rollback != inventory.RolledBack && rollback != inventory.RollingBack {
o.Kind, o.Token, o.Severity = kindRollbackFailed, kindRollbackFailed, conditions.Urgent
o.Resolver = conditions.ResolverOperator
o.Summary = fmt.Sprintf("%s's build %s failed its gate on %s and was NOT put back: %s — %s", module, short(failed),
machine, why, orNone(notWhy))
}
return o
}
// probeGates is DG: no build that failed its gate is left without its condition, and no witness's
// rollback goes unsaid. Each module whose newest verdict is a failure keeps its condition; a newer build
// that passes clears it. Each core component a machine's witness says it put back is `core.<component>.
// <machine>.rolled-back`, urgent, until the machine's reports stop saying it.
func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
latest, err := d.open.inventory.LatestGates(ctx)
if err != nil {
return nil, err
}
var out []conditions.Observation
for _, v := range latest {
if v.Verdict != inventory.GateFailed {
continue
}
notWhy := ""
if v.Rollback == inventory.NotRolledBack {
notWhy = v.Why
}
out = append(out, gateObservation(v.Module, v.Component, v.Machines, v.Rollback, v.Why, notWhy, v.Commit, v.Previous))
}
for node, list := range witnessed.all() {
for _, r := range list {
out = append(out, witnessObservation(node, r))
}
}
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
out = append(out, backlogObservation()...)
return sortedFound(dedupeObservations(out)), nil
}
// dedupeObservations keeps one observation per key, the first.
func dedupeObservations(list []conditions.Observation) []conditions.Observation {
seen := map[string]bool{}
var out []conditions.Observation
for _, o := range list {
if seen[o.Key()] {
continue
}
seen[o.Key()] = true
out = append(out, o)
}
return out
}
// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules'
// tools are served where. A variable so a test needs no runtime.
var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) {
inbox := conn.NewRespInbox()
sub, err := conn.SubscribeSync(inbox)
if err != nil {
return nil, err
}
defer func() { _ = sub.Unsubscribe() }()
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
}
out := map[string]served{}
add := func(node string) served {
s, ok := out[node]
if !ok {
s = served{tools: map[string]bool{}}
}
return s
}
deadline := time.Now().Add(discoveryPatience)
for time.Now().Before(deadline) {
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
msg, err := sub.NextMsgWithContext(wait)
cancel()
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
break
}
var info micro.Info
if json.Unmarshal(msg.Data, &info) != nil {
continue
}
if info.Name == broker.RuntimeModule {
node := info.Metadata["node"]
if node == "" {
node = info.ID
}
s := add(node)
s.runtime = true
out[node] = s
}
for _, e := range info.Endpoints {
if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" {
continue
}
node := e.Metadata["node"]
if node == "" {
node = info.ID
}
s := add(node)
s.tools[e.Metadata["module"]] = true
out[node] = s
}
}
return out, nil
}
// gateLines is a plan's gates as `plans <id>` says them: the rollout's record.
func gateLine(g *inventory.PlanGate) string {
if g == nil {
return ""
}
what := "judging"
switch g.Verdict {
case inventory.GatePassed:
what = "passed"
case inventory.GateFailed:
what = "FAILED"
}
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
if g.Component != "" {
line += " (core: " + g.Component + ")"
}
if g.From != "" || g.To != "" {
line += fmt.Sprintf(", %s → %s", short(orNone(g.From)), short(g.To))
}
if g.Took != "" {
line += ", after " + g.Took
}
if g.Why != "" {
line += ": " + g.Why
} else if g.Last != "" {
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
} else if g.Verdict == "" {
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
}
if g.Rollback != "" {
line += "; " + g.Rollback
}
return line
}
// witnessObservation is a witness's verdict as a condition (ADR 0236, the host's contract):
// `core.<component>.<node>.<outcome>`, urgent for rolled-back, not-reversible, restore-failed and
// halted, a warning for nothing-to-restore and unwitnessed.
func witnessObservation(node string, r lease.Rollback) conditions.Observation {
severity := conditions.Warning
if lease.Urgent(r.Outcome) {
severity = conditions.Urgent
}
outcome := r.Outcome
if outcome == "" {
outcome = lease.OutcomeRolledBack
}
what := map[string]string{
lease.OutcomeRolledBack: "and put back " + short(orNone(r.To)),
lease.OutcomeNotReversible: "and left it: it is not reversible",
lease.OutcomeNothingToRestore: "and had nothing to put back",
lease.OutcomeRestoreFailed: "and could not put the previous build back",
lease.OutcomeUnwitnessed: "and could not judge it at all",
lease.OutcomeHalted: "and gave up: the build before it fails too",
}[outcome]
return conditions.Observation{Scope: conditions.ScopeCore, ID: r.Component + "." + node, Kind: kindRolledBack,
Token: outcome, Machine: node, Severity: severity,
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
}
cmd/mesh-controller/gate_test.go
-551
View File
@@ -1,551 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).
// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
// sent and reports it — with the gate's bounds shortened to judge in three steps.
type gateMesh struct {
open *stores
sent [][]string
health map[string]health // per machine, what a judging finds; healthy when unsaid
keeper *conditions.Keeper
told *conditions.Told
}
func aGateMesh(t *testing.T) *gateMesh {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
g := &gateMesh{open: open, health: map[string]health{}}
g.keeper, _ = withConditionsInMemory(t)
g.told = &conditions.Told{}
was := doctorFrom
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
t.Cleanup(func() { doctorFrom = was })
for _, b := range []inventory.Build{
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
} {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
b.Manifest = manifest
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
register := func(commit string, asked time.Time) {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
Head: commit, Asked: asked}); err != nil {
t.Fatal(err)
}
}
register("c1", time.Now().Add(-2*time.Hour))
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, n, "app"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
}
register("c2", time.Now().Add(-time.Minute))
// Every send: recorded with the build the module is at, applied and reported by the machine.
n := 0
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
g.sent = append(g.sent, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
// What a judging reads: the store's reports, and a health the test says per machine.
wasGather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
if g.health[r.Node] == healthBroken {
r.Outcome = inventory.OutcomeFailed
}
f.reports[r.Node] = r
}
for node, h := range g.health {
if h == healthNotYet {
f.rolledBack[node] = nil
r := f.reports[node]
r.Current = false
f.reports[node] = r
}
}
return f, nil
}
t.Cleanup(func() { gatherGateFacts = wasGather })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
built := time.Now().UTC()
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
Build: "build-2"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
return g
}
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
t.Helper()
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
if err != nil {
t.Fatal(err)
}
return p
}
// A module's build that fails its gate on the first machine is put back there — the previous build
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
// condition and an event, never registered or rolled back again.
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
advancePlans(ctx, g.open) // the first machine is sent the new build
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v, not the first machine alone", g.sent)
}
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
}
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
gateEvery = 0
advancePlans(ctx, g.open)
p := g.plan(t)
gate := p.Modules["app"].Gate
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
gate.Rollback != inventory.RolledBack {
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
}
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
}
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
t.Fatalf("anchor was last sent %v, not the previous build", sent)
}
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
t.Fatalf("laptop was sent the failed build: %v", sent)
}
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
}
// Said: a condition for the operator, and the event.
open, err := g.keeper.Open(ctx)
if err != nil {
t.Fatal(err)
}
var key string
for _, c := range open {
if c.Kind == kindRolledBack {
key = c.Key
}
}
if key != "build.app.anchor.rolled-back" {
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
}
saidIt := false
for _, e := range g.told.Said() {
saidIt = saidIt || e.Event == link.KeyRolledBack
}
if !saidIt {
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
}
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
// heard again is not registered.
advancePlans(ctx, g.open)
state := p.Modules["app"]
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
if len(g.sent) != 2 {
t.Fatalf("sent again after the rollback: %v", g.sent)
}
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"})})
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
t.Fatalf("the failed build was registered again: %v", err)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("the module moved to %s", current["app"].Commit)
}
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
}
// The probe keeps the condition while the newest verdict is the failure.
obs, err := probeGates(ctx, doctorFrom)
if err != nil || len(obs) != 1 || obs[0].Key() != key {
t.Fatalf("the gate's probe found %+v %v", obs, err)
}
}
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
// the rest are sent, the verdict kept, and the plan done.
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
gateEvery = 0
for i := 0; i < 6; i++ {
advancePlans(ctx, g.open)
}
p := g.plan(t)
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("sent %v", g.sent)
}
gate := p.Modules["app"].Gate
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
}
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
}
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
}
}
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
advancePlans(ctx, g.open)
g.health["anchor"] = healthNotYet
gateEvery = 0
advancePlans(ctx, g.open)
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
}
gateBound = -time.Second
advancePlans(ctx, g.open)
p := g.plan(t)
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
}
}
// The health a judging finds, per core component and for a module.
func TestTheHealthDefinitions(t *testing.T) {
now := time.Now()
since := now.Add(-time.Minute)
applied := func(node string) gateFacts {
at := now
return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
}
m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}
f := applied("anchor")
f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
t.Errorf("a module whose tools are not served is %v (%s)", h, why)
}
f.served["anchor"].tools["app"] = true
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
t.Errorf("a module applied with its tools served is %v (%s)", h, why)
}
// A condition raised about it on that machine since it was sent: not yet healthy.
f.judged = true
f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
t.Errorf("a module with a new condition about it is %v", h)
}
f.open[0].Raised = since.Add(-time.Hour)
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
t.Errorf("a condition older than the send counted against it: %v", h)
}
// A witness that put it back: broken.
f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
t.Errorf("a component a witness put back is %v", h)
}
// The node tools answer, or not.
f = applied("anchor")
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
t.Errorf("node tools not answering are %v", h)
}
f.served["anchor"] = served{runtime: true}
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
t.Errorf("node tools answering are %v", h)
}
// The controller: the lease held since the send, by a controller that says it is ready.
f = applied("control")
f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
t.Errorf("a lease held by a controller older than the build is %v", h)
}
f.holder.Taken = now
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
t.Errorf("a new controller holding the lease and ready is %v", h)
}
f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
!strings.Contains(why, "first run") {
t.Errorf("a controller not ready is %v (%s)", h, why)
}
// A machine that refused what it was sent: broken.
f = applied("anchor")
r := f.reports["anchor"]
r.Outcome = inventory.OutcomeRefused
f.reports["anchor"] = r
if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
t.Errorf("a refusal is %v", h)
}
}
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
t.Fatal(err)
}
u, err := inv.UpgradeOf(ctx, "nats")
if err != nil || u.RollOut || u.From != catalogue.FromBus {
t.Fatalf("the bus's policy is %+v %v", u, err)
}
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
t.Fatalf("a person rolled the bus out: %v", err)
}
var sent [][]string
was := sendRollout
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
sent = append(sent, names)
return names, nil
}
t.Cleanup(func() { sendRollout = was })
now := time.Now().UTC()
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
advancePlans(ctx, open)
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
}
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
}
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
t.Fatalf("a send to the bus's machine was not refused: %v", err)
}
// A rebuild that made the same artifacts is no move.
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
b.Asked, b.At = time.Now(), time.Now()
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
Reference: "registry/nats@sha256:new"}}}); err != nil {
t.Fatal(err)
}
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
// first by the bus machine's backup holder.
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
}
wasSnapshot := takeBusSnapshot
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
}
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
return node + "'s restore point of " + module, nil
}
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
t.Fatalf("the step sent %v, not the bus's machine", sent)
}
s, found, err := inv.LatestBusStep(ctx)
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
t.Fatalf("the step is %+v %v %v", s, found, err)
}
}
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := asksRecorded(t)
for _, name := range []string{"gone", "kept"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
}
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
if len(*asked) != 0 {
t.Fatalf("a deleted module was asked to build: %v", *asked)
}
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
if _, still := shelf["gone"]; still {
t.Fatal("a deleted module nothing holds was not forgotten")
}
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
asked0 := time.Now().UTC().Add(-time.Minute)
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
p, _ := inv.PlanByID(ctx, "plan-deleted")
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
}
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
b, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return b
}
func nodeID(t *testing.T, open *stores, name string) string {
t.Helper()
n, err := open.inventory.NodeByName(context.Background(), name)
if err != nil {
t.Fatal(err)
}
return n.ID
}
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
// gone with the first report that no longer carries it.
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
open := aMesh(t)
d := &doctor{open: open}
at := time.Now().UTC()
witnessed.heard("control", []lease.Rollback{
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
Why: "the new controller did not take the lease within 60s", At: at},
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
}, at)
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
obs, err := probeGates(t.Context(), d)
if err != nil {
t.Fatal(err)
}
got := map[string]conditions.Severity{}
for _, o := range obs {
got[o.Key()] = o.Severity
}
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
"core.node-tools.control.unwitnessed": conditions.Warning}
if !reflect.DeepEqual(got, want) {
t.Fatalf("the witness's verdicts are %v", got)
}
witnessed.heard("control", nil, time.Now())
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
}
}
cmd/mesh-controller/given.go
-90
View File
@@ -1,90 +0,0 @@
package main
import (
"context"
"encoding/json"
"log"
"strings"
"sync"
"time"
"github.com/novox/mesh-controller/internal/link"
)
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
//
// The serving controller hears every report; a clean one about the declaration a machine was last
// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is
// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
// so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log
// is read as the count of repairs a healer is wanted for.
// SecretReplaced is the controller seat's fact that a value given by hand was replaced
// (link.KeySecretReplaced). Never the value: neither the old one, which the mesh cannot read,
// nor the new one, sealed to the machine as it was made.
type SecretReplaced struct {
Node string `json:"node"`
Module string `json:"module"`
Name string `json:"name"`
Given time.Time `json:"given"`
// Sent are the machines sent so the module starts again on the new value; Unsent says why
// they could not be, in which case the next push carries it.
Sent []string `json:"sent"`
Unsent string `json:"unsent,omitempty"`
Why string `json:"why"`
}
// givenEvents is where the serving controller states it; nil in a command.
var givenEvents link.Bus
// replacing keeps one replacement per machine at a time: two reports arriving together find the
// same rows, and the store's claim makes one of them the replacer, but the sends need not race.
var replacing sync.Map
// startedWell says a report is a machine's clean account of a declaration: everything applied,
// nothing failed or refused. Whether it is the declaration last sent is the store's to answer.
func startedWell(report link.Report) bool {
return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil
}
const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
// replaceGiven replaces what the report makes due, sends the machines, and says so.
func replaceGiven(ctx context.Context, open *stores, report link.Report) {
if _, busy := replacing.LoadOrStore(report.Node, true); busy {
return
}
defer replacing.Delete(report.Node)
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared)
if err != nil {
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err)
}
for _, r := range replaced {
said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
Sent: r.Machines, Why: givenWhy}
log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; sending %s",
r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
if err := sendTo(ctx, open, r.Machines); err != nil {
said.Sent, said.Unsent = nil, err.Error()
log.Printf("the new %q of %s is sealed and not yet delivered to %s — the next push carries it: %v",
r.Name, r.Module, strings.Join(r.Machines, ", "), err)
}
stateReplaced(ctx, said)
}
}
func stateReplaced(ctx context.Context, said SecretReplaced) {
if givenEvents == nil {
return
}
body, err := json.Marshal(said)
if err != nil {
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
return
}
stating, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := givenEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySecretReplaced, body); err != nil {
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
}
}
cmd/mesh-controller/given_test.go
-45
View File
@@ -1,45 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A rotation asked through the seat carries why to the command, which records it in the hand-act
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password --why leaked into logs --cause leaked" {
t.Fatalf("%v %v", argv, err)
}
argv, err = argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password"})
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
t.Fatalf("without why: %v %v", argv, err)
}
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
t.Fatalf("why beside a provision was passed over: %v", argv)
}
}
// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that
// the machine is there is not (novox/hq ADR 0228).
func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) {
for _, c := range []struct {
report link.Report
good bool
}{
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true},
{link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true},
{link.Report{Node: "anchor"}, false},
{link.Report{Node: "anchor", Applied: []string{"x"}}, false},
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false},
{link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false},
} {
if got := startedWell(c.report); got != c.good {
t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good)
}
}
}
cmd/mesh-controller/grant_follows_binding_test.go
-197
View File
@@ -1,197 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 274: a provider is granted exactly the consumers whose own resolution binds them to
// it — not every consumer a pair credential from it was ever made for.
// grantOf is the grant of one provision to one consuming module, and whether there is one at all.
func grantOf(grants []catalogue.Grant, provision, consumer, module string) (catalogue.Grant, bool) {
for _, g := range grants {
if g.Provision == provision && g.Consumer == consumer && (g.From == module || g.From == "") {
return g, true
}
}
return catalogue.Grant{}, false
}
// The morning after issue 273, through the stores: a consumer was bound to the store on another
// machine, a credential from there was made, and a person pinned it back to the store beside it. Both
// credentials are on record. The store it left is no longer granted it — so it retires it and keeps
// its data (ADR 0230) — and says so; the store it is bound to keeps its grant; and the credential from
// the store it left stays on record.
func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
for _, m := range storeManifests() {
register(t, open, m)
}
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
t.Fatal(err)
}
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
t.Fatal(err)
}
for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
// Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded.
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
t.Fatal(err)
}
plan, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
t.Fatalf("pinned to the anchor and bound to %s", n.From)
}
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
t.Fatal(err)
}
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound)
}
// Pinned back beside its data, as the operator did.
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil {
t.Fatal(err)
}
plan, _, err = planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" {
t.Fatalf("pinned back to the laptop and bound to %s", n.From)
}
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
t.Fatal(err)
}
holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop")
if err != nil {
t.Fatal(err)
}
if len(holders) != 2 {
t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders)
}
// The store it left: no longer granted, and said.
grants, _, unbound, err = grantsFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" {
t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g)
}
if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" ||
strings.Join(unbound[0].BoundTo, ",") != "laptop" {
t.Fatalf("the consumer that moved is not the one said: %+v", unbound)
}
planned, settings, err := planFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, "anchor", planned, settings)
if err != nil {
t.Fatal(err)
}
if got := declared.Received["store"]["postgres-database"]; len(got) != 0 {
t.Fatalf("the anchor's store is still told about %+v", got)
}
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") ||
!strings.Contains(said, "cleanup delete") {
t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said)
}
// The store it is bound to: granted.
grants, _, unbound, err = grantsFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound)
}
// And the credential from the store it left is kept: the key to the login and data held there.
if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 {
t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err)
}
}
// A consumer whose resolution cannot be read is an error, never a consumer bound nowhere — withdrawing
// a grant on that reading would take its access away (issue 152).
func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
for _, m := range storeManifests() {
register(t, open, m)
}
for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
t.Fatal(err)
}
// The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and
// that is not its set failing to compose.
laptop, err := inv.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil {
t.Fatal(err)
}
if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) {
t.Fatalf("the seam this test relies on moved: %v", err)
}
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
if err == nil {
t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound)
}
if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") {
t.Fatalf("the error does not say whose resolution could not be read: %v", err)
}
}
// The rule itself, on a resolution: bound is the provider a need for exactly that credential is
// answered by, never one answered by a record, and a different local name is a different credential.
func TestBindsFromIsTheProviderOfThatCredential(t *testing.T) {
r := catalogue.Resolution{Needs: []catalogue.Needed{
{Name: "postgres-database", For: "board", From: "laptop"},
{Name: "postgres-database", For: "board", From: "laptop", Local: "reports"},
{Name: "postgres-database", For: "wiki", From: "anchor"},
{Name: "a-licence", For: "board", From: "the-licence", ByRecord: true},
}}
s := inventory.Secret{Name: "postgres-database", ConsumerModule: "board"}
if got := r.BindsFrom(s.Name, s.ConsumerModule, s.Local); strings.Join(got, ",") != "laptop" {
t.Fatalf("board's credential is bound to %v", got)
}
if got := r.BindsFrom("postgres-database", "board", "archive"); len(got) != 0 {
t.Fatalf("a local name nothing asks for is bound to %v", got)
}
if got := r.BindsFrom("a-licence", "board", ""); len(got) != 0 {
t.Fatalf("a need answered by a record is bound to %v", got)
}
}
cmd/mesh-controller/handacts.go
-268
View File
@@ -1,268 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// Acts done by hand, and why (novox/hq to-be 45 §7).
//
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
// required, because the installer and the lab push by command line as a step of what they do, and a
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
// (Phase 1).
// handActVerb is one verb that writes the hand-act log, and whether what it records is a repair.
type handActVerb struct {
Verb string
// Decision says why an act of this verb is a person's decision by design rather than a repair a
// healer could take over; empty for a repair.
Decision string
// DecidedFor limits Decision to these causes; empty, it holds for every act of the verb.
DecidedFor []string
}
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
const causeLeakedInLogs = "leaked-in-logs"
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
var handActVerbs = []handActVerb{
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
// default (ADR 0236) exists to end.
{Verb: "push"},
{Verb: "plans stop"},
{Verb: "plans close"},
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
// not trusted with it — either is a repair the owner should have made.
{Verb: "plans go"},
{Verb: "broker consumer-reset"},
// Silencing the same condition twice says the condition, or what it watches, wants mending.
{Verb: "conditions silence"},
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts.
{Verb: "hand-act record"},
// A person's decisions by design.
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
"person starts (ADR 0236)"},
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
"person releases it (ADR 0236)"},
// A leak is judged by a person — which value was exposed, to whom — and its rotation is the answer
// to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the
// module that prints them, an issue against it, not a healer that rotates. A rotation for any other
// cause — a credential that stopped working — counts: a schedule or a healer could take it over.
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
DecidedFor: []string{causeLeakedInLogs}},
}
// personsDecision is whether an act in the log is a person's decision by design, by the verb that
// recorded it (handActVerbs).
func personsDecision(a link.HandAct) bool {
for _, v := range handActVerbs {
if v.Verb != a.Verb {
continue
}
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause))
}
return false
}
// handActFlags are the flags every repairing verb takes.
type handActFlags struct {
why, cause, condition *string
}
func addHandActFlags(set *flag.FlagSet) handActFlags {
return handActFlags{
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
}
}
// given is whether a reason was given.
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
// require refuses an act without a reason, before anything is done.
func (f handActFlags) require(verb string) error {
if f.given() {
return nil
}
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
"log, novox/hq to-be 45 §7). Nothing was done", verb)
}
// handActConn is the serving controller's connection, for what it reads of the log itself; a
// command dials its own.
var handActConn *nats.Conn
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
func onTheBus(f func(*nats.Conn) error) error {
if handActConn != nil {
return f(handActConn)
}
address, err := broker.BusAddress()
if err != nil {
return err
}
js, err := broker.Dial(address)
if err != nil {
return fmt.Errorf("cannot reach the bus: %w", err)
}
defer js.Close()
return f(js.Conn())
}
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
// whose bus is down is exactly the mesh somebody is repairing by hand.
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
if !f.given() {
return
}
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
err := onTheBus(func(conn *nats.Conn) error {
written, err := link.RecordHandAct(ctx, conn, act)
act = written
return err
})
if err != nil {
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
return
}
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
}
// handActCommand is `hand-act record` and `hand-acts`.
func handActCommand(ctx context.Context, args []string) error {
if len(args) > 0 && args[0] == "record" {
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
f := addHandActFlags(set)
positionals, err := parseAround(set, args[1:])
if err != nil {
return err
}
what := strings.TrimSpace(strings.Join(positionals, " "))
if what == "" {
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
}
if err := f.require("hand-act record"); err != nil {
return err
}
if strings.TrimSpace(*f.cause) == "" {
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
"act for the same reason will use — it is how a repair done twice is found")
}
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
return onTheBus(func(conn *nats.Conn) error {
written, err := link.RecordHandAct(ctx, conn, act)
if err != nil {
return fmt.Errorf("the act could not be recorded: %w", err)
}
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
written.Why, written.Cause)
return nil
})
}
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
}
if len(args) > 0 && args[0] == "list" {
args = args[1:]
}
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
days := set.Int("days", 14, "how many days back")
asJSON := set.Bool("json", false, "as data")
if _, err := parseAround(set, args); err != nil {
return err
}
return onTheBus(func(conn *nats.Conn) error {
now := time.Now()
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
if err != nil {
return err
}
repeated := link.RepeatedCauses(repairs(acts), now)
if *asJSON {
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
if len(acts) == 0 {
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
return nil
}
for i := len(acts) - 1; i >= 0; i-- {
a := acts[i]
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
if a.Condition != "" {
fmt.Printf(", condition %s", a.Condition)
}
fmt.Println(")")
}
if len(repeated) > 0 {
causes := make([]string, 0, len(repeated))
for c, n := range repeated {
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
}
sort.Strings(causes)
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
strings.Join(causes, ", "))
}
return nil
})
}
// repairs are the acts that are not a person's decision by design: what S15 counts.
func repairs(acts []link.HandAct) []link.HandAct {
out := make([]link.HandAct, 0, len(acts))
for _, a := range acts {
if !personsDecision(a) {
out = append(out, a)
}
}
return out
}
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
// the log could not be read, which status says rather than reading as none.
func handActsThisWeek(ctx context.Context) (int, string) {
n := -1
err := onTheBus(func(conn *nats.Conn) error {
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
n = len(acts)
return err
})
if err != nil {
return -1, err.Error()
}
return n, ""
}
cmd/mesh-controller/handacts_test.go
-94
View File
@@ -1,94 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
// the verb.
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
}{
{"push", map[string]any{"node": "anchor"}},
{"push", map[string]any{}},
{"plans", map[string]any{"close": "plan-1"}},
{"plans", map[string]any{"stop": "plan-1"}},
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
{"command", map[string]any{"command": "push anchor"}},
{"command", map[string]any{"command": "plans close plan-1"}},
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
} {
argv, err := argvFor(c.verb, c.args)
if c.verb == "plans" && err == nil {
// The seat composes the command line; the command refuses it, before opening anything.
err = plansCommand(context.Background(), argv[1:])
}
if err == nil || !strings.Contains(err.Error(), "why") {
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
}
}
for _, args := range [][]string{{"EVENTS", "controller"}} {
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
t.Errorf("consumer-reset without why: %v", err)
}
}
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
!strings.Contains(err.Error(), "--why") {
t.Errorf("hand-act record without why: %v", err)
}
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
!strings.Contains(err.Error(), "--cause") {
t.Errorf("hand-act record without a cause: %v", err)
}
}
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
func TestARepairByHandCarriesItsReason(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want string
}{
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
"plans close plan-1 --why the report will not come"},
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
}
}
}
// The summary of durations says, per kind and subject, what a bound would be set from.
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
var ds []inventory.Duration
for i := 1; i <= 10; i++ {
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
Took: time.Duration(i) * time.Second})
}
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
got := summarise(ds)
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
t.Fatalf("%+v", got)
}
if !strings.Contains(got[1].Suggests, "3m0s") {
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
}
}
cmd/mesh-controller/healers.go
-876
View File
@@ -1,876 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The healers (novox/hq to-be 45 §7, ADR 0227 rule 7, Phase 3).
//
// **A known failure heals itself, under a brake, and every repair is said.** Research 031 counted the
// repairs people made by hand in six days: a push to unstick a plan waiting on a report (four times),
// a controller restarted to make an object again (twice), a plan closed (twice), a consumer re-made from
// now (once). Each was the ordinary path, taken again by a person who noticed. A healer is that act,
// registered against the one condition kind it answers, so the mesh takes it itself:
//
// - **its repair is the ordinary path again** — the send a push makes, the plan's own close, the
// assertion every send makes, the consumer reset the verb makes — never a withdrawal, a deletion of
// data or a recreation of it;
// - **its budget** is how many acts it may take against one thing in a window, and **its settle** how
// long after an act it leaves the observation to say whether it worked;
// - **success is never the healer's to say.** The condition clears when the watchdog or the probe that
// raised it no longer sees it. A healer marking its own work done would be the second opinion of a
// fact that rule 1 forbids;
// - **a spent budget stops it**: the condition is the operator's, urgent, with every attempt in
// `tried`, and no healer touches it again until observation clears it;
// - **every act is said**: begun in the store before it is made (so a controller dying mid-act has
// still spent it), kept in the condition's `tried` as `healer Hn`, and said on the bus as the
// controller seat's `healer-acted`. A heal is not a hand act and is never in the hand-act log —
// which is how S15 can tell a repair the mesh made from one a person had to.
//
// **And the mesh-wide brake**: more than healBrakeLimit acts in an hour, all healers together, and every
// healer stops — an urgent condition says so — until an hour has passed with none. A healer looping is
// then at most a dozen acts, said, and never the incident itself.
//
// Only the controller holding the lease heals, under its epoch: a controller serving without the lease
// (S12) heals nothing, because a repair is the one act that can always wait for the lease.
// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour.
const (
healBrakeLimit = 12
healBrakeWindow = time.Hour
)
// healEvery is how often the healers look at what is open.
var healEvery = 30 * time.Second
// What raises the healers' own conditions, and their kinds.
const (
sourceHealers = "healers"
kindHealersBraked = "healers-braked"
kindConsumerBehind = "consumer-behind"
kindHealersBlind = "probe-failed"
)
// healerRow is one row of the registry.
type healerRow struct {
ID string
// Kinds are the condition kinds it answers: from the signals table, the probe registry, or an event.
Kinds []string
// Condition, Repair, Then, From are the row in words, as to-be 45 §7 and `healers` say it.
Condition string
Repair string
Then string
From string
// Budget acts against one budget key within Window; Settle after an act before the next, or the
// escalation, so the observation has had its turn to say whether it worked.
Budget int
Window time.Duration
Settle time.Duration
// ActsIn is where the repair runs: the controller, or a module that repairs its own (H5).
ActsIn string
// Event is what each act is said as.
Event string
// applies says whether this condition is one the healer may act on, and what its budget is
// counted against; why when it is not. Reads, never acts. Nil where the repair is a module's.
applies func(ctx context.Context, h *healing, c conditions.Condition) (budget string, ok bool, why string, err error)
// repair is the act: what it did in words, what came of it, or an error when it could not be done.
repair func(ctx context.Context, h *healing, c conditions.Condition) (act, said string, err error)
}
// actsInController is a healer whose repair the controller makes.
const actsInController = "controller"
// healerRegistry is to-be 45 §7's table, compiled in. **The registry is the design's live form**: a
// test generated from it holds every row to a condition kind the mesh raises, a budget, a brake and its
// event (healers_test.go).
var healerRegistry = []healerRow{
{ID: "H1", Kinds: []string{"sent-not-reported"},
Condition: "a machine was sent a declaration and has not reported it (S2)",
Repair: "ask the machine's node-engine to say again what it last applied (the `report` verb); if what " +
"comes back is not the declaration it was sent, or nothing comes, send it its current declaration " +
"again — what a push of that one machine does, never moving a build a policy or a plan holds back",
Then: "resolver operator, urgent", From: "a push by hand to unstick a plan waiting on a report (230, 257, 264, 267)",
Budget: 2, Window: 6 * time.Hour, Settle: 3 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAMachine, repair: repairReport},
{ID: "H2", Kinds: []string{"stalled"},
Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " +
"exists — or already finished — every module of every tier built or failed, and every one that " +
"rolls out sent (S3); or a delivery held past its state's bound where mesh-delivery's table lets H2 " +
"take a transition (D14, novox/hq ADR 0239)",
Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " +
"done; what it asked still builds and registers — or, for a delivery, mesh-delivery's `close`, which " +
"reads its walk again and takes only the transition its table names",
Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)",
Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAStalePlan, repair: repairPlan},
{ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"},
Condition: "a seat's holder that does not answer (D3), or a durable consumer the mesh expects and the " +
"bus does not hold (D6, S9)",
Repair: "assert the bus's streams, consumers and seat workers again — the assertion every send makes " +
"(issue 208)",
Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)",
Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAnObject, repair: repairObjects},
{ID: "H4", Kinds: []string{kindConsumerBehind},
Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable",
Repair: "re-make the consumer to deliver from now — `broker consumer-reset` (issue 248); what it drops " +
"is caught up where the table says",
Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)",
Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAResettableConsumer, repair: repairConsumer},
{ID: "H5", Kinds: []string{kindProviderFailing},
Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing, " +
"credentials-rejected)",
Repair: "the provider repairs it itself through the server's own bootstrap command, checks again and " +
"says what it did (ADR 0224 §5); the controller keeps its word as the condition",
Then: "announced failing by the provider, braked from ten minutes doubling to six hours (ADR 0224 §5)",
From: "the identity provider's admin reset through its bootstrap command (179)",
// Its budget and brake are the module's own: ten minutes, doubling, to six hours.
Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute,
ActsIn: "the provider module (keycloak), ADR 0224 §5", Event: "provisioner.failing, provisioner.recovered"},
}
// healerFor is the healer registered for a kind, and false when none acts in the controller.
func healerFor(kind string) (healerRow, bool) {
for _, r := range healerRegistry {
if r.repair != nil && slices.Contains(r.Kinds, kind) {
return r, true
}
}
return healerRow{}, false
}
// healerNamedFor is any healer registered for a kind, wherever it acts: what S15 says beside a cause.
func healerNamedFor(kind string) string {
for _, r := range healerRegistry {
if slices.Contains(r.Kinds, kind) {
return r.ID
}
}
return ""
}
// healing is the healers' runner and what their repairs reach.
type healing struct {
open *stores
keeper *conditions.Keeper
teller conditions.Teller
// js is the bus, for the repairs that assert or reset its objects; nil where there is none.
js *broker.JetStream
// epoch is the lease's gate; acting says this controller is the one acting, not one standing by.
epoch func(ctx context.Context) (uint64, error)
acting func() bool
now func() time.Time
say func(format string, args ...any)
// The acts, as seams a test replaces: asking a machine to report, sending it again, asserting the
// bus's objects, resetting a consumer.
askReport func(ctx context.Context, node string) error
sendAgain func(ctx context.Context, node string) error
assertObjects func(ctx context.Context) error
resetConsumer func(stream, name string) (string, error)
// reportWait is how long H1 waits for the machine's report after asking.
reportWait time.Duration
mu sync.Mutex
// declined is why each condition was last passed over, so it is said once and `healers` can show it.
declined map[string]string
paused string
}
// newHealing is the serving controller's runner, its acts the real ones.
func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Teller, js *broker.JetStream) *healing {
h := &healing{open: open, keeper: keeper, teller: teller, js: js, acting: link.Holding,
epoch: func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) },
now: time.Now, say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
reportWait: 45 * time.Second, declined: map[string]string{}}
h.askReport = func(ctx context.Context, node string) error {
if h.js == nil {
return errors.New("this controller is not on the bus")
}
return askToReport(ctx, h.js.Conn(), node)
}
h.sendAgain = func(ctx context.Context, node string) error {
// **Never what a policy or a plan holds back** (ADR 0221): a send by the mesh itself is a push
// that did not name the machine — a person's word sends a held build, a healer's does not.
held, err := heldMachines(ctx, open, []string{node})
if err != nil {
return err
}
if why := held[node]; len(why) > 0 {
return fmt.Errorf("not sent again: it would move what a policy or a plan holds back (ADR 0221) — %s; "+
"`push %s` sends it, on a person's word", strings.Join(why, "; "), node)
}
return sendTo(ctx, open, []string{node})
}
h.assertObjects = func(ctx context.Context) error {
if h.js == nil {
return errors.New("this controller is not on the bus")
}
return assertOnSend(ctx, open.inventory, h.js, " ")
}
h.resetConsumer = func(stream, name string) (string, error) {
if h.js == nil {
return "", errors.New("this controller is not on the bus")
}
before, after, err := h.js.ResetConsumer(stream, name)
if err != nil {
return "", err
}
return fmt.Sprintf("it was %d behind with %d unacknowledged; it delivers from now, %d pending", before.Pending,
before.AckPending, after.Pending), nil
}
return h
}
// askToReport asks one machine's node-engine to say again what it last applied (to-be 45 §6). On core
// NATS, fired and flushed: the answer is the machine's ordinary report, read from the store.
func askToReport(ctx context.Context, conn *nats.Conn, node string) error {
body, err := json.Marshal(map[string]any{"asked": time.Now().UTC(), "by": "the controller's healer H1"})
if err != nil {
return err
}
if err := conn.Publish(broker.AskReportSubject(node), body); err != nil {
return err
}
flushing, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
return conn.FlushWithContext(flushing)
}
// keep runs the healers until ctx ends.
func (h *healing) keep(ctx context.Context) {
tick := time.NewTicker(healEvery)
defer tick.Stop()
for {
h.tick(ctx)
select {
case <-ctx.Done():
return
case <-tick.C:
}
}
}
// tick is one look at what is open, and every act it calls for.
func (h *healing) tick(ctx context.Context) {
if h.acting != nil && !h.acting() {
return
}
epoch, err := h.epoch(ctx)
switch {
case err != nil:
h.pause(fmt.Sprintf("this controller may not act: %v", err))
return
case epoch == 0:
h.pause("this controller serves without the lease (S12): a repair waits for it")
return
}
inv := h.open.inventory
now := h.now()
heals, err := inv.HealsSince(ctx, now.Add(-24*time.Hour))
if err != nil {
// Blind: nothing is done, and that is said — never read as "no heals, budgets whole".
h.reconcile(ctx, []conditions.Observation{{Scope: conditions.ScopeProbe, ID: "healers", Token: "failed",
Kind: kindHealersBlind, Severity: conditions.Warning,
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
Said: firstLine(err.Error())}})
return
}
open, err := h.keeper.Open(ctx)
if err != nil {
h.say("the healers cannot read the open conditions, and act on nothing: %v", err)
return
}
acts := actsWithin(heals, now.Add(-healBrakeWindow))
if braked, said := brakeHolds(acts, open, now); braked {
h.reconcile(ctx, []conditions.Observation{brakeObservation(acts, said)})
h.pause("the mesh-wide brake holds: " + said)
return
}
h.reconcile(ctx, nil)
h.resume()
for _, c := range open {
row, ok := healerFor(c.Kind)
if !ok || c.Escalated() {
continue
}
budget, applies, why, err := row.applies(ctx, h, c)
if err != nil {
h.decline(c.Key, row.ID, "could not tell whether it applies: "+err.Error())
continue
}
if !applies {
h.decline(c.Key, row.ID, why)
continue
}
h.forget(c.Key)
spent := spentOn(heals, row, budget, now)
if n := len(spent); n > 0 && now.Sub(spent[n-1].At) < row.Settle {
continue // its last act is still the observation's to judge
}
if len(spent) >= row.Budget {
h.escalate(ctx, row, c, budget, spent)
continue
}
if len(acts) >= healBrakeLimit {
return // the brake takes hold on the next look, said there
}
if h.act(ctx, row, c, budget, len(spent)) {
acts = append(acts, inventory.Heal{At: now})
}
}
}
// act is one healer's act on one condition: begun in the store, made, finished, kept in the
// condition's tried and said. False when it could not even be begun.
func (h *healing) act(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent int) bool {
inv := h.open.inventory
begun, err := inv.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
BudgetKey: budget, Act: row.Repair, At: h.now()})
if err != nil {
h.say("healer %s did not act on %s: %v", row.ID, c.Key, err)
return false
}
act, said, err := row.repair(ctx, h, c)
outcome := inventory.HealActed
if err != nil {
outcome, said = inventory.HealFailed, err.Error()
}
if act == "" {
act = row.Repair
}
finishing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if ferr := inv.FinishHeal(finishing, begun.ID, outcome, act+" — "+said); ferr != nil {
h.say("healer %s acted on %s and could not record what came of it: %v", row.ID, c.Key, ferr)
}
budgetWords := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window)
attempt := conditions.Attempt{What: act, Outcome: outcome + ": " + said + " (" + budgetWords + ")",
By: "healer " + row.ID}
if _, _, terr := h.keeper.Tried(finishing, c.Key, attempt, conditions.ResolverHealer(row.ID)); terr != nil {
h.say("healer %s acted on %s and could not keep it in the condition: %v", row.ID, c.Key, terr)
}
h.tell(finishing, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: act, Outcome: outcome,
Said: said, Budget: budgetWords, Epoch: begun.Epoch})
h.say("healer %s %s %s: %s — %s (%s)", row.ID, outcome, c.Key, act, said, budgetWords)
return true
}
// escalate is a spent budget: the condition is the operator's now, urgent, with what was tried. Said
// once: an escalated condition is passed over by every healer until observation clears it.
func (h *healing) escalate(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent []inventory.Heal) {
var tried []string
for _, s := range spent {
tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.At.UTC().Format("15:04 MST"), firstLine(s.Said)))
}
said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window,
c.Key, strings.Join(tried, "; "))
if _, err := h.open.inventory.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
BudgetKey: budget, Act: "handed the condition to the operator", Outcome: inventory.HealEscalated, Said: said,
At: h.now()}); err != nil {
h.say("healer %s could not record handing %s to the operator, and does not: %v", row.ID, c.Key, err)
return
}
attempt := conditions.Attempt{What: "handed to the operator: nothing in the mesh will repair it now",
Outcome: inventory.HealEscalated + ": " + said, By: "healer " + row.ID}
if _, _, err := h.keeper.Escalate(ctx, c.Key, attempt); err != nil {
h.say("healer %s could not hand %s to the operator: %v", row.ID, c.Key, err)
}
h.tell(ctx, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator",
Outcome: inventory.HealEscalated, Said: said, Budget: fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window)})
h.say("healer %s handed %s to the operator: %s", row.ID, c.Key, said)
}
// healerActed is the body of `healer-acted` (to-be 45 §7): the healer, the condition, the act and its
// outcome, and where the budget stands. **A contract**, like a condition's events: the operator-channel's
// holder may say it.
type healerActed struct {
Event string `json:"event"`
At time.Time `json:"at"`
Healer string `json:"healer"`
By string `json:"by"`
Condition string `json:"condition"`
Kind string `json:"kind"`
Act string `json:"act"`
// Outcome is acted, failed or escalated. Acted says the act was made, not that it worked: the
// condition clearing says that.
Outcome string `json:"outcome"`
Said string `json:"said"`
Budget string `json:"budget"`
Epoch uint64 `json:"epoch,omitempty"`
Show string `json:"show"`
}
// tell says a heal on the bus. Not said is said in the log: the act and the condition's tried still
// hold it.
func (h *healing) tell(ctx context.Context, e healerActed) {
e.Event, e.At, e.By = link.KeyHealerActed, h.now().UTC(), "healer "+e.Healer
e.Show = "mesh-controller.conditions key=" + e.Condition
if h.teller == nil {
return
}
body, err := json.Marshal(e)
if err != nil {
return
}
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := h.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyHealerActed, body); err != nil {
h.say("healer %s %s %s, and that could NOT be said on the bus: %v", e.Healer, e.Outcome, e.Condition, err)
}
}
// reconcile keeps the healers' own conditions: the brake, and their being blind.
func (h *healing) reconcile(ctx context.Context, observed []conditions.Observation) {
if err := h.keeper.Reconcile(ctx, sourceHealers, observed); err != nil {
h.say("the healers' own conditions could not be kept: %v", err)
}
}
func (h *healing) pause(why string) {
h.mu.Lock()
defer h.mu.Unlock()
if h.paused != why {
h.say("the healers act on nothing: %s", why)
}
h.paused = why
}
func (h *healing) resume() {
h.mu.Lock()
defer h.mu.Unlock()
if h.paused != "" {
h.say("the healers act again")
}
h.paused = ""
}
// decline remembers why a healer passed a condition over, said once.
func (h *healing) decline(key, healer, why string) {
h.mu.Lock()
defer h.mu.Unlock()
if h.declined[key] != why {
h.say("healer %s leaves %s alone: %s", healer, key, why)
}
h.declined[key] = why
}
func (h *healing) forget(key string) {
h.mu.Lock()
defer h.mu.Unlock()
delete(h.declined, key)
}
// actsWithin are the heals since a moment that were acts — begun, made or failed; an escalation is a
// saying, not an act, and the brake does not count it.
func actsWithin(heals []inventory.Heal, since time.Time) []inventory.Heal {
var out []inventory.Heal
for _, h := range heals {
if !h.At.Before(since) && h.Outcome != inventory.HealEscalated {
out = append(out, h)
}
}
return out
}
// spentOn is one healer's acts against one budget key within its window, oldest first.
func spentOn(heals []inventory.Heal, row healerRow, budget string, now time.Time) []inventory.Heal {
var out []inventory.Heal
for _, h := range actsWithin(heals, now.Add(-row.Window)) {
if h.Healer == row.ID && h.BudgetKey == budget {
out = append(out, h)
}
}
return out
}
// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake
// already said and an act within the hour still — it lets go an hour after the last act, not the first.
func brakeHolds(acts []inventory.Heal, open []conditions.Condition, now time.Time) (bool, string) {
said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), healBrakeWindow, healBrakeLimit)
if len(acts) >= healBrakeLimit {
return true, said
}
held := slices.ContainsFunc(open, func(c conditions.Condition) bool { return c.Kind == kindHealersBraked })
if held && len(acts) > 0 {
last := acts[len(acts)-1].At
return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said,
last.Add(healBrakeWindow).UTC().Format("15:04 MST"))
}
return false, said
}
// brakeObservation is the brake, as the urgent condition that says it.
func brakeObservation(acts []inventory.Heal, said string) conditions.Observation {
counts := map[string]int{}
for _, a := range acts {
if a.Healer != "" {
counts[a.Healer+" on "+a.ConditionKey]++
}
}
var most []string
for what, n := range counts {
most = append(most, fmt.Sprintf("%s ×%d", what, n))
}
sort.Strings(most)
return conditions.Observation{Scope: conditions.ScopeMesh, ID: "healers", Token: "braked", Kind: kindHealersBraked,
Severity: conditions.Urgent,
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
"healer looping, not repairing — " + said,
Said: strings.Join(most, ", ")}
}
// --- H1 ----------------------------------------------------------------------------------------------
// appliesToAMachine is H1's: a machine named, its budget counted against the condition.
func appliesToAMachine(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
if c.Subject.Machine == "" {
return "", false, "it names no machine", nil
}
return c.Key, true, "", nil
}
// repairReport is H1: ask the machine to report; if what comes back is not what it was sent, or nothing
// comes, send it again.
func repairReport(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
node := c.Subject.Machine
inv := h.open.inventory
// The store's clock, as the report's time is: not the runner's, which a test moves by hand.
asked := time.Now()
askErr := h.askReport(ctx, node)
current, heard := false, false
if askErr == nil {
deadline := time.Now().Add(h.reportWait)
for {
r, found, err := lastReportOf(ctx, inv, node)
if err != nil {
return "", "", err
}
if found && r.At != nil && r.At.After(asked) {
heard, current = true, r.Current
if current {
break
}
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
return "", "", ctx.Err()
case <-time.After(min(2*time.Second, h.reportWait/4+time.Millisecond)):
}
}
}
if current {
return "asked " + node + "'s node-engine to say again what it last applied",
"it reported the declaration it was sent: its report had not reached the mesh", nil
}
why := "it said nothing within " + h.reportWait.String() + " — its node-engine may be older than the report verb"
switch {
case askErr != nil:
why = "it could not be asked: " + askErr.Error()
case heard:
why = "it reported a declaration other than the one it was sent"
}
if err := h.sendAgain(ctx, node); err != nil {
return "asked " + node + " to report, then sent it its current declaration again", why + "; the send failed",
err
}
return "asked " + node + " to report, then sent it its current declaration again", why + "; sent again", nil
}
// lastReportOf is one machine's last report beside its last send.
func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (inventory.Reported, bool, error) {
reports, err := inv.LastReports(ctx)
if err != nil {
return inventory.Reported{}, false, err
}
for _, r := range reports {
if r.Node == node {
return r, true, nil
}
}
return inventory.Reported{}, false, nil
}
// --- H2 ----------------------------------------------------------------------------------------------
// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty
// when it is neither, which is not H2's to repair.
func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) {
if p.Release != nil {
return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236)
}
recent, err := inv.RecentPlans(ctx, 50)
if err != nil {
return "", "", err
}
for _, newer := range recent {
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded {
continue
}
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
"and branch", newer.ID, newer.Repository, short(newer.Commit)), nil
}
for _, tier := range p.Tiers {
for _, m := range tier {
s := p.Modules[m]
if s == nil || (s.State != "built" && s.State != "failed") {
return "", "", nil
}
if s.State == "built" && s.SentAt == nil {
u, err := inv.UpgradeOf(ctx, m)
if err != nil {
return "", "", err
}
if u.RollOut {
return "", "", nil
}
}
}
}
return inventory.PlanDone, "finished: every module of every tier is built or failed, and every one that rolls " +
"out was sent — nothing is left to wait on", nil
}
// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished.
func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
if c.Subject.Scope == conditions.ScopeDelivery {
return appliesToAStalledDelivery(ctx, h, c)
}
p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID)
if err != nil {
return "", false, "", err
}
if !p.Open() {
return "", false, "the plan is " + p.State + " already: its condition clears on the next look", nil
}
state, _, err := planStale(ctx, h.open.inventory, p)
if err != nil {
return "", false, "", err
}
if state == "" {
return "", false, "its wait is neither superseded nor finished: it waits on something still to come, " +
"which its own signal says", nil
}
return c.Key, true, "", nil
}
// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set.
func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
if c.Subject.Scope == conditions.ScopeDelivery {
return repairDelivery(ctx, h, c)
}
inv := h.open.inventory
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return "", "", err
}
defer release()
p, err := inv.PlanByID(ctx, c.Subject.ID)
if err != nil {
return "", "", err
}
if !p.Open() {
return "closed nothing", "the plan is " + p.State + " already", nil
}
state, why, err := planStale(ctx, inv, p)
if err != nil {
return "", "", err
}
if state == "" {
return "closed nothing", "its wait is no longer superseded or finished", nil
}
p.State = state
p.Note = fmt.Sprintf("closed by healer H2 at tier %d: %s", p.Tier, why)
if state != inventory.PlanDone {
sayUnsent(&p, func(m string) bool {
u, err := inv.UpgradeOf(ctx, m)
return err == nil && u.RollOut
})
}
if err := inv.SavePlan(ctx, &p); err != nil {
return "", "", err
}
return "closed the plan " + p.ID + " (" + state + ")", why, nil
}
// --- H3 ----------------------------------------------------------------------------------------------
// appliesToAnObject is H3's: every holder or consumer the probe or the bus names, its budget per object.
func appliesToAnObject(_ context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
if h.assertObjects == nil {
return "", false, "this controller is not on the bus", nil
}
return c.Key, true, "", nil
}
// repairObjects is H3: the send's own assertion of every stream, consumer and seat worker.
func repairObjects(ctx context.Context, h *healing, _ conditions.Condition) (string, string, error) {
if err := h.assertObjects(ctx); err != nil {
return "", "", err
}
return "asserted the bus's streams, consumers and seat workers again",
"every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there", nil
}
// --- H4 ----------------------------------------------------------------------------------------------
// resettable is why a consumer may be reset by the mesh itself, from the stream table; empty when not.
func resettable(stream, name string) string {
for _, c := range broker.MeshConsumers() {
if c.Stream == stream && c.Name == name {
return c.Resettable
}
}
return ""
}
// consumerOf is the stream and consumer a bus condition names: `<stream>.<consumer>`.
func consumerOf(c conditions.Condition) (string, string, bool) {
stream, name, found := strings.Cut(c.Subject.ID, ".")
return stream, name, found && stream != "" && name != ""
}
// appliesToAResettableConsumer is H4's: only a consumer the stream table marks resettable.
func appliesToAResettableConsumer(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
stream, name, ok := consumerOf(c)
if !ok {
return "", false, "it names no consumer", nil
}
if resettable(stream, name) == "" {
return "", false, fmt.Sprintf("%s on %s is not marked resettable in the stream table: what a reset drops, "+
"nothing would catch up", name, stream), nil
}
return c.Key, true, "", nil
}
// repairConsumer is H4: `broker consumer-reset`, made by the mesh.
func repairConsumer(_ context.Context, h *healing, c conditions.Condition) (string, string, error) {
stream, name, _ := consumerOf(c)
said, err := h.resetConsumer(stream, name)
if err != nil {
return "", "", err
}
return fmt.Sprintf("re-made %s on %s to deliver from now", name, stream),
said + "; " + resettable(stream, name), nil
}
// --- the verb ----------------------------------------------------------------------------------------
// healersCommand is `healers`: the registry, what the healers did lately, and the brake.
func healersCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("healers", flag.ContinueOnError)
daysFlag := set.Int("days", 7, "how many days of acts back")
jsonFlag := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("healers [--days N] [--json]")
}
days, asJSON := *daysFlag, *jsonFlag
if days <= 0 {
return fmt.Errorf("healers --days takes a number of days, not %d", days)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
now := time.Now()
heals, err := open.inventory.HealsSince(ctx, now.Add(-time.Duration(days)*24*time.Hour))
if err != nil {
return fmt.Errorf("what the healers did cannot be read: %w", err)
}
conds, condErr := openConditions(ctx)
braked, said := brakeHolds(actsWithin(heals, now.Add(-healBrakeWindow)), conds, now)
brake := map[string]any{"holds": braked, "said": said, "limit": healBrakeLimit, "window": healBrakeWindow.String()}
if condErr != nil {
brake["conditions"] = "the open conditions could not be read, so whether the brake was said is not known: " +
condErr.Error()
}
var rows []map[string]any
for _, r := range healerRegistry {
rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair,
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
"then": r.Then, "acts-in": r.ActsIn, "event": r.Event, "from": r.From})
}
if heals == nil {
heals = []inventory.Heal{}
}
if asJSON {
return printJSON(map[string]any{"healers": rows, "heals": heals, "brake": brake, "days": days,
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"})
}
for _, r := range healerRegistry {
fmt.Printf("%s %s\n → %s\n budget %d within %s; then %s (in %s)\n", r.ID, r.Condition, r.Repair, r.Budget,
r.Window, r.Then, r.ActsIn)
}
fmt.Printf("\nthe brake: %s — %s\n\n", map[bool]string{true: "HOLDS, every healer has stopped", false: "off"}[braked], said)
if len(heals) == 0 {
fmt.Printf("no healer acted in the last %d day(s)\n", days)
return nil
}
for i := len(heals) - 1; i >= 0; i-- {
x := heals[i]
fmt.Printf("%s %s %-9s %s\n %s\n", x.At.Local().Format("2006-01-02 15:04"), x.Healer, x.Outcome, x.ConditionKey,
firstLine(x.Said))
}
return nil
}
// forgettingOldHeals removes heals past their keeping once a day, by the controller acting only.
func forgettingOldHeals(ctx context.Context, inv *inventory.Inventory) {
for {
if link.Holding() {
if n, err := inv.ForgetOldHeals(ctx); err != nil {
fmt.Printf("heals older than %s could not be removed: %v\n", inventory.HealsKeptFor, err)
} else if n > 0 {
fmt.Printf("removed %d heal(s) older than %s\n", n, inventory.HealsKeptFor)
}
}
select {
case <-ctx.Done():
return
case <-time.After(24 * time.Hour):
}
}
}
// healsCount is what the healers did, counted for `status`.
type healsCount struct {
Acts int `json:"acts"`
Escalated int `json:"escalated"`
}
// countHeals counts acts and escalations.
func countHeals(heals []inventory.Heal) *healsCount {
out := &healsCount{}
for _, h := range heals {
if h.Outcome == inventory.HealEscalated {
out.Escalated++
} else {
out.Acts++
}
}
return out
}
cmd/mesh-controller/healers_test.go
-640
View File
@@ -1,640 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"slices"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
// checked"): **every row is walked.** Its kinds are ones the mesh raises — a row of the signals table, a
// probe of the self-check, or a provider's event — it has a budget, a window and a settle inside it,
// what happens when the budget is spent, and the event each act is said as; a healer the controller runs
// has its applies and its repair, and an induced failure below that sees it act, say so and brake. A
// row added without one fails, so the registry cannot grow a healer nobody has seen act.
// raisedKinds is every condition kind the mesh raises, and what raises it.
func raisedKinds() map[string]string {
out := map[string]string{kindProviderFailing: "the provisioner.failing event (ADR 0224)"}
for _, r := range signalsTable {
for _, k := range kindsOf(r) {
out[k] = r.Row
}
}
for _, p := range probeRegistry {
out[p.Kind] = p.ID
for _, k := range p.Raises {
out[k] = p.ID
}
}
return out
}
// inducedFailures are the healers seen acting in this file, by id: a row without one fails.
var inducedFailures = map[string]string{
"H1": "TestH1AsksAMachineToReportAndSendsItAgain",
"H2": "TestH2ClosesAPlanAnotherHasTakenOver",
"H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget",
"H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers",
}
func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) {
kinds := raisedKinds()
source, err := os.ReadFile("healers_test.go")
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
answered := map[string]string{}
for _, r := range healerRegistry {
t.Run(r.ID, func(t *testing.T) {
if seen[r.ID] {
t.Fatalf("%s is in the registry twice", r.ID)
}
seen[r.ID] = true
if len(r.Kinds) == 0 || r.Condition == "" || r.Repair == "" || r.Then == "" || r.From == "" || r.ActsIn == "" {
t.Fatalf("%s does not say what it answers, what it does, what then, and which hand act it replaces: %+v", r.ID, r)
}
for _, k := range r.Kinds {
if _, ok := kinds[k]; !ok {
t.Errorf("%s answers %q, which nothing in the mesh raises", r.ID, k)
}
if other, twice := answered[k]; twice {
t.Errorf("%q is answered by %s and %s: one healer per kind", k, other, r.ID)
}
answered[k] = r.ID
}
if r.Budget <= 0 || r.Window <= 0 || r.Settle <= 0 || r.Settle > r.Window {
t.Errorf("%s has no budget it can spend: %d within %s, settled after %s", r.ID, r.Budget, r.Window, r.Settle)
}
if r.Event == "" {
t.Errorf("%s says nothing when it acts", r.ID)
}
if r.ActsIn != actsInController {
if r.repair != nil || r.applies != nil {
t.Errorf("%s acts in %s and the controller would act for it too", r.ID, r.ActsIn)
}
return
}
if r.repair == nil || r.applies == nil {
t.Fatalf("%s acts in the controller and has no repair or no applies", r.ID)
}
if r.Event != link.KeyHealerActed || !slices.Contains(broker.ControllerStates, r.Event) {
t.Errorf("%s is said as %q, which the controller's grant does not permit", r.ID, r.Event)
}
if inducedFailures[r.ID] == "" {
t.Errorf("%s has no induced failure: a healer nobody has seen act", r.ID)
} else if !strings.Contains(string(source), "func "+inducedFailures[r.ID]+"(t *testing.T)") {
t.Errorf("%s's induced failure %s is not a test in this file", r.ID, inducedFailures[r.ID])
}
})
}
for id := range inducedFailures {
if !seen[id] {
t.Errorf("an induced failure for %s, which the registry does not have", id)
}
}
if healBrakeLimit <= 0 || healBrakeWindow <= 0 {
t.Error("the mesh-wide brake holds nothing")
}
}
// heard keeps the healer-acted events said.
type heard struct {
mu sync.Mutex
acts []healerActed
}
func (h *heard) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
if seat != conditions.Seat || event != link.KeyHealerActed {
return errors.New("said under the wrong seat or name: " + seat + " " + event)
}
var e healerActed
if err := json.Unmarshal(body, &e); err != nil {
return err
}
h.mu.Lock()
defer h.mu.Unlock()
h.acts = append(h.acts, e)
return nil
}
func (h *heard) said() []healerActed {
h.mu.Lock()
defer h.mu.Unlock()
return append([]healerActed(nil), h.acts...)
}
// testClock is a moment a test moves by hand.
type testClock struct {
mu sync.Mutex
at time.Time
}
func (c *testClock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.at
}
func (c *testClock) pass(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.at = c.at.Add(d)
}
// healingOn is a runner over a mesh's stores and its condition store, under epoch 57, every act a
// fake that fails the test unless the test gives it.
func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) {
t.Helper()
if conditionsFrom == nil {
t.Fatal("the mesh has no condition store")
}
told, clock := &heard{}, &testClock{at: time.Now()}
// The store's gate, as the serving controller's is the lease's (stores.go).
open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
h := &healing{open: open, keeper: conditionsFrom, teller: told,
epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now,
say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond,
declined: map[string]string{}}
h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil }
h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil }
h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil }
h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil }
return h, told, clock
}
// sentNotReported raises S2 for a machine, as the watchdog does.
func sentNotReported(t *testing.T, node string) string {
t.Helper()
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "sent-not-reported", Machine: node,
Severity: conditions.Warning, Summary: node + " was sent a declaration and has not reported it", Source: "S2"}
if _, err := conditionsFrom.Observe(t.Context(), o); err != nil {
t.Fatal(err)
}
return o.Key()
}
// **H1, the commonest hand act** (031/01 §f: a push by hand to unstick a plan waiting on a report, four
// times): the machine is asked to report; a report that names what it was sent is all it takes, and one
// that does not — or none — is a send of its current declaration again. Each act kept in `tried` as
// `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more.
func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, clock := healingOn(t, open)
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil {
t.Fatal(err)
}
key := sentNotReported(t, "laptop")
// First: the report had not reached the mesh, and asking for it brings it.
asked := 0
h.askReport = func(ctx context.Context, node string) error {
asked++
if node != "laptop" {
t.Errorf("asked %s", node)
}
_, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied,
At: time.Now().Add(time.Second), Declared: "d2"})
return err
}
h.tick(ctx)
c, found, err := conditionsFrom.Get(ctx, key)
if err != nil || !found {
t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err)
}
if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") ||
c.Resolver != "healer:H1" {
t.Fatalf("after the first act: asked %d, %+v", asked, c)
}
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed ||
acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 {
t.Fatalf("the act was not said as healer-acted: %+v", acts)
}
// Within its settle, nothing more: the observation has its turn.
clock.pass(time.Minute)
h.tick(ctx)
if asked != 1 {
t.Fatalf("acted again inside the settle: asked %d", asked)
}
// Second: the machine says nothing, so it is sent again.
clock.pass(3 * time.Minute)
sent := 0
h.askReport = func(context.Context, string) error { asked++; return nil }
h.sendAgain = func(_ context.Context, node string) error { sent++; return nil }
h.tick(ctx)
if asked != 2 || sent != 1 {
t.Fatalf("the second act: asked %d, sent %d", asked, sent)
}
c, _, _ = conditionsFrom.Get(ctx, key)
if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") {
t.Fatalf("tried %+v", c.Tried)
}
// The budget is spent: the operator's, urgent, said; and no healer touches it again.
clock.pass(4 * time.Minute)
h.tick(ctx)
c, _, _ = conditionsFrom.Get(ctx, key)
if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 ||
!strings.Contains(c.Tried[2].Outcome, "budget of 2") {
t.Fatalf("not handed to the operator: %+v", c)
}
if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated {
t.Fatalf("the escalation was not said: %+v", acts)
}
clock.pass(time.Hour)
h.tick(ctx)
if asked != 2 || sent != 1 || len(told.said()) != 3 {
t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said()))
}
heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour))
if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed ||
heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 {
t.Fatalf("the heals kept: %+v %v", heals, err)
}
// And a heal is not a hand act: what S15 counts never sees it.
for _, x := range heals {
if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") {
t.Errorf("a heal reads as a hand act: %+v", x)
}
}
}
// **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing.
func TestNoHealerActsWithoutTheLease(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, _ := healingOn(t, open)
sentNotReported(t, "laptop")
h.epoch = func(context.Context) (uint64, error) { return 0, nil }
h.tick(ctx)
h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }
h.tick(ctx)
h.epoch = func(context.Context) (uint64, error) { return 57, nil }
h.acting = func() bool { return false }
h.tick(ctx)
if len(told.said()) != 0 {
t.Fatalf("a healer acted without the lease: %+v", told.said())
}
}
// **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour
// after the last.
func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, clock := healingOn(t, open)
for i := 0; i < healBrakeLimit; i++ {
if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent",
Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed,
At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil {
t.Fatal(err)
}
}
sentNotReported(t, "laptop")
h.tick(ctx) // the fakes fail the test if anything acts
braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked")
if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") {
t.Fatalf("the brake was not said: %+v %v", braked, err)
}
if len(told.said()) != 0 {
t.Fatalf("a healer acted under the brake: %+v", told.said())
}
// Past the hour of the first act, still held: it lets go an hour after the last.
clock.pass(30 * time.Minute)
h.tick(ctx)
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held {
t.Fatal("the brake let go before an hour had passed since the last act")
}
clock.pass(31 * time.Minute)
asked := 0
h.askReport = func(context.Context, string) error { asked++; return nil }
h.sendAgain = func(context.Context, string) error { return nil }
h.tick(ctx)
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held {
t.Fatal("the brake held an hour after the last act")
}
if asked != 1 {
t.Fatalf("the healers did not act again after the brake let go: asked %d", asked)
}
}
// **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is
// still to come: that one is its own signal's, not a healer's.
func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, _ := healingOn(t, open)
created := time.Now().UTC().Add(-time.Hour)
older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created,
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}}
waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created,
State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}}
newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute),
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}
for _, p := range []*inventory.Plan{&older, &waiting, &newer} {
if err := open.inventory.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
}
for _, id := range []string{"plan-old", "plan-other"} {
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled",
Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil {
t.Fatal(err)
}
}
h.tick(ctx)
closed, err := open.inventory.PlanByID(ctx, "plan-old")
if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") ||
!strings.Contains(closed.Note, "plan-new") {
t.Fatalf("the superseded plan: %+v %v", closed, err)
}
if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding {
t.Fatalf("a plan still waiting was closed: %+v", other)
}
if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf {
t.Fatalf("a plan H2 does not repair was touched: %+v", c)
}
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" {
t.Fatalf("said %+v", acts)
}
// Finished: every module built, none rolled out unsent — closed as done.
done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created,
State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}}
if err := open.inventory.SavePlan(ctx, &done); err != nil {
t.Fatal(err)
}
if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") {
t.Fatalf("a finished plan reads %q %q %v", state, why, err)
}
}
// **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said
// and left — what a reset drops, nothing would catch up for it.
func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, _, _ := healingOn(t, open)
marked := 0
for _, c := range broker.MeshConsumers() {
if c.Resettable != "" {
marked++
if c.Stream != broker.EventsStream || c.Name != broker.ControllerName {
t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream)
}
}
}
if marked != 1 {
t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked)
}
for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} {
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil {
t.Fatal(err)
}
}
h.tick(ctx) // the fake reset fails the test if it is called
reset := ""
h.resetConsumer = func(stream, name string) (string, error) {
reset = stream + "." + name
return "it was 1500 behind", nil
}
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller",
Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil {
t.Fatal(err)
}
h.tick(ctx)
if reset != "EVENTS.controller" {
t.Fatalf("reset %q", reset)
}
}
// **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
url := testbus.URL(t)
open := aMesh(t)
ctx := t.Context()
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
_ = js.Context().DeleteStream(s)
}
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
t.Fatal(err)
}
h, told, clock := healingOn(t, open)
h.js = js
h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") }
d := &doctor{open: open, js: js}
probe := func() {
t.Helper()
found, err := probeConsumers(ctx, d)
if err != nil {
t.Fatal(err)
}
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
t.Fatal(err)
}
}
const key = "bus.NODES.laptop.missing"
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
t.Fatal(err)
}
probe()
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" {
t.Fatalf("the deleted consumer was not raised: %+v", c)
}
h.tick(ctx)
if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil {
t.Fatalf("H3 did not make the consumer again: %v", err)
}
c, still, _ := conditionsFrom.Get(ctx, key)
if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" {
t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c)
}
probe()
if _, still, _ := conditionsFrom.Get(ctx, key); still {
t.Fatal("the probe's next run did not clear what H3 repaired")
}
// Kept in the history as the keeper says it, a moment later.
var cleared *conditions.Event
for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) {
history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute))
if err != nil {
t.Fatal(err)
}
for i := range history {
if history[i].Key == key && history[i].Change == conditions.ChangeCleared {
cleared = &history[i]
}
}
}
if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 {
t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared)
}
// Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops.
clock.pass(10 * time.Minute)
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
t.Fatal(err)
}
probe()
h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil }
h.tick(ctx)
c, _, _ = conditionsFrom.Get(ctx, key)
if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 {
t.Fatalf("the spent budget was not handed to the operator: %+v", c)
}
acts := told.said()
if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated {
t.Fatalf("said %+v", acts)
}
clock.pass(2 * time.Hour)
h.tick(ctx)
if len(told.said()) != 2 {
t.Fatal("a healer acted on what the operator holds")
}
}
// **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
// receives what comes next: a reset that left the controller deaf would be the incident.
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
url := testbus.URL(t)
open := aMesh(t)
ctx := t.Context()
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
_ = js.Context().DeleteStream(s)
}
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
t.Fatal(err)
}
// Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none.
events := make(chan *nats.Msg, 64)
sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
followed := broker.ControllerFollows[0]
for i := 0; i < consumerFarBehind+200; i++ {
if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil {
t.Fatal(err)
}
}
d := &doctor{open: open, js: js}
probe := func() []conditions.Observation {
t.Helper()
found, err := probeConsumers(ctx, d)
if err != nil {
t.Fatal(err)
}
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
t.Fatal(err)
}
return found
}
probe()
const key = "bus.EVENTS.controller.behind"
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind {
t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c)
}
h, told, _ := healingOn(t, open)
h.resetConsumer = func(stream, name string) (string, error) {
before, after, err := js.ResetConsumer(stream, name)
if err != nil {
return "", err
}
return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) +
" pending now", nil
}
h.tick(ctx)
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed {
t.Fatalf("said %+v", acts)
}
if found := probe(); len(found) != 0 {
t.Fatalf("after the reset the probe still finds %+v", found)
}
if _, still, _ := conditionsFrom.Get(ctx, key); still {
t.Fatal("the probe's next run did not clear what H4 repaired")
}
// What comes next still reaches the controller.
for len(events) > 0 {
<-events
}
if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil {
t.Fatal(err)
}
deadline := time.After(10 * time.Second)
for {
select {
case m := <-events:
if strings.Contains(string(m.Data), "after") {
return
}
_ = m.Ack()
case <-deadline:
t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf")
}
}
}
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop"))
if err != nil {
t.Fatal(err)
}
if err := askToReport(t.Context(), conn, "laptop"); err != nil {
t.Fatal(err)
}
msg, err := asked.NextMsg(5 * time.Second)
if err != nil || !strings.Contains(string(msg.Data), "healer H1") {
t.Fatalf("the machine heard %v, %v", msg, err)
}
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") ||
slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") {
t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err)
}
}
cmd/mesh-controller/health_probes.go
-226
View File
@@ -1,226 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The core components' health definitions, as probes in the self-check's registry (novox/hq to-be 45
// §8, §4 `H-*`). The same definitions judge a core component's new build on its first machine (the
// gate, gate.go); here they are run against every machine on the self-check's schedule, so a core
// component that stops being healthy between upgrades is said too.
//
// controller holds the lease, and says it is ready: its self-check ran, `status` answered in bound
// node-engine has reported its current declaration, under a build the mesh delivered
// node tools announced, and answer the bus's discovery
// bus every stream and durable consumer the mesh defines is there, and a request crosses the
// bus to every machine's node tools and back
const kindCoreUnhealthy = "core-unhealthy"
// probeControllerHealth is H-controller: the lease is held, fresh, by a controller that says it is
// ready — or one that started less than the witness's bound ago.
func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
h, found, err := theLease.holder(ctx)
if err != nil {
return nil, err
}
unhealthy := func(why string) []conditions.Observation {
node := d.host
if found && h.Host != "" {
node = h.Host
}
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: lease.ComponentController + "." + node,
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Urgent,
Summary: "the controller is not healthy: " + why}}
}
switch {
case !found:
return unhealthy("nobody holds the controller lease"), nil
case time.Since(h.Renewed) > lease.FreshWithin:
return unhealthy(fmt.Sprintf("the lease was last renewed %s ago", time.Since(h.Renewed).Round(time.Second))), nil
case (h.Health == nil || !h.Health.Ready) && time.Since(h.Taken) > lease.ReadyWithin:
why := "the controller holding the lease does not say it is ready"
if h.Health != nil && h.Health.Why != "" {
why += ": " + h.Health.Why
}
return unhealthy(why), nil
}
return nil, nil
}
// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the
// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send.
func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
return machinesHealth(ctx, d, hostModule)
}
// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them
// announced, answering the bus's discovery.
func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
return machinesHealth(ctx, d, broker.RuntimeModule)
}
// machinesHealth judges a component on every machine running it and heard from, by its health
// definition, as the gate does — with no condition taken as the build's doing.
func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) {
inv := d.open.inventory
running, err := inv.Running(ctx, component)
if err != nil {
return nil, err
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
f, err := gatherGateFacts(ctx, d.open, coreComponent(component))
if err != nil {
return nil, err
}
f.judged = false
heard := heardMachines(d)
var out []conditions.Observation
for _, node := range running {
if !heard[node] {
continue // a machine not heard from is S1's
}
if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound {
continue // inside the bound of a send: S2's, and the gate's
}
// What the machine did with what it was sent is not the component's health: the node-engine's is
// that it reported its current declaration at all, the node tools' that they answer.
if r := f.reports[node]; r.Current || component == broker.RuntimeModule {
now := time.Now()
r.Current, r.Outcome = true, inventory.OutcomeApplied
if r.At == nil {
r.At = &now
}
f.reports[node] = r
}
if component == hostModule {
// A node-engine reporting a build the mesh delivered, current or previous, is the version
// split's (D10), not ill health; a build the mesh did not deliver is.
f.engines[node] = deliveredOr(shelf[component], f.engines[node])
}
h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f)
if h == healthGood {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node,
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning,
Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)})
}
return sortedFound(out), nil
}
// deliveredOr is the reported engine build, or the current delivered one when the report names any
// build at all: what H-engine judges is that it reports, not which.
func deliveredOr(m catalogue.Manifest, reported string) string {
if reported == "" {
return reported
}
if v := deliveredVersions(m); len(v) > 0 {
return v[0]
}
return reported
}
// componentWords is a core component as a sentence names it.
func componentWords(component string) string {
switch component {
case hostModule:
return "node-engine"
case broker.RuntimeModule:
return "node tools"
case catalogue.ControllerSeatName:
return "controller"
}
return component
}
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a
// request crosses it to every machine's node tools and back.
func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
problems, err := busHealth(ctx, d)
if err != nil {
return nil, err
}
if len(problems) == 0 {
return nil, nil
}
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: "mesh", Token: "unhealthy",
Kind: kindCoreUnhealthy, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the bus is not healthy: %s", strings.Join(problems, "; ")),
Said: strings.Join(problems, "; ")}}, nil
}
// busHealth is the bus's health definition, as what is wanting: nothing when healthy. What the bus's
// planned step checks after the bus is replaced, too.
var busHealth = func(ctx context.Context, d *doctor) ([]string, error) {
if d.js == nil {
return nil, errors.New("this controller has no bus to ask")
}
streams, consumers, err := expectedBusObjects(ctx, d.open.inventory)
if err != nil {
return nil, err
}
js := d.js.Context()
var problems []string
for _, s := range streams {
if _, err := js.StreamInfo(s.Name, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
problems = append(problems, "the stream "+s.Name+" is missing")
} else if err != nil {
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
}
}
for _, c := range consumers {
_, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
if errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound) {
problems = append(problems, consumerWords(c)+" is missing")
} else if err != nil {
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
}
}
// The round trip: every machine heard from that runs the node tools answers across the bus.
running, err := d.open.inventory.Running(ctx, broker.RuntimeModule)
if err != nil {
return nil, err
}
heard := heardMachines(d)
var expected []string
for _, n := range running {
if heard[n] {
expected = append(expected, n)
}
}
if len(expected) > 0 {
answered, err := servedOnTheBus(ctx, d.js.Conn())
if err != nil {
return nil, err
}
var silent []string
for _, n := range expected {
if !answered[n].runtime {
silent = append(silent, n)
}
}
// One machine's tools silent is that machine's (H-tools); none answering is the bus.
if len(silent) == len(expected) {
slices.Sort(silent)
problems = append(problems, "no request crossed the bus and came back: no machine's node tools answered ("+
strings.Join(silent, ", ")+")")
}
}
return problems, nil
}
cmd/mesh-controller/held_back.go
-264
View File
@@ -1,264 +0,0 @@
package main
import (
"context"
"fmt"
"io"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
// issue 259, ADR 0221).
//
// A named push ends by sending every other machine whose declaration differs from what it was last
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
// out makes every machine running it differ from the merge on, and so did a module whose plan was
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
// everywhere at once.
//
// What tells the two apart is which build of each module the machine was last sent, kept with every
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
// heldBack is why a push that did not name a machine must not send it, empty when it may.
//
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
// machine was last sent — including a module the machine was never sent at all. A move is held when
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
// is not known, so a held upgrade cannot be told from anything else.
func heldBack(node string, modules []string, sent map[string]string, known bool,
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
if !known {
return []string{"which builds it was last sent is not known — it was last sent before the " +
"mesh kept them, or sent a declaration by hand"}
}
var why []string
for _, m := range modules {
now := current[m]
was, carried := sent[m]
if carried && was == now.Commit {
continue
}
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
if !now.RollOut {
why = append(why, move+", which its upgrade policy records rather than rolls out")
continue
}
if id := planStillToSend(plans, m, node); id != "" {
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
}
}
sort.Strings(why)
return why
}
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
// one holding the module that has neither finished sending it nor sent it here first, and has not
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
// per machine.
func planStillToSend(plans []inventory.Plan, module, node string) string {
for _, p := range plans {
s, holds := p.Modules[module]
if !p.Open() || !holds {
continue
}
if s == nil {
return p.ID
}
if s.SentAt != nil || s.State == "failed" {
continue
}
first := false
for _, n := range s.First {
if n == node {
first = true
}
}
if !first {
return p.ID
}
}
return ""
}
func fromBuild(was string, carried bool) string {
if !carried {
return "(never sent it) "
}
return "from " + buildName(was) + " "
}
func buildName(commit string) string {
if commit == "" {
return "a build with no source"
}
return shortCommit(commit)
}
// heldMachines reads, for each machine named, why a push that did not name it must not send it
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
// to the send, which says why.
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
out := map[string][]string{}
if len(names) == 0 {
return out, nil
}
inv := open.inventory
current, err := inv.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
plans, err := inv.OpenPlans(ctx)
if err != nil {
return nil, err
}
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, err
}
for _, node := range names {
plan, _, err := planFor(ctx, open, node)
if err != nil {
continue
}
modules := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
modules = append(modules, m.Module)
}
sent, known, err := inv.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
// A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it
// runs.
same := map[string]string{}
for m, was := range sent {
same[m] = was
if f.identical(m, was, current[m].Commit) {
same[m] = current[m].Commit
}
}
why := heldBack(node, modules, same, known, current, plans)
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
for _, mv := range f.moves(node, modules, same, known, false) {
if planStillToSend(plans, mv.Module, node) == "" {
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
}
}
if len(why) > 0 {
sort.Strings(why)
out[node] = why
}
}
return out, nil
}
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
func sayHeld(w io.Writer, node string, why []string) {
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
"grant from this push among it — waits with it. `push %s` sends it\n",
node, strings.Join(why, "; "), node)
}
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
//
// `handled` is every machine already sent or already said; it is not considered again. Answers the
// machines that could not be composed, as refusals.
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
w io.Writer) ([]string, error) {
inv := open.inventory
var refusals []string
// Bounded by the node count: a node is marked handled the round it is considered and is never
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
// cascade legitimately still converging, so it is said rather than passed over in silence.
rounds := 0
for {
would, err := wouldSend(ctx, open, nodes)
if err != nil {
return refusals, err
}
behind, err := inv.Waiting(ctx, would)
if err != nil {
return refusals, err
}
var also []string
for _, m := range behind {
if !handled[m.Node] {
also = append(also, m.Node)
}
}
if len(also) == 0 {
return refusals, nil
}
if rounds++; rounds > len(nodes) {
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
"should not happen; run `push --behind` to finish\n",
rounds-1, strings.Join(also, ", "))
return refusals, nil
}
sort.Strings(also)
held, err := heldMachines(ctx, open, also)
if err != nil {
return refusals, err
}
var sending []string
for _, name := range also {
// Every candidate this round is marked handled — the sent ones so they are not
// re-listed, the held ones because they stay held, and the refused ones so a machine
// that cannot be composed does not make the loop spin on it for ever.
handled[name] = true
if why, isHeld := held[name]; isHeld {
sayHeld(w, name, why)
continue
}
sending = append(sending, name)
}
if len(sending) == 0 {
continue
}
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
// Held for this round only, and after the last round's were given back, so two pushes
// cascading into each other's machines never each wait on the other.
refused, err := sendRound(ctx, open, sending, compose, d, holder)
refusals = append(refusals, refused...)
if err != nil {
return refusals, err
}
}
}
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
// is left out of it said, and its declaration allocated.
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
return func(held context.Context, node string) (sendable, error) {
plan, settings, err := planFor(held, open, node)
if err != nil {
return sendable{}, err
}
reportUnhostable(node, plan)
reportKept(held, plan)
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
if err == nil {
reportLeftOut(node, declared)
}
return declared, err
}
}
cmd/mesh-controller/held_back_test.go
-353
View File
@@ -1,353 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/json"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
// else that moved is still a consequence the push sends (ADR 0083).
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
current := map[string]inventory.CurrentBuild{
"resolver": {Commit: "c2c2c2c2c2"},
"agent": {Commit: "a2", RollOut: true},
"network": {},
}
modules := []string{"network", "resolver", "agent"}
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
// A module whose policy records moved: held, naming it, both builds and why.
why := heldBack("laptop", modules, sent, true, current, nil)
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
!strings.Contains(why[0], "upgrade policy records") {
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
}
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
sent["resolver"] = "c2c2c2c2c2"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
t.Fatalf("a machine whose builds are all current was held: %v", why)
}
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
sent["agent"] = "a1"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
}
// The last send's builds are not known: held whole.
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
!strings.Contains(why[0], "not known") {
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
}
// A module the machine was never sent, under a recording policy: held, and said so.
delete(sent, "resolver")
sent["agent"] = "a2"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
t.Fatalf("a module never sent under a recording policy: %v", why)
}
}
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
// naming some other machine must not send them for it.
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
at := time.Now()
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
sent := map[string]string{"agent": "a1"}
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
t.Fatalf("a machine the plan has not reached was not held: %v", why)
}
// The first machine itself was sent by the plan: not held by it.
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
t.Fatalf("the plan's first machine was held: %v", why)
}
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
Modules: map[string]*inventory.PlanModule{"agent": s}}}
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
t.Errorf("%s: not held: %v", what, why)
}
}
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
for what, plans := range map[string][]inventory.Plan{
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "failed"}}}},
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built"}}}},
} {
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
t.Errorf("%s: held: %v", what, why)
}
}
}
// A send records the build of each module it carried; a module left out of it keeps the build it
// was last sent, since the machine keeps that one.
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
current, map[string]string{"a": "a1", "b": "b1"})
if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
t.Fatalf("carried %v, wanted %v", got, want)
}
// Not known before: the left-out module is not recorded at all, so it reads as never sent.
got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
t.Fatalf("carried %v, wanted %v", got, want)
}
}
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
// reads the machine as current — and writes down which machines it declared.
type recordedDelivery struct {
inv *inventory.Inventory
declared []string
}
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
r.declared = append(r.declared, s.node)
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
}
// aResolver is a module built from a repository, at a commit, with something on the machine that
// says which build it is.
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
t.Helper()
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
}}
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
t.Fatal(err)
}
}
// A third machine on the private network: once it is sent, every other machine's peers change with
// it, which is a consequence a push must still send — no build moved.
func aThirdMachine(t *testing.T, open *stores) {
t.Helper()
ctx := t.Context()
record, err := open.inventory.AddNode(ctx, "spare")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
t.Fatal(err)
}
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
{"name": "systemd", "present": true}}})
if err != nil {
t.Fatal(err)
}
var profile map[string]any
if err := json.Unmarshal(reported, &profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
t.Fatal(err)
}
}
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := time.Now().Add(-time.Hour)
aResolver(t, open, "c1c1c1c1c1", asked)
for _, node := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
t.Fatal(err)
}
}
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
compose := composeForPush(open, gens)
d := &recordedDelivery{inv: inv}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
}
digestOfLaptop := func() string {
sent, err := inv.Outstanding(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
return sent
}
before := digestOfLaptop()
// The change merges; the policy is record. `push anchor` sends the anchor...
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
d.declared = nil
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
t.Fatal(err)
}
// ...and its cascade leaves the laptop, saying so.
var said bytes.Buffer
d.declared = nil
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
if err != nil || len(refused) != 0 {
t.Fatalf("the cascade failed: %v %v", refused, err)
}
if len(d.declared) != 0 {
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
}
if digestOfLaptop() != before {
t.Fatal("the laptop's last send moved: it was sent the held build")
}
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
"upgrade policy records", "`push laptop` sends it"} {
if !strings.Contains(said.String(), want) {
t.Errorf("the push did not say %q:\n%s", want, said.String())
}
}
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
// is held, and that what else it is owed waits with it.
aThirdMachine(t, open)
d.declared = nil
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
t.Fatal(err)
}
d.declared = nil
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil {
t.Fatal(err)
}
if len(d.declared) != 0 || digestOfLaptop() != before {
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
}
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
t.Fatalf("the push did not say both:\n%s", said.String())
}
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
// (novox/hq ADR 0236).
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
t.Fatal(err)
}
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil {
t.Fatal(err)
}
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
}
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
t.Fatal(err)
}
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
}
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
t.Fatalf("the new send did not record the new build: %v", builds)
}
}
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
// for another reason is sent by a push that names someone else.
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
compose := composeForPush(open, gens)
d := &recordedDelivery{inv: inv}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
t.Fatal(err)
}
// `push spare`, the machine just placed: the others' peers change with it.
aThirdMachine(t, open)
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
t.Fatal(err)
}
d.declared = nil
var said bytes.Buffer
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
}
if strings.Contains(said.String(), "was not sent") {
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
}
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
record, err := inv.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
t.Fatal(err)
}
d.declared = nil
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
t.Fatal(err)
}
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
// question.
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
}
}
cmd/mesh-controller/hold_test.go
+4 -4
View File
@@ -18,16 +18,16 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
plain := func(context.Context, string) (sendable, error) {
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
}
failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
fine := &recordingDelivery{}
failing := func(readyNode, []byte) error { return errors.New("the broker went away") }
fine := func(readyNode, []byte) error { return nil }
for name, round := range map[string]func() error{
"a body that cannot be marshalled": func() error {
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
return err
},
"a send that fails": func() error {
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
return err
},
} {
cmd/mesh-controller/holdings.go
-71
View File
@@ -6,8 +6,6 @@ import (
"sort"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -92,72 +90,3 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
}
return said, nil
}
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
//
// **The holders on record, and only the sole claimant when there are none** — the same answer the
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
// listed here. A holder off the private network is left out: a resolver named at an address nothing
// answers is a lookup that waits out its timeout on every name.
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
var replicated []catalogue.Seat
for _, s := range catalogue.Seats() {
if s.Replicated && s.Scope == catalogue.ScopeMesh {
replicated = append(replicated, s)
}
}
if len(replicated) == 0 {
return nil, nil
}
recorded, err := inv.Holdings(ctx)
if err != nil {
return nil, err
}
places, err := onTheNetwork(ctx, inv, shelf)
if err != nil {
return nil, err
}
address := map[string]string{}
for _, p := range places {
address[p.Name] = p.Address
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, err
}
out := map[string]map[string]string{}
for _, seat := range replicated {
var nodes []string
for _, h := range recorded {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
nodes = append(nodes, h.Node)
}
}
if len(nodes) == 0 {
var derived []string
for _, e := range entries {
for _, c := range e.Manifest.Claims {
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
derived = append(derived, e.On...)
}
}
}
if len(derived) == 1 {
nodes = derived
}
}
at := map[string]string{}
for _, n := range nodes {
if address[n] != "" {
at[overlay.InternalName(n)] = address[n]
}
}
out[seat.Name] = at
}
return out, nil
}
cmd/mesh-controller/identity_bound_test.go
-150
View File
@@ -1,150 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
// one consumer's identity.
// `module check` refuses the pull request that introduces an overflow, naming the module.
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
dir := t.TempDir()
write := func(name, body string) string {
p := filepath.Join(dir, name+".json")
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
return p
}
objects := write("objects", `{"module":"objects","version":"1",
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
resolver := write("resolver", `{"module":"resolver","version":"1",
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
var out bytes.Buffer
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
}
out.Reset()
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
if err == nil {
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
}
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
strings.Contains(out.String(), "networkmanager wants") {
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
}
}
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
// resolver provision, and a second consumer there overflows an object store's access key. The
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
// by push and by `status`, and the keyless consumer is granted with its long name.
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
Requires: []string{"wildcard-resolution"}})
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
// The consumer's machine resolves, and says which of its modules no provider will grant.
consumer, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
over := consumer.Overflowing()
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
}
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
// networkmanager, and no push to the provider could go through.
plan, settings, err := planFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
if err != nil {
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
}
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
}
grants, _, _, err := grantsFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
var keyless bool
for _, g := range grants {
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
}
if !keyless {
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
}
var granted []string
for _, c := range declared.Received["objects"]["s3-bucket"] {
granted = append(granted, c.From)
}
if strings.Join(granted, ",") != "files" {
t.Fatalf("the object store grants %v; files and only files fit", granted)
}
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
!strings.Contains(said, "left out of anchor's grants") {
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
}
// And `status` names it, and does not call the mesh well while it stands.
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
}
if asked.well() {
t.Fatal("a mesh with a consumer left out of its grants reads as well")
}
shown := printed(t, func() error { return printStatus(asked) })
if !strings.Contains(shown, "identified too long for a provision they require") ||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
t.Fatalf("status does not say it:\n%s", shown)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
Overflowing []catalogue.Overflow `json:"overflowing"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
doc.Overflowing[0].Bound.Max != 20 {
t.Fatalf("the document does not carry it: %v\n%s", err, body)
}
}
cmd/mesh-controller/issue_204_test.go
+4 -4
View File
@@ -45,11 +45,11 @@ func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *test
// fails leaves nothing composed for that machine, and the others are still composed.
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
calls := 0
allot := func(node string) (order, error) {
allot := func(node string) (int64, error) {
if node == "anchor" {
return order{}, context.DeadlineExceeded
return 0, context.DeadlineExceeded
}
return order{sequence: 7}, nil
return 7, nil
}
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
calls++
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
}
cancel() // the sender is going away: its context is cancelled between the send and the record
body := []byte(`{"declaration":1,"resources":[]}`)
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
digest, err := recordSent(ctx, inv, "anchor", body)
if err != nil {
// NodeByName on the cancelled context may itself refuse; the record must still be possible
// through the detached context, so look the node up again on a live one.
cmd/mesh-controller/lease_command_test.go
-68
View File
@@ -1,68 +0,0 @@
package main
import (
"testing"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/lease"
)
// A command run at a shell (novox/hq to-be 45 §6): under the holder's epoch while a controller holds the
// lease, read at the moment it acts; under a lease of its own while none does, given back as it ends.
func TestACommandActsUnderTheHoldersEpochOrItsOwn(t *testing.T) {
url, kv := aBusForTheLease(t)
t.Setenv(broker.NATSVar, url)
ctx := t.Context()
// Nobody holds it: the command takes it, and gives it back.
cmd := &actor{}
own, err := cmd.epoch(ctx)
if err != nil || own == 0 {
t.Fatalf("a command with nobody holding the lease acts as %d (%v)", own, err)
}
if h, found, _ := lease.Current(ctx, kv); !found || h.Epoch != own {
t.Fatalf("the command's lease is not on the bus: %+v", h)
}
cmd.release()
if _, found, _ := lease.Current(ctx, kv); found {
t.Fatal("the command did not give its lease back as it ended")
}
// A controller holds it: a command acts under that epoch.
l, err := lease.Open(ctx, mustJetStream(t, url), broker.LeaseBucket, lease.Options{Holder: lease.Holder{Instance: "serving"}})
if err != nil {
t.Fatal(err)
}
held, err := l.TryTake(ctx)
if err != nil {
t.Fatal(err)
}
borrower := &actor{}
defer borrower.release()
if got, err := borrower.epoch(ctx); err != nil || got != held {
t.Fatalf("a command acts as %d (%v), want the holder's %d", got, err, held)
}
// The holder lets go: the command does not go on under an epoch nobody holds.
l.Release(ctx)
if _, err := borrower.epoch(ctx); err == nil {
t.Fatal("a command acted under an epoch nobody holds any more")
}
}
// mustJetStream is a connection of its own to the test bus.
func mustJetStream(t *testing.T, url string) jetstream.JetStream {
t.Helper()
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
js, err := jetstream.New(conn)
if err != nil {
t.Fatal(err)
}
return js
}
cmd/mesh-controller/lease_test.go
-173
View File
@@ -1,173 +0,0 @@
package main
import (
"context"
"errors"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/testbus"
)
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
// serving controllers over one store and one bus. The second waits while the first holds the lease; on
// a handover it takes it at a higher epoch, the record says which held what and how each ended; and the
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
// moment it lost it.
//
// MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus)
// aBusForTheLease is the test bus with the controller's lease bucket new.
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
t.Helper()
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
js, err := jetstream.New(conn)
if err != nil {
t.Fatal(err)
}
_ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket)
if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil {
t.Fatal(err)
}
kv, err := js.KeyValue(t.Context(), broker.LeaseBucket)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) })
return url, kv
}
func TestTwoControllersOneActs(t *testing.T) {
url, kv := aBusForTheLease(t)
inv := inventory.ForTest(t)
ctx := t.Context()
// Controller A takes the lease.
a := &actor{}
aCtx, stopA := context.WithCancel(ctx)
defer stopA()
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
if err != nil {
t.Fatal(err)
}
epochA, err := a.epoch(ctx)
if err != nil || epochA == 0 {
t.Fatalf("A holds no epoch: %d, %v", epochA, err)
}
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
b := &actor{}
bCtx, stopB := context.WithCancel(ctx)
defer stopB()
tookB := make(chan (<-chan struct{}), 1)
go func() {
lost, err := b.serveUnderTheLease(bCtx, inv, url)
if err != nil {
t.Errorf("B: %v", err)
close(tookB)
return
}
tookB <- lost
}()
select {
case <-tookB:
t.Fatal("B took the lease while A held it")
case <-time.After(3 * time.Second):
}
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
t.Fatalf("B, waiting, may act: %v", err)
}
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
stopA()
a.release()
var lostB <-chan struct{}
select {
case lostB = <-tookB:
case <-time.After(10 * time.Second):
t.Fatal("B did not take the lease A gave back")
}
select {
case <-lostA:
default:
t.Fatal("A, having given the lease back, is not told it no longer holds it")
}
epochB, err := b.epoch(ctx)
if err != nil || epochB <= epochA {
t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err)
}
if _, err := a.epoch(ctx); err == nil {
t.Fatal("A acts after giving the lease back")
}
ea, _, _ := inv.EpochOf(ctx, epochA)
eb, _, _ := inv.EpochOf(ctx, epochB)
if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance {
t.Fatalf("the record of the handover reads %+v then %+v", ea, eb)
}
// Something else writes the lease's key — a third controller on a clock that read it as expired:
// B's next renewal is refused, and B stops acting at once.
if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil {
t.Fatal(err)
}
select {
case <-lostB:
case <-time.After(2 * lease.RenewEvery):
t.Fatal("B was not told it lost the lease")
}
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
t.Fatalf("B acts after losing the lease: %v", err)
}
// Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition
// is not raised.
inv.ActsUnder(b.epoch)
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
t.Fatal(err)
}
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
State: inventory.PlanBuilding}
if err := inv.SavePlan(ctx, &saved); err == nil {
t.Fatal("B wrote a plan after losing the lease")
}
store := conditions.NewInMemory()
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
defer keeper.Close(context.Background())
if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x",
Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil {
t.Fatal("B raised a condition after losing the lease")
}
if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost {
t.Fatalf("B's epoch does not say it was lost: %+v", ended)
}
}
// A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease:
// it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can.
func TestAControllerTheBusRefusesTheLeaseServesUnleasedAndSaysSo(t *testing.T) {
a := &actor{serving: true, unleased: "the bus refused the lease's key"}
if epoch, err := a.epoch(context.Background()); err != nil || epoch != 0 {
t.Fatalf("an unleased controller answers %d, %v: it acts, claiming no epoch", epoch, err)
}
if st := a.standing(); st.Unleased == "" || st.Held {
t.Fatalf("its standing says %+v", st)
}
// One that neither holds nor is unleased — still waiting — acts on nothing.
waiting := &actor{serving: true}
if _, err := waiting.epoch(context.Background()); !errors.Is(err, lease.ErrNotHeld) {
t.Fatalf("a controller waiting for the lease may act: %v", err)
}
}
cmd/mesh-controller/main.go
+8 -117
View File
@@ -55,9 +55,6 @@ func run() error {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
// 45 §6), so the next controller takes it at once rather than after its age.
defer theLease.release()
switch args[0] {
case "build":
@@ -76,28 +73,8 @@ func run() error {
return askCommand(ctx, args[1:])
case "builds":
return buildsCommand(ctx, args[1:])
// The build queue, controlled by hand (novox/hq ADR 0219).
case "queue":
return queueCommand(ctx, args[1:])
case "cancel":
return cancelCommand(ctx, args[1:])
case "clear":
return clearCommand(ctx, args[1:])
case "rebuild":
return rebuildCommand(ctx, args[1:])
case "replay":
return replayCommand(ctx, args[1:])
case "kill":
return killCommand(ctx, args[1:])
case "pause", "resume":
return pauseCommand(ctx, args[0], args[1:])
case "collection":
return collectionCommand(ctx, args[1:])
case "plans":
return plansCommand(ctx, args[1:])
case "delivery":
// The verbs the delivery's owner asks with (novox/hq ADR 0239).
return deliveryCommand(ctx, args[1:])
case "pin":
return pinCommand(ctx, args[1:], true)
case "unpin":
@@ -117,20 +94,8 @@ func run() error {
return brokerCommand(ctx, args[1:])
case "serve":
return serve(ctx)
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9).
case "facts":
return factsCommand(ctx, args[1:])
// The merge gate: every machine of the snapshot composed with a change (novox/hq to-be 45 §9).
case "merge-gate":
return mergeGateCommand(ctx, args[1:])
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 286).
case "check-here":
return checkHereCommand(ctx, args[1:])
case "upgrade":
return upgradeCommand(ctx, args[1:])
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
case "bus":
return busCommand(ctx, args[1:])
case "declare":
return declare(ctx, args[1:])
case "overlay":
@@ -163,31 +128,6 @@ func run() error {
return seatCommand(ctx, args[1:])
case "status":
return statusCommand(ctx, args[1:])
// Acts done by hand, and why (novox/hq to-be 45 §7).
case "hand-act":
return handActCommand(ctx, args[1:])
case "hand-acts":
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
case "durations":
return durationsCommand(ctx, args[1:])
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
case "conditions":
return conditionsCommand(ctx, args[1:])
case "doctor":
return doctorCommand(ctx, args[1:])
// What the healers did, and their brake (novox/hq to-be 45 §7).
case "healers":
return healersCommand(ctx, args[1:])
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
// (novox/hq ADR 0230).
case "retire":
return retireCommand(ctx, args[1:])
case "cleanup":
return cleanupCommand(ctx, args[1:])
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
case "data":
return dataCommand(ctx, args[1:])
case "version":
fmt.Println(version)
return nil
@@ -234,20 +174,13 @@ func usage() {
upgrade <name> roll-out [--together] ...send it to the machines running it
upgrade <name> record ...record that they are behind, and send nothing
status [--json] what is wrong, what is quiet, and what is out of date
retire [--json] every provider waiting for a person to approve a retirement (ADR 0230)
retire approve <node> <module> --why <text> retire what it waits with: access off, data kept
retire reject <node> <module> --why <text> keep them active; a warning stays open
cleanup [list] [--json] every retired consumer per provider: age, size, why
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
seats [--json] every seat this mesh defines, what it delivers, and who holds it
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
board [--listen ADDR] the same three questions, as a page that holds nothing
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
assign <node> <module>... put modules on a node, judged together (ADR 0207)
unassign <node> <module>... take them off
assign <node> <module> put a module on a node
unassign <node> <module> take it off
take <node> <module> preview a module's cutover on an adopted node: what runs beside
what it declares; --yes <digest> cuts it over as previewed
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
@@ -267,42 +200,18 @@ func usage() {
build --behind build every module the mesh holds older than its source
build --on <module> rebuild every module that stands on this module's artifacts, bases first
builds [<module>] what has been built lately, and what came of it
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
kill <id> end a build where it runs; recorded failed, killed by hand
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
hand-act record <what> --why <text> --cause <word> [--condition <key>]
record an act done by hand outside the mesh (to-be 45 §7)
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
conditions [--scope S] [--severity S] [--machine M] [--json]
what is wrong now: every open condition, urgent first (to-be 45 §2)
conditions show <key> one condition whole, with its evidence
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
conditions history [--days N] [--key K] every raising, change and clearing lately
doctor [run|probes|signals] [--json]
the self-check: the last verdict, a run now, the probes, the signals' ages
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
durations [--kind K] [--days N] [--json]
apply, heartbeat, plan-tier and build durations, per machine or module
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
builder issue <name> a broker account for a build machine, scoped to build work,
delivered as the builder module's broker secret (module add it first)
licence add|list|use|key model access, under the name a person calls it
licence manager <name> <node> the node that holds a refreshable licence's refresh token
licence refresh <name> mint a new access token and seal it to every holder
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
pin <node> <provision> <from-node> <module>
which provider this one gets a provision from: the module, and its node
unpin <node> <provision> put that question back
plan <node> [--files|--json] what that node would run, and why
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
that need it; --why records it in the hand-act log
push [<node>] [--behind] send a node everything it should be, or only those that need it
version what this binary is
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
@@ -344,31 +253,15 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
// became of a build nobody was watching.
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
// recorded or registered.
if result.Check != nil || result.Checked != nil {
checked(ctx, result)
return nil
}
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
// unmerged branch was heard here like any build, registered, and its definition reached a machine
// before anyone had reviewed it.
if result.DryRun {
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
return nil
}
manifest, _, err := takeIn(ctx, b.inv, result)
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
asked, _ := link.BuildAskedAt(result.ID)
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
// Phase 0). Said if lost; never a reason not to take the build in.
recordBuildDuration(ctx, b.inv, result, asked)
switch {
case err != nil && result.Failed != "":
fmt.Printf("%s: %v\n", result.ID, err)
if result.Module != "" {
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked)
} else {
planFailedBuild(ctx, b.open, result)
}
@@ -377,20 +270,18 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
// Not a failure: the module is already at what a later request built. A plan that asked
// before that later request is answered by it; one that asked after it ignores this.
fmt.Printf("%s: %v\n", result.ID, err)
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked)
return nil
case err != nil:
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
if manifest.Module != "" {
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked)
}
return nil
}
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
// A module registered may be one a machine is now behind: `status` is composed again.
statusFrom.nudge()
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked)
return nil
}
cmd/mesh-controller/merge_gate.go
-1290
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/merge_gate_test.go
-414
View File
@@ -1,414 +0,0 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
"github.com/novox/mesh-controller/internal/inventory"
)
// The merge gate (novox/hq to-be 45 §9): a change judged against every machine of the snapshot, by the
// incidents it is written from. Each case raises a mesh, takes its snapshot, and judges a pull request's
// tree against it in throwaway stores of its own.
// catalogueMesh is two machines and a catalogue repository: a resolver and an object store on the
// anchor, and on the laptop a network manager requiring the resolver, an album requiring the object
// store, and a login manager. Every module is registered from novox/mesh-catalog, as the mesh's are.
func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
manifests := map[string]string{
"objects": `{"module":"objects","version":"1",
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
"resolver": `{"module":"resolver","version":"1",
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`,
"networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`,
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
"lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"],
"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`,
}
for name, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
}
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"},
{"laptop", "album"}, {"laptop", "lemurs"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
f, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
return f, manifests
}
// aTree is a checkout of the catalogue repository holding these manifests.
func aTree(t *testing.T, manifests map[string]string) string {
t.Helper()
dir := t.TempDir()
for name, raw := range manifests {
at := filepath.Join(dir, "modules", name)
if err := os.MkdirAll(at, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(raw), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict {
t.Helper()
admin := os.Getenv("MESH_TEST_POSTGRES")
in := mergeCheckInput{facts: f, admin: admin, changed: changed}
if tree != "" {
in.repository, in.tree = "novox/mesh-catalog", tree
}
v, err := judgeChange(t.Context(), in)
if err != nil {
t.Fatal(err)
}
return v
}
func withEdit(manifests map[string]string, name, raw string) map[string]string {
out := map[string]string{}
for k, v := range manifests {
out[k] = v
}
if raw == "" {
delete(out, name)
} else {
out[name] = raw
}
return out
}
// The mesh as it is passes: every machine composes in the gate's store as the controller composed it.
func TestTheMeshAsItIsPassesTheGate(t *testing.T) {
f, manifests := catalogueMesh(t)
for _, m := range f.Machines {
if !m.Declaration.Composes {
t.Fatalf("the mesh itself does not compose: %+v", m.Declaration)
}
}
v := gateJudged(t, f, aTree(t, manifests))
if v.Verdict != "pass" {
t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report())
}
for _, m := range v.Machines {
if !m.Base.Composes || !m.Change.Composes {
t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change)
}
}
}
// **Issue 263**: a change makes the network manager require the object store's provision; on a machine
// whose name is six characters its identity is 26 against a bound of 20. Refused in the pull request,
// naming the module, and not on the anchor after it merged.
func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) {
f, manifests := catalogueMesh(t)
tree := aTree(t, withEdit(manifests, "networkmanager",
`{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`))
v := gateJudged(t, f, tree)
if v.Verdict != "fail" {
t.Fatalf("the overflow passed the gate:\n%s", v.Report())
}
report := v.Report()
if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") {
t.Errorf("the refusal does not name the module and the provision:\n%s", report)
}
}
// **Issue 236**: a login manager's service that omits its state passed the catalogue check and was
// refused whole by the node-engine on the first machine. A change to a module a machine runs, and a
// module nobody runs yet, are both refused before merge, naming the machine and the module.
func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) {
f, manifests := catalogueMesh(t)
broken := `{"module":"lemurs","version":"1","capabilities":["systemd"],
"resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}`
v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken)))
if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") {
t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report())
}
laptop := snapshot.Pseudonym("machine", "laptop")
if !strings.Contains(v.Report(), laptop) {
t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report())
}
// And as a new module, which no machine runs: tried on one that could.
newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`)
newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service")
v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer)))
if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") {
t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report())
}
}
// **Version skew**: a manifest the controller judging it cannot read — the one the mesh runs, for a
// catalogue change — fails here, not at registration after the merge.
func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) {
f, manifests := catalogueMesh(t)
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album",
`{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`)))
if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") {
t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report())
}
}
// A module a machine runs, removed from its source, fails until it is unassigned (ADR 0236).
func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) {
f, manifests := catalogueMesh(t)
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", "")))
if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") {
t.Fatalf("removing a module a machine runs passed:\n%s", v.Report())
}
}
// **Issues 278 and 280**: a file in no held module's directory read as shared code, and a merge rebuilt the
// catalogue. A file is a module's only by being inside it — no build reads one outside — so it rebuilds
// nothing, and the gate says why; a merge that does rebuild widely is warned of.
func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) {
f, manifests := catalogueMesh(t)
was := wideRebuild
wideRebuild = 2
t.Cleanup(func() { wideRebuild = was })
for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} {
v := gateJudged(t, f, aTree(t, manifests), file)
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" {
t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict)
}
if !strings.Contains(v.Report(), "read by no module's build") {
t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report())
}
}
v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go")
if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" {
t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict)
}
// With the reference module's definition in the tree, its directory is a module, held or not.
withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`)
v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts")
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 {
t.Fatalf("a file of a module nobody holds reads %+v", v.Width)
}
v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json")
if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" {
t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict)
}
}
// **A delivery group is judged as one future state** (novox/hq ADR 0239): a catalogue change that needs a
// provision only another repository's change adds fails alone and passes composed with it; and a group
// whose other head breaks what the first needs fails, naming it.
func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) {
f, manifests := catalogueMeshWithAnApp(t)
needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`)
catTree := aTree(t, needsTheApp)
alone := gateJudged(t, f, catTree, "modules/album/module.json")
if alone.Verdict != "fail" {
t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report())
}
app := t.TempDir()
if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1",
"provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil {
t.Fatal(err)
}
in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog",
tree: catTree, changed: []string{"modules/album/module.json"},
group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}}
together, err := judgeChange(t.Context(), in)
if err != nil {
t.Fatal(err)
}
if together.Verdict == "fail" {
t.Fatalf("the group composed together fails:\n%s", together.Report())
}
if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" {
t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules)
}
}
// catalogueMeshWithAnApp is catalogueMesh with an application built from a repository of its own, at its
// root, on the anchor.
func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
manifests := map[string]string{
"objects": `{"module":"objects","version":"1",
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
}
for name, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
}
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
f, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
return f, manifests
}
// busMesh is aMesh with a module on the bus on the laptop, its account issued as `module issue` issues
// one: minted, and its credential held as the module's own secret named broker.
func busMesh(t *testing.T) snapshot.Facts {
t.Helper()
open := aMesh(t)
ctx := t.Context()
m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1",
"own-secrets":{"broker":"/var/lib/speaker/broker"}}`))
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "laptop", "speaker"); err != nil {
t.Fatal(err)
}
user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username()
password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule,
Node: "laptop", Module: "speaker"})
if err != nil {
t.Fatal(err)
}
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker",
`{"user":"`+user+`","password":"`+password+`"}`); err != nil {
t.Fatal(err)
}
f, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
for _, mc := range f.Machines {
if !mc.Declaration.Composes {
t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration)
}
}
return f
}
// **Issue 285**: every machine running a module on the bus failed to compose in the gate's store, with
// the change and without — the store held the module's credential and no account for it, which
// composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the
// mesh issued is raised with its credential, so the machine composes in the gate as on the mesh.
func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) {
f := busMesh(t)
v := gateJudged(t, f, "")
if v.Verdict != "pass" {
t.Fatalf("the mesh as it is does not pass:\n%s", v.Report())
}
for _, m := range v.Machines {
if !m.Base.Composes {
t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems)
}
}
if !strings.Contains(v.Summary, "2 of 2 compose") {
t.Errorf("the summary does not say every machine composes: %s", v.Summary)
}
}
// **Issue 285**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
// against a machine that is not the mesh's — broken against broken, which passes whatever the change does.
// That is an error, never a pass.
func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
f := busMesh(t)
for i := range f.Machines {
// A fact the snapshot does not carry: the module's credential, gone from the laptop's.
var kept []snapshot.Accepted
for _, a := range f.Machines[i].Accepted {
if a.Name != "broker" {
kept = append(kept, a)
}
}
f.Machines[i].Accepted = kept
}
v := gateJudged(t, f, "")
if v.Verdict != "error" {
t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report())
}
if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") {
t.Errorf("the error does not name what could not be raised: %v", v.Errors)
}
}
// A path the snapshot withheld is given a path of its own where an access or a place needs one: a bare
// "withheld" is no absolute path, and a machine whose setting composes on the mesh would not in the gate.
func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
got := standInPaths(map[string]any{
"accesses": map[string]any{"races": "withheld", "films": "/media/films", "shows": "/withheld"},
"places": map[string]any{"config": map[string]any{"path": "withheld", "owner": "1000:1000"}},
"puid": 1000,
})
accesses := got["accesses"].(map[string]any)
if accesses["races"] == accesses["shows"] || !strings.HasPrefix(accesses["races"].(string), "/") ||
!strings.HasPrefix(accesses["shows"].(string), "/") || accesses["films"] != "/media/films" {
t.Errorf("accesses: %v", accesses)
}
place := got["places"].(map[string]any)["config"].(map[string]any)
if !strings.HasPrefix(place["path"].(string), "/") || place["owner"] != "1000:1000" || got["puid"] != 1000 {
t.Errorf("places: %v", got)
}
}
// **Issue 286**: a check run by hand clones beside a change what the seat clones — one rule, read by the
// controller's ask and by the facts — and finds the repository it checks from its origin.
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
for dir, refs := range map[string]map[string]string{
"mesh-catalog": {"mesh-catalog": "main"},
"mesh-host": {"mesh-host": "c0ffee"},
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
} {
got := besideRefs(dir, "c0ffee")
for d, ref := range refs {
if got[d] != ref {
t.Errorf("beside %s, %s is cloned at %q, not %q", dir, d, got[d], ref)
}
}
}
for owner, want := range map[string][3]string{
"ssh://git@git.example:222/novox/mesh-host.git": {"novox", "mesh-host", "ssh://git@git.example:222/novox"},
"git@git.example:novox/mesh-tools.git": {"novox", "mesh-tools", "git@git.example:novox"},
"http://git.example/novox/hq": {"novox", "hq", "http://git.example/novox"},
} {
o, r, p := ownerRepoOf(owner)
if [3]string{o, r, p} != want {
t.Errorf("%s reads as %s %s %s", owner, o, r, p)
}
}
}
cmd/mesh-controller/mesh_for_test.go
-20
View File
@@ -1,14 +1,11 @@
package main
import (
"context"
"crypto/ecdh"
"crypto/rand"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/novox/mesh-controller/internal/conditions"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
@@ -40,9 +37,6 @@ func aMesh(t *testing.T) *stores {
t.Fatal(err)
}
t.Cleanup(open.Close)
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
withConditionsInMemory(t)
for _, m := range provided {
if err := open.inventory.Provide(t.Context(), m); err != nil {
t.Fatal(err)
@@ -112,17 +106,3 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
}
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
t.Helper()
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
before := conditionsFrom
conditionsFrom = k
t.Cleanup(func() {
conditionsFrom = before
k.Close(context.Background())
})
return k, store
}
cmd/mesh-controller/missed_merges.go
-189
View File
@@ -1,189 +0,0 @@
package main
import (
"context"
"fmt"
"sync"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
//
// The forge's poll announces every merge on the events stream, and the controller acts on what its
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
// server moved the consumer past it — a fault of consumers with several filters in the server the
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
// built from that repository stayed behind and were built by hand.
//
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
// would still move something was never acted on — it is said, and acted on now.
const (
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
mergeGrace = 10 * time.Minute
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
mergeLookBack = 24 * time.Hour
// mergeCatchUpEvery is how often the stream is read back.
mergeCatchUpEvery = 5 * time.Minute
)
// merges is what reads back the forge's announcements; the link server, or a test's list.
type merges interface {
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
}
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
f := following{open}
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
return nil, nil, err
}
read, err := open.inventory.ReadRepositories(ctx)
return entries, read, err
}
failing := ""
tick := time.NewTicker(mergeCatchUpEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
}
watchedMerges.begin()
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
fmt.Printf(format+"\n", args...)
})
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
// says that instead, and leaves what the last one found standing.
watchedMerges.end(time.Now(), err)
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
why := ""
if err != nil {
why = err.Error()
}
if why != failing {
if why != "" {
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
} else {
fmt.Println("merges the bus may not have handed over are looked for again")
}
failing = why
}
}
}
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
// move something is said and acted on, oldest first.
//
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
// because acting on an earlier missed merge of the same repository may have moved what a later one
// would have.
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
if err != nil {
return err
}
entries, read, err := catalogued(ctx)
if err != nil {
return err
}
for _, a := range all {
if now.Sub(a.At) < mergeGrace {
continue
}
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
continue
}
if entries, read, err = catalogued(ctx); err != nil {
return err
}
moves := wouldMove(a.SourceMoved, entries, read)
if len(moves) == 0 {
continue
}
var names []string
for _, e := range moves {
names = append(names, e.Manifest.Module)
}
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
At: a.At, Modules: names})
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
isAre(len(names)))
if err := act(ctx, a.SourceMoved); err != nil {
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
a.Owner, a.Repo, a.Commit, err)
continue
}
if entries, read, err = catalogued(ctx); err != nil {
return err
}
}
return nil
}
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
type missedMerge struct {
Owner, Repo, Base, Commit string
// At is when the bus took the announcement.
At time.Time
Modules []string
}
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
// the fault. The next pass, finding it acted on, clears it.
type mergeWatch struct {
mu sync.Mutex
passed time.Time
err error
finding []missedMerge
missed []missedMerge
}
var watchedMerges = &mergeWatch{}
func (w *mergeWatch) begin() {
w.mu.Lock()
defer w.mu.Unlock()
w.finding = nil
}
func (w *mergeWatch) found(m missedMerge) {
w.mu.Lock()
defer w.mu.Unlock()
w.finding = append(w.finding, m)
}
func (w *mergeWatch) end(at time.Time, err error) {
w.mu.Lock()
defer w.mu.Unlock()
w.passed, w.err = at, err
if err == nil {
w.missed = w.finding
}
}
// last is when the last pass ended, what the last pass that read found, and what the last pass
// could not read.
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
w.mu.Lock()
defer w.mu.Unlock()
return w.passed, append([]missedMerge(nil), w.missed...), w.err
}
cmd/mesh-controller/missed_merges_test.go
-180
View File
@@ -1,180 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// announcedList is the events stream's merges as a test gives them.
type announcedList []link.AnnouncedMerge
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
var out []link.AnnouncedMerge
for _, m := range a {
if !m.At.Before(since) {
out = append(out, m)
}
}
return out, nil
}
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
type aCatalogue struct {
entries []inventory.Entry
acted []string
fail error
}
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
return append([]inventory.Entry(nil), c.entries...), nil, nil
}
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
return func(_ context.Context, m link.SourceMoved) error {
if c.fail != nil {
return c.fail
}
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
for _, moved := range wouldMove(m, c.entries, nil) {
for i := range c.entries {
if c.entries[i].Manifest.Module == moved.Manifest.Module {
c.entries[i].Source.Head = m.Commit
c.entries[i].Source.Seen = now()
}
}
}
return nil
}
}
func at(s string) time.Time {
t, err := time.Parse(time.RFC3339, s)
if err != nil {
panic(err)
}
return t
}
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
return link.AnnouncedMerge{
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
MergedAt: mergedAt, Paths: paths,
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
At: at(onTheBus),
}
}
func built(module, repo, path, commit, seen string) inventory.Entry {
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
return e
}
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
// events stream; the bus never handed it to the controller, which acted on the merges around it and
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
// move something — so it is said and acted on, once, and only after the controller's own consumer
// has had its time with it.
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
cat := &aCatalogue{entries: []inventory.Entry{
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
// Acted on when it was announced: looked at after it was merged.
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
}}
stream := announcedList{
// Nothing the mesh holds is built from the records repository.
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
// Acted on: its module was looked at since.
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
// Never handed over.
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
"node-tools/internal/console/console.go"),
}
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
clock := at("2026-10-05T22:50:00Z")
now := func() time.Time { return clock }
pass := func() {
t.Helper()
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
}
pass()
if len(cat.acted) != 0 {
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
}
clock = at("2026-10-05T22:58:00Z")
pass()
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
}
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
t.Fatalf("the missed merge was not said as one: %q", said)
}
clock = at("2026-10-05T23:03:00Z")
pass()
if len(cat.acted) != 1 || len(said) != 1 {
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
}
}
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
// that could not be acted on is said and tried again on the next pass.
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
cat := &aCatalogue{
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
fail: errors.New("the store is restarting"),
}
stream := announcedList{
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
"modules/plex/module.json", "modules/plex/x.ts"),
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
}
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
clock := at("2026-10-05T22:00:00Z")
now := func() time.Time { return clock }
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
t.Fatalf("a failed catch-up was not said: %q", said)
}
cat.fail = nil
clock = at("2026-10-05T22:05:00Z")
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
}
}
// A merge older than the look-back is left to the operator: a controller that did not run this
// missed it, and acting on it days later would be a surprise rebuild.
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
clock := at("2026-10-05T22:00:00Z")
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
func(string, ...any) {}); err != nil {
t.Fatal(err)
}
if len(cat.acted) != 0 {
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
}
}
cmd/mesh-controller/modules.go
+9 -27
View File
@@ -40,12 +40,7 @@ func providedModules() []catalogue.Manifest {
// and neither could be swapped for anything, which is the test of whether a thing is a
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
// to live, and now a module says where it wants it — `facts` in its own manifest.
//
// **And the bundle that required it is gone** (novox/hq ADR 0226): `networking` named this
// module's requirement and nothing else, so every machine carried two modules for one
// network. A machine is assigned the private network itself; what a release stops shipping
// is retired at the next start (stores.go, Inventory.RetireUnshipped).
overlay.Manifest(),
overlay.Manifest(), overlay.DomainManifest(),
} {
var m catalogue.Manifest
b, _ := json.Marshal(raw)
@@ -64,19 +59,8 @@ func moduleCommand(ctx context.Context, args []string) error {
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
if args[0] == "check" {
set := flag.NewFlagSet("module check", flag.ContinueOnError)
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
"judge each module's identity on a machine name this many characters long")
given, err := parseAround(set, args[1:])
if err != nil {
return err
}
if *longest < 1 {
return errors.New("--longest-machine-name is a length, at least 1")
}
var paths []string
for _, a := range given {
for _, a := range args[1:] {
if info, err := os.Stat(a); err == nil && info.IsDir() {
under, err := manifestsUnder(a)
if err != nil {
@@ -87,7 +71,7 @@ func moduleCommand(ctx context.Context, args []string) error {
}
paths = append(paths, a)
}
return moduleCheckFor(paths, *longest, os.Stdout)
return moduleCheck(paths, os.Stdout)
}
open, err := openStores(ctx)
if err != nil {
@@ -350,10 +334,8 @@ func moduleCommand(ctx context.Context, args []string) error {
}
func assignCommand(ctx context.Context, verb string, args []string) error {
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
// service manager and the package manager — can only go on, or come off, together.
if len(args) < 2 {
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
if len(args) != 2 {
return fmt.Errorf("%s <node> <module>", verb)
}
open, err := openStores(ctx)
if err != nil {
@@ -367,7 +349,7 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
if verb == "unassign" {
act = unassign
}
said, err := act(ctx, open, args[0], args[1:]...)
said, err := act(ctx, open, args[0], args[1])
if said != "" {
fmt.Println(said)
}
@@ -510,8 +492,8 @@ const theBrokerSeat = "mesh-broker"
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
// has adopted it) does the hub stand in, which is where the foundation is by convention.
//
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the private network
// — not "has an address", which is true of every placed machine and says nothing about
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the networking
// module — not "has an address", which is true of every placed machine and says nothing about
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
@@ -677,7 +659,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
// durable subscription nobody reads.
if consumer, needed := broker.ConsumerFor(broker.Principal{
Kind: broker.KindModule, Node: node, Module: m.Module,
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
}); needed {
if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
cmd/mesh-controller/network.go
+18 -2
View File
@@ -275,9 +275,25 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
if err != nil {
return nil, err
}
// No registry trust is composed here any more: the container runtime's module states it, told
// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
g, err := overlay.From(nodes, cidr, "")
if g != nil {
// The artifact store, as this network reaches it. Found rather than configured: the
// provider is whichever module offers it, on whichever machine holds that module — and if
// nothing does yet (genesis raises the registry before the catalogue knows it), there is
// no trust to write and nothing is written (novox/hq ADR 0082).
//
// Refused rather than composed without it when the question could not be answered: a
// declaration missing the trust because a lookup failed is a machine that cannot pull,
// delivered by a push that reported success — and nothing recomposes it until the next
// push (the shape of novox/hq issues 042/048, reappearing as a race).
at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
if storeErr != nil {
return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
}
if found {
g.TrustRegistry(overlay.InternalName(at) + ":" + port)
}
}
if err != nil && len(refused) > 0 {
// The network is missing something, and some machines could not be resolved at all. Those
// are almost always the same fact: a node that does not resolve contributes nothing, so
cmd/mesh-controller/network_test.go
+8 -8
View File
@@ -254,10 +254,11 @@ func theResolver(t *testing.T) catalogue.Manifest {
return m
}
// The resolver is handed every machine on the private network as a wildcard, and is handed it again
// when a machine leaves — through the module's own manifest asking for the fact, with no module of the
// mesh's own in between (hal dnsmasq-app conversion, novox/hq 08-connectivity). It is the mesh's one
// resolver (ADR 0194), and the container runtime is given no resolver of its own (ADR 0196).
// The resolver is handed every machine on the private network as a wildcard, the same set and the
// same source as the hosts file, and is handed it again when a machine leaves — through the
// module's own manifest asking for the fact, with no module of the mesh's own in between (hal
// dnsmasq-app conversion, novox/hq 08-connectivity). The runtime on that machine is pointed at the
// machine's own address, where the resolver answers for its containers.
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
@@ -292,12 +293,11 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
}
}
// The container runtime is given no resolver of its own (novox/hq ADR 0196): it copies its
// machine's, which name the mesh's resolver first. A `dns` key would be a second account of where a
// container asks, read only when the runtime starts.
for _, r := range composed(t, open, "anchor").Resources {
if r["id"] == "dnsmasq.runtime-dns" {
t.Errorf("the resolver still writes the runtime's own dns: %v", r)
if !strings.Contains(r["content"].(string), `"10.77.0.1"`) || r["into"] != "json" {
t.Errorf("the runtime is not pointed at this machine's own address, written into its file: %v", r)
}
}
}
cmd/mesh-controller/nodes.go
+1 -65
View File
@@ -6,7 +6,6 @@ import (
"errors"
"flag"
"fmt"
"github.com/novox/mesh-controller/internal/conditions"
"strings"
"time"
@@ -387,12 +386,8 @@ func brokerCommand(ctx context.Context, args []string) error {
if len(args) > 0 && args[0] == "accounts" {
return busAccounts(ctx, args[1:])
}
if len(args) > 0 && args[0] == "consumer-reset" {
return consumerReset(ctx, args[1:])
}
if len(args) == 0 || args[0] != "show" {
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
"broker consumer-reset <stream> <consumer>")
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file>")
}
known, err := broker.FromEnvironment()
if errors.Is(err, broker.ErrNotConfigured) {
@@ -412,45 +407,6 @@ func brokerCommand(ctx context.Context, args []string) error {
return nil
}
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
func consumerReset(ctx context.Context, args []string) error {
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
// A repair by hand, which says why (novox/hq to-be 45 §7).
why := addHandActFlags(set)
args, err := parseAround(set, args)
if err != nil {
return err
}
if len(args) != 2 {
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
}
if err := why.require("broker consumer-reset"); err != nil {
return err
}
why.record(ctx, "broker consumer-reset", args)
address, err := broker.BusAddress()
if err != nil {
return err
}
js, err := broker.Dial(address)
if err != nil {
return fmt.Errorf("cannot reach the bus: %w", err)
}
defer js.Close()
before, after, err := js.ResetConsumer(args[0], args[1])
if err != nil {
return err
}
fmt.Printf("consumer %s on %s re-made to deliver from now\n", args[1], args[0])
fmt.Printf(" before: delivers %s, delivered to %d, acknowledged to %d, %d pending, %d unacknowledged\n",
before.DeliverPolicy, before.Delivered, before.AckFloor, before.Pending, before.AckPending)
fmt.Printf(" after: delivers %s, %d pending; what was pending is dropped. A holder bound to it may need its "+
"process restarted to bind again\n", after.DeliverPolicy, after.Pending)
return nil
}
// heardFrom says when a node was last heard from, in a form somebody can act on.
//
// "never" and "an hour ago" are different answers and are kept different. A node that has never
@@ -517,26 +473,6 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
fmt.Printf(" public domain %s\n", domain)
}
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
// is something not working now and they are a description. Unreadable is said, not passed over.
open, err := openConditions(ctx)
if err != nil {
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
}
var here []conditions.Condition
for _, c := range open {
if concerns(c, name) {
here = append(here, c)
}
}
if len(here) > 0 {
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
for _, line := range conditionLines(here, time.Now()) {
fmt.Printf(" %s\n", line)
}
}
held, err := inv.Profile(ctx, name)
if err != nil {
return err
cmd/mesh-controller/order_test.go
+1 -76
View File
@@ -1,7 +1,6 @@
package main
import (
"encoding/json"
"reflect"
"strings"
"testing"
@@ -144,18 +143,10 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
}{
{"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"},
{"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"},
{"a file at the root no build reads (issue 280)", merge([]string{"tsconfig.json"}, false), ""},
{"a file they share", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
// novox/hq issue 252: a module the mesh has never registered is still a module, when the merge
// shows it is one — and a directory that may be shared code is still shared.
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), ""},
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""},
{"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"},
} {
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
@@ -244,69 +235,3 @@ func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) {
t.Error("a module following another branch was matched")
}
}
// novox/hq issues 278 and 280: a merge touching the code of a module the mesh does not hold — the
// catalogue's reference module, whose manifest it left alone — read as shared and rebuilt every module
// built from the repository (103 of them on 2026-10-06, 88 byte-identical); so did a merge-check.sh added at
// the root. No build reads a file outside its module's directory, so neither rebuilds anything, said by the
// announcer or not.
func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) {
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
gitea := fromRepo("gitea", repo, "modules/gitea")
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
known := []inventory.Entry{gitea, keycloak}
candidates := []inventory.Entry{gitea, keycloak}
merge := func(paths, modules []string, said bool) link.SourceMoved {
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Paths: paths,
ModuleDirs: modules, ModuleDirsSaid: said}
}
named := func(entries []inventory.Entry) string {
var names []string
for _, e := range entries {
names = append(names, e.Manifest.Module)
}
return strings.Join(names, ",")
}
showcase := []string{"modules/showcase/index.ts"}
for _, c := range []struct {
what string
m link.SourceMoved
want string
}{
{"a module held by none, said", merge(showcase, []string{"modules/showcase"}, true), ""},
{"beside a held one's change", merge(append([]string{"modules/gitea/index.ts"}, showcase...),
[]string{"modules/gitea", "modules/showcase"}, true), "gitea"},
{"deeper inside it", merge([]string{"modules/showcase/daemon/index.ts"}, []string{"modules/showcase"}, true), ""},
{"a directory holding no manifest, read by no build", merge([]string{"modules/lib/x.go"}, nil, true), ""},
{"one of two files in no module", merge([]string{"modules/showcase/index.ts", "modules/lib/x.go"},
[]string{"modules/showcase"}, true), ""},
{"the root is never a module directory", merge([]string{"tsconfig.json"}, []string{"", "/", "."}, true), ""},
{"not said: still no build reads it", merge(showcase, []string{"modules/showcase"}, false), ""},
{"an old announcer saying nothing", merge(showcase, nil, false), ""},
{"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""},
} {
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
}
}
}
// What the announcer says reaches the controller as it is sent: the two fields, by their names on the
// wire, and an older announcement without them reads as not said.
func TestTheAnnouncerSaysWhichDirectoriesAreModules(t *testing.T) {
var m link.SourceMoved
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc",`+
`"paths":["modules/showcase/index.ts"],"module_dirs":["modules/showcase"],"module_dirs_said":true}`), &m); err != nil {
t.Fatal(err)
}
if !m.ModuleDirsSaid || !reflect.DeepEqual(m.ModuleDirs, []string{"modules/showcase"}) {
t.Fatalf("the announcer's word was lost: %+v", m)
}
var old link.SourceMoved
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc","paths":["x"]}`), &old); err != nil {
t.Fatal(err)
}
if old.ModuleDirsSaid || old.ModuleDirs != nil {
t.Fatalf("an older announcement says nothing about module directories: %+v", old)
}
}
cmd/mesh-controller/plan.go
+54 -403
View File
@@ -8,10 +8,8 @@ import (
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"sync"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
@@ -90,12 +88,6 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
if err != nil {
return catalogue.Resolution{}, nil, err
}
// Where each of its consumers of a provision that keeps data was last sent (novox/hq ADR 0232):
// a resolution that would answer one from anywhere else keeps it there, and says so.
world.Bound, err = inv.BindingsFor(ctx, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
@@ -135,7 +127,6 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
// a mesh-wide gatherer may pass over — see notResolvable.
return catalogue.Resolution{}, nil, notResolvable{err}
}
logUnheld(nodeName, resolved.Unheld)
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
// password a provider is told to create is the one its consumer was given — and sealed to
@@ -378,19 +369,13 @@ func declarationFor(ctx context.Context, open *stores, node string,
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
// machine "waiting" means — the read needs no number to be right about that.
type Choosing int
type Choosing bool
const (
// Reading is every question: what is assigned is used, and nothing is created.
Reading Choosing = iota
// Allocating is the send path: what is not assigned yet is assigned now and kept.
Allocating
// Foreseeing is Reading, asked ahead of a send (the self-check's D1, novox/hq issue 275): nothing
// is created, and an own secret the next send WOULD make is composed with a stand-in and named
// (sendable.foreseen) rather than failing the composition — while one the send would be refused
// (a bus credential nobody issued, a given secret under an old key) is refused here as it would
// be there. Never sent: the stand-in is not a sealed value.
Foreseeing
Allocating Choosing = true
// Reading is every question: what is assigned is used, and nothing is created.
Reading Choosing = false
)
func declarationWith(ctx context.Context, open *stores, node string,
@@ -409,75 +394,9 @@ func declarationWith(ctx context.Context, open *stores, node string,
if err != nil {
return sendable{}, err
}
out := sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
unbound: with.Unbound, foreseen: composed.Foreseen}
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
// 0221). Read only on the send path: a question about what would be sent records nothing.
if choosing == Allocating {
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return sendable{}, err
}
before, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return sendable{}, err
}
if !known {
before = nil
}
names := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
names = append(names, m.Module)
}
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
out.Bindings = boundToData(plan, composed.LeftOut)
}
return out, nil
}
// boundToData is every binding of this machine's consumers to a provision that keeps their data
// (novox/hq ADR 0232), as a send records it. Not a consumer left out of the declaration: the machine
// is not told anything new about it, so nothing about where it is bound has been sent.
func boundToData(plan catalogue.Resolution, leftOut map[string]string) []inventory.Binding {
var out []inventory.Binding
seen := map[[2]string]bool{}
for _, n := range plan.Needs {
if !n.KeepsData || n.ByRecord || n.Module == "" {
continue
}
if _, left := leftOut[n.For]; left {
continue
}
key := [2]string{n.For, n.Name}
if seen[key] {
continue
}
seen[key] = true
out = append(out, inventory.Binding{Machine: plan.Node, Consumer: n.For, Provision: n.Name,
Provider: catalogue.Chosen{Node: n.From, Module: n.Module}})
}
return out
}
// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
// issue 259): the module's current build for each module in it, and for a module left out of it
// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
// that is not known. Never nil, so a send through here always records what it knows.
func carriedBuilds(modules []string, leftOut map[string]string, current map[string]inventory.CurrentBuild,
before map[string]string) map[string]string {
out := map[string]string{}
for _, m := range modules {
if _, left := leftOut[m]; left {
if was, kept := before[m]; kept {
out[m] = was
}
continue
}
out[m] = current[m].Commit
}
return out
return sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh,
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
}
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
@@ -502,39 +421,6 @@ func reportLeftOut(node string, declared sendable) {
"what the machine holds for it is kept and its containers are untouched. %s\n",
node, m, declared.leftOutWhy[m])
}
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
// 0225): the machine is sent everything else, and the consumer is named.
for _, o := range declared.withheld {
fmt.Printf("%s: %s\n", node, o)
}
// And whom it no longer serves because they are bound elsewhere (novox/hq issue 274).
for _, u := range declared.unbound {
fmt.Printf("%s: %s\n", node, u)
}
}
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
//
// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is
// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted
// by `module issue` and sealed by it; a push that made a random one would deliver a file the process
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
if name != "broker" {
return nil
}
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return err
} else if !minted {
return fmt.Errorf(
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
"`module issue %s --node %s`, then push again",
module, node, user, module, node)
}
return nil
}
// renderingFor is everything a node's declaration is composed with, and the node's record.
@@ -542,7 +428,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
plan catalogue.Resolution, settings catalogue.SettingsBy,
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
inv := open.inventory
grants, withheld, unbound, err := grantsFor(ctx, open, node)
grants, err := grantsFor(ctx, open, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
@@ -554,7 +440,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
// consumer is told are all derived from it.
// What this machine was already given, for a composition that may not allocate.
already := map[string]map[int]int{}
if choosing != Allocating {
if choosing == Reading {
held, err := inv.PortsFor(ctx, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
@@ -640,7 +526,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
// And each module's own secrets — a superuser password, an administrator, an account. Made
// per node, so a module running on three machines has three.
needed := map[string]map[string]string{}
foreseen := map[string]map[string]bool{}
for _, m := range plan.Modules {
for name := range m.OwnSecrets {
// Minted on the send path and only read on every other. Making one is an insert, and
@@ -648,29 +533,27 @@ func renderingFor(ctx context.Context, open *stores, node string,
var sealed string
var err error
if choosing == Allocating {
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
// **The broker credential is never invented here** (novox/hq issue 203). Every other
// own secret is the mesh's to make — a password nobody else knows — but this one
// is an account on the bus, minted by `module issue` and sealed by it; a push that
// made a random one would deliver a file the process cannot read and report the
// machine applied. Refused by name, with the verb.
if name == "broker" {
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
} else if !minted {
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
"`module issue %s --node %s`, then push again",
m.Module, node, user, m.Module, node)
}
}
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
} else {
var held bool
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
if err == nil && !held && choosing == Foreseeing {
// Asked ahead of the send: what the send would do about it, by the send's own
// rules. Made by it — a stand-in, named; refused by it — refused here, the same
// words (novox/hq issue 275).
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if foreseen[m.Module] == nil {
foreseen[m.Module] = map[string]bool{}
}
foreseen[m.Module][name] = true
continue
}
if err == nil && !held {
// Never issued, so this machine cannot be running it. Left out rather than
// invented: an empty string here would compose a declaration that differs
@@ -774,20 +657,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
machines[name] = at
}
// And every zone a module in the mesh answers itself (novox/hq ADR 0199), for the mesh's resolver
// to forward.
zones, err := zonesInTheMesh(ctx, open)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
// lists every holder of the mesh's resolver.
replicas, err := replicatedHolders(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -853,90 +722,18 @@ func renderingFor(ctx context.Context, open *stores, node string,
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
// 0222): the artifact store as this network reaches it, which the container runtime is told to
// trust (ADR 0082). The same address composed into every reference the mesh built.
var reach map[string]string
if artifactStore != "" {
reach = map[string]string{"mesh-artifact-store": artifactStore}
}
return catalogue.Rendering{
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines, Zones: zones, Holders: replicas,
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
Machines: machines,
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
BusUsers: busUsers, Withheld: withheld, Unbound: unbound,
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
BusUsers: busUsers,
}, record, nil
}
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
// 0199): the zone settled from that node's settings, the node's private address, the port the
// answering listen is published on there.
//
// Read across every machine's resolution, as the roster once read routed names: a node whose set does
// not compose declares nothing and is passed over, so one broken machine does not cost the rest their
// zones; a store that cannot be read is raised, naming the machine, because returning the zones
// without it would withdraw them from the resolver as if the operator had (novox/hq 04-ISSUES/152).
// What the mesh refuses about the zones together — one declared twice, one shadowing the mesh's
// suffix or a node's public domain — is refused here, by name.
func zonesInTheMesh(ctx context.Context, open *stores) ([]catalogue.ZoneAt, error) {
inv := open.inventory
places, err := inv.Overlays(ctx)
if err != nil {
return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
}
address := map[string]string{}
for _, p := range places {
if strings.TrimSpace(p.Address) != "" {
address[p.Name] = p.Address
}
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
}
var zones []catalogue.ZoneAt
var public []string
for _, n := range nodes {
plan, _, err := planFor(ctx, open, n.Name)
switch {
case unresolvable(err):
continue
case err != nil:
return nil, fmt.Errorf("the zones %s answers cannot be read: %w", n.Name, err)
}
if plan.PublicDomain != "" {
public = append(public, plan.PublicDomain)
}
for _, m := range plan.Modules {
if m.Zone == nil {
continue
}
at := address[n.Name]
if at == "" {
// Not on the private network yet: nothing could reach its answerer.
continue
}
published, layers, err := portsGivenOn(ctx, inv, n.Name, m)
if err != nil {
return nil, fmt.Errorf("the zone %s declares on %s cannot be read: %w", m.Module, n.Name, err)
}
z, err := catalogue.ZoneOn(m, layers, published, n.Name, at)
if err != nil {
return nil, err
}
zones = append(zones, *z)
}
}
if problems := catalogue.ZonesProblems(zones, overlay.Suffix(), public); len(problems) > 0 {
return nil, fmt.Errorf("the mesh's zones cannot be forwarded:\n - %s", strings.Join(problems, "\n - "))
}
return zones, nil
}
// certificateFor is what the mesh certifies about one machine's internal name.
//
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
@@ -1004,45 +801,28 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
// The mirror of what a consumer is given, and the half that makes the credential real: a password
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
// the mesh hands over something it cannot itself use.
//
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
//
// **A provider is granted exactly the consumers whose own resolution binds them to it** (novox/hq
// issue 274). The pair credentials on record say only whom this node was ever asked by: after a
// consumer of a provision that keeps its data was moved and pinned back (issue 273, ADR 0232), the
// credential from the provider it left was still on record, so that provider went on being asked for
// five databases nobody used and never retired them. A credential whose consumer is bound elsewhere
// is withdrawn here like one nobody asks for — the provider retires it and keeps its data (ADR 0230)
// — and returned beside the grants, for plan and push to say. It stays on record: it is the key to
// the login the provider keeps until `cleanup delete`, and to that data should a person pin it back.
func grantsFor(ctx context.Context, open *stores, node string) (
[]catalogue.Grant, []catalogue.Overflow, []catalogue.Unbound, error) {
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
inv := open.inventory
issued, err := inv.SecretsFrom(ctx, node)
if err != nil {
return nil, nil, nil, err
return nil, err
}
// Where each consumer is, so a provider that must reach back to one does not have to know how
// the mesh names machines.
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, nil, nil, err
return nil, err
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return nil, nil, nil, err
return nil, err
}
// What each consumer actually asked for, taken from that machine's own resolution rather than
// from a record beside it. A provider told to create a password and not what to create it for
// can do nothing with it, and the name a consumer wants is the consumer's to say.
out := make([]catalogue.Grant, 0, len(issued))
var withheld []catalogue.Overflow
var unbound []catalogue.Unbound
for _, s := range issued {
plan, settings, err := planFor(ctx, open, s.Consumer)
switch {
@@ -1055,11 +835,11 @@ func grantsFor(ctx context.Context, open *stores, node string) (
// The mesh could not be asked what they wanted, which is not the same as their wanting
// nothing — and withholding a grant on that reading takes a consumer's access away
// (novox/hq 04-ISSUES/152).
return nil, nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
}
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
if err != nil {
return nil, nil, nil, err
return nil, err
}
// A port in there is the consumer's software port until this. The consumer is on another
// machine, so the assignment that moved it is that machine's — fetched here rather than
@@ -1067,7 +847,7 @@ func grantsFor(ctx context.Context, open *stores, node string) (
// this case (novox/hq 04-ISSUES/038, the cross-node half).
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
if err != nil {
return nil, nil, nil, err
return nil, err
}
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
from := s.ConsumerModule
@@ -1077,20 +857,10 @@ func grantsFor(ctx context.Context, open *stores, node string) (
// working for ever after its consumer went away.
from = ""
}
if bound := plan.BindsFrom(s.Name, s.ConsumerModule, s.Local); from != "" && !slices.Contains(bound, node) {
// It still asks, and not of this node: its resolution — the same one read above, so an
// unreadable one is the error above and never an empty answer here (issue 152) — binds
// this credential to another provider, or under this local name to none. Withdrawn
// exactly as a credential nobody asks for, and said.
from = ""
unbound = append(unbound, catalogue.Unbound{Provision: s.Name, Provider: node,
Consumer: s.Consumer, Module: s.ConsumerModule, Local: s.Local, BoundTo: bound})
}
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
// this provision, as the consumer's resolution states it from the provider's offer (ADR
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
// provider silently shortened — and rather than this whole machine refused for it.
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
// remedy a short slug rather than a login a provider silently shortened.
slug := ""
for _, mm := range plan.Modules {
if mm.Module == s.ConsumerModule {
@@ -1099,32 +869,15 @@ func grantsFor(ctx context.Context, open *stores, node string) (
}
}
if from != "" {
bound := boundOfGrant(plan, s, node)
source := catalogue.IdentitySource(slug, s.ConsumerModule)
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
Consumer: s.Consumer, Module: s.ConsumerModule,
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
continue
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
return nil, err
}
}
out = append(out, catalogue.Grant{
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
}
return out, withheld, unbound, nil
}
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
// than to none: what the provider keeps of it is not known here.
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
for _, n := range consumer.Needs {
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
return n.Identity
}
}
return catalogue.DefaultIdentityBound
return out, nil
}
// listensLines is what a person is told about what this module would open, and why — the same
@@ -1200,11 +953,6 @@ func planCommand(ctx context.Context, args []string) error {
left := plan.LeftOut(settings, record.Adopted)
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
}
// And which of its modules no provider will grant, because the identity overflows the bound of
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
for _, o := range plan.Overflowing() {
fmt.Printf("%s: %s\n", args[0], o)
}
// And a setting that reaches nothing — refused where it is stored, and said here for one
// stored before its definition moved from under it.
for _, m := range plan.Modules {
@@ -1518,20 +1266,6 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
//
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
// resource of that module, so the question is whether it is here.
list, missing, err := busUserList(ctx, inv, onThisNode)
if len(missing) > 0 {
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
"for: %s. Each is a user that cannot connect until one is issued\n",
len(missing), strings.Join(missing, ", "))
}
return list, err
}
// busUserList is composeBusUsers without saying anything: the list, and the users left out of it
// for want of a credential. Asked on every send to decide whether the machine holding the bus must
// go first (novox/hq issue 249), where saying the same missing users each time would bury them.
func busUserList(ctx context.Context, inv *inventory.Inventory,
onThisNode []catalogue.Manifest) (string, []string, error) {
holdsTheBus := false
for _, m := range onThisNode {
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
@@ -1539,33 +1273,37 @@ func busUserList(ctx context.Context, inv *inventory.Inventory,
}
}
if !holdsTheBus {
return "", nil, nil
return "", nil
}
records, err := inv.BusRecords(ctx)
if err != nil {
return "", nil, err
return "", err
}
users, err := broker.Users(records)
if err != nil {
return "", nil, err
return "", err
}
kept, err := inv.BusUsers(ctx)
if err != nil {
return "", nil, err
return "", err
}
hashes := make(map[string]string, len(kept))
for name, u := range kept {
hashes[name] = u.PasswordHash
}
filled, missing := broker.WithPasswords(users, hashes)
if len(missing) > 0 {
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
"for: %s. Each is a user that cannot connect until one is issued\n",
len(missing), strings.Join(missing, ", "))
}
if len(filled) == 0 {
return "", missing, fmt.Errorf(
return "", fmt.Errorf(
"this machine runs the bus and not one user has a credential, so the composed list " +
"would refuse every connection in the mesh")
}
list, err := broker.ComposeAccounts(filled)
return list, missing, err
return broker.ComposeAccounts(filled)
}
// providerModuleOf is which module answers a need on the providing node: the one in this node's
@@ -1587,90 +1325,3 @@ func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.C
}
return ""
}
// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
//
// **The serving controller's log only.** planFor runs for every node on every push, assignment and
// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
// burying the line about the node it acted on. A command says what concerns its own act instead
// (unheldChange, reportUnheldPushed); the full list is `status`'s.
var (
unheldLogged = map[string]string{}
unheldLoggedMu sync.Mutex
logUnheldChanges bool
)
// logUnheld logs a node's unmet seat dependencies when they differ from what was last logged for
// it, including when they become none — in the serving controller, and nowhere else.
func logUnheld(node string, unheld []catalogue.Unheld) {
if !logUnheldChanges {
return
}
lines := make([]string, 0, len(unheld))
for _, u := range unheld {
lines = append(lines, u.String())
}
now := strings.Join(lines, "\n")
unheldLoggedMu.Lock()
before, seen := unheldLogged[node]
unheldLogged[node] = now
unheldLoggedMu.Unlock()
if (seen && before == now) || (!seen && now == "") {
return
}
if now == "" {
fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
return
}
fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n %s\n",
node, len(lines), strings.Join(lines, "\n "))
}
// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over
// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not —
// which includes every one of a module just assigned — and each now met that was not. Nothing about
// any other node, and nothing that was already true before the act.
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
judge := func(names []string) map[string]catalogue.Unheld {
var set []catalogue.Manifest
for _, n := range names {
if m, known := shelf[n]; known {
set = append(set, m)
}
}
out := map[string]catalogue.Unheld{}
for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
out[u.Module+" "+u.Seat] = u
}
return out
}
was, now := judge(before), judge(after)
var lines []string
for _, k := range sortedNames(now) {
if _, already := was[k]; !already {
lines = append(lines, "but "+now[k].String())
}
}
for _, k := range sortedNames(was) {
if _, still := now[k]; still {
continue
}
u := was[k]
if !slices.Contains(after, u.Module) {
continue // went with its module, which says nothing about the seat
}
lines = append(lines, fmt.Sprintf("and %s on %s now has %s held", u.Module, node, u.Seat))
}
return lines
}
func sortedNames[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
cmd/mesh-controller/plan_retry.go
-506
View File
@@ -1,506 +0,0 @@
package main
import (
"context"
"fmt"
"os"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A plan follows what is done to the build queue (novox/hq ADR 0219).
//
// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
//
// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
// sends a reader looking for a fault that is a decision. It says what it waits on instead.
// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
// failed plan's failed modules and the plan goes on from that tier as if they had built the
// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
// beside it — beside it, the plan would either ask it again or stay failed on an outcome the
// rebuild has already replaced.
// pauseView is whether the build seat takes work: the holders that said they are paused, and
// whether that is every holder.
type pauseView struct {
Nodes []string
All bool
}
// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
return "", false
}
var asked *time.Time
for _, m := range p.Tiers[p.Tier] {
if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
if asked == nil || s.AskedAt.Before(*asked) {
asked = s.AskedAt
}
}
}
if asked == nil {
return "", false
}
waited := now.Sub(*asked)
said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
// longer than a day is named.
if waited > pausedTooLong {
said += " — PAUSED OVER A DAY: `resume` takes builds again"
}
return said, true
}
// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
const pausedTooLong = 24 * time.Hour
// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
// where the seat being paused changes what a plan says.
func awaitsABuild(plans []inventory.Plan) bool {
for _, p := range plans {
if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
continue
}
for _, m := range p.Tiers[p.Tier] {
if s := p.Modules[m]; s != nil && s.State == "asked" {
return true
}
}
}
return false
}
// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
// reads as late, which is what it said before this existed.
func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
if !awaitsABuild(plans) {
return pauseView{}
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return pauseView{}
}
seat := buildSeatAmong(entries)
holders := holdersAmong(entries, seat)
if len(holders) == 0 {
return pauseView{}
}
address, err := broker.BusAddress()
if err != nil {
return pauseView{}
}
js, err := broker.Dial(address)
if err != nil {
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
return pauseView{}
}
defer js.Close()
said, err := link.PausedSaid(js, seat, holders)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
return pauseView{}
}
return pauseOf(holders, said)
}
// pauseOf is the view from what each holder said.
func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
var v pauseView
for _, n := range holders {
if said[n].Paused {
v.Nodes = append(v.Nodes, n)
}
}
sort.Strings(v.Nodes)
v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
return v
}
// failedIn is the modules of a plan's current tier that failed to build, sorted.
func failedIn(p inventory.Plan) []string {
if p.Tier >= len(p.Tiers) {
return nil
}
var out []string
for _, m := range p.Tiers[p.Tier] {
if s := p.Modules[m]; s != nil && s.State == "failed" {
out = append(out, m)
}
}
sort.Strings(out)
return out
}
// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
// that holds what this one held now (issue 254, ADR 0218).
func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
for _, q := range plans {
if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
continue
}
return q, true
}
return inventory.Plan{}, false
}
// retryRefusal is why a plan cannot be retried, or nothing.
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
switch {
case p.State == inventory.PlanDone:
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
case p.State == inventory.PlanSuperseded:
return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
case p.Open():
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
}
if len(failedIn(p)) > 0 {
if q, found := newerOpenPlan(p, plans); found {
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
}
return nil
}
stopped := stoppedRollouts(p)
if len(stopped) == 0 {
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
p.Tier, p.ID, p.Note)
}
// **A build that failed its gate is not sent again** (novox/hq ADR 0236): it was put back on its first
// machine, and retrying would judge the build the mesh put back, or send the failed one by hand.
for _, m := range stopped {
if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed && !noVerdictOnItsBuild(g) {
return fmt.Errorf("%s failed its gate on %s (%s) and was put back: a build that failed its gate is not "+
"sent again — a newer merge, or `rebuild %s`, makes a new build, judged at the gate again",
m, strings.Join(g.Machines, ", "), g.Why, m)
}
}
// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
// sent — a newer build, and sending this one again would put the older build back on its machines.
for _, m := range stopped {
if q, found := newerPlanFor(m, p, plans); found {
return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
}
}
return nil
}
// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
func stoppedRollouts(p inventory.Plan) []string {
if p.Tier >= len(p.Tiers) {
return nil
}
var out []string
for _, m := range p.Tiers[p.Tier] {
if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
len(s.First) > 0 && s.Why != "" {
out = append(out, m)
}
}
sort.Strings(out)
return out
}
// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
for _, q := range plans {
if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
continue
}
for _, tier := range q.Tiers {
for _, m := range tier {
if m == module {
return q, true
}
}
}
}
return inventory.Plan{}, false
}
// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
// variable so a test of a retried rollout needs no machine.
var sendRollout = sendToEach
// resumed sets a failed plan building again once nothing in its tier is failed.
func resumed(p *inventory.Plan, why string) {
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
p.State = inventory.PlanBuilding
p.Note = why
}
}
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
// that tier: what follows is the plan going on as if they had built the first time.
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
inv := open.inventory
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return "", err
}
defer release()
p, err := inv.PlanByID(ctx, id)
if err != nil {
return "", err
}
plans, err := inv.OpenPlans(ctx)
if err != nil {
return "", err
}
recent, err := inv.RecentPlans(ctx, 50)
if err != nil {
return "", err
}
plans = append(plans, recent...)
if err := retryRefusal(p, plans); err != nil {
return "", err
}
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
return retryTierWhole(ctx, open, &p, again)
}
if len(failedIn(p)) == 0 {
return retryRollouts(ctx, open, &p)
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return "", err
}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
failed := failedIn(p)
var asked []string
for _, m := range failed {
askModule(ctx, &p, m, byName)
if s := p.Modules[m]; s.State == "asked" {
asked = append(asked, m+" as "+s.Build)
}
}
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
if err := inv.SavePlan(ctx, &p); err != nil {
return "", err
}
if p.State != inventory.PlanBuilding {
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
}
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
}
// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
inv := open.inventory
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return false, "", err
}
defer release()
openPlans, err := inv.OpenPlans(ctx)
if err != nil {
return false, "", err
}
recent, err := inv.RecentPlans(ctx, 20)
if err != nil {
return false, "", err
}
p, found := planHolding(module, openPlans, recent)
if !found {
return false, "", nil
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return false, "", err
}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
was := p.State
askModule(ctx, &p, module, byName)
s := p.Modules[module]
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
if err := inv.SavePlan(ctx, &p); err != nil {
return false, "", err
}
if s.State != "asked" {
return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
}
said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
switch {
case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
said += "; the plan had failed and builds again from this tier"
case was == inventory.PlanFailed:
said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
}
return true, said, nil
}
// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
// yet built, else the newest failed plan whose current tier does, and that no open plan of its
// repository supersedes.
func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
holds := func(p inventory.Plan) bool {
if p.Tier >= len(p.Tiers) {
return false
}
for _, m := range p.Tiers[p.Tier] {
if m == module {
s := p.Modules[m]
return s == nil || s.State != "built"
}
}
return false
}
for _, p := range openPlans {
if holds(p) {
return p, true
}
}
sorted := append([]inventory.Plan(nil), recent...)
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
for _, p := range sorted {
if p.State != inventory.PlanFailed || !holds(p) {
continue
}
if _, superseded := newerOpenPlan(p, openPlans); superseded {
continue
}
return p, true
}
return inventory.Plan{}, false
}
// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
// Every machine once, for all the modules stopped there (novox/hq issue 281).
stopped := stoppedRollouts(*p)
var machines []string
for _, m := range stopped {
for _, n := range p.Modules[m].First {
if !slices.Contains(machines, n) {
machines = append(machines, n)
}
}
}
sort.Strings(machines)
sent, err := sendRollout(ctx, open, machines)
if err != nil {
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
strings.Join(stopped, ", "), strings.Join(machines, ", "), p.ID, err)
}
now := time.Now().UTC()
var said []string
for _, m := range stopped {
s := p.Modules[m]
var again []string
for _, n := range sent {
if slices.Contains(s.First, n) {
again = append(again, n)
}
}
s.First, s.FirstAt, s.Why = again, &now, ""
said = append(said, m+" to "+strings.Join(again, ", "))
}
p.State = inventory.PlanRolling
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
if err := open.inventory.SavePlan(ctx, p); err != nil {
return "", err
}
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
}
// noVerdictOnItsBuild says a failed gate said nothing about the module's build (novox/hq issue 281): its
// send changed nothing of the module there — the machine already ran that build, carried there by an
// earlier send of the same tier, or one identical to it. Such a module was blamed for its machine.
func noVerdictOnItsBuild(g *inventory.PlanGate) bool {
return g.Rollback == gateUnchanged || (g.From != "" && sameCommit(g.From, g.To))
}
// unjudgedAtGate is the modules of a plan's current tier stopped at a gate that was no verdict on their
// build, sorted.
func unjudgedAtGate(p inventory.Plan) []string {
if p.Tier >= len(p.Tiers) {
return nil
}
var out []string
for _, m := range p.Tiers[p.Tier] {
if s := p.Modules[m]; s != nil && s.Gate != nil && s.Gate.Verdict == inventory.GateFailed && noVerdictOnItsBuild(s.Gate) {
out = append(out, m)
}
}
sort.Strings(out)
return out
}
// retryTierWhole retries a plan stopped at a gate that judged no build of the module it stopped on
// (issue 281): that module is asked again under a new id — its old build may be marked failed, and a new
// verdict is what takes the gate's condition away — and every module of the tier sent first and never
// passed is sent again, the tier whole, one send per machine, judged again. What passed stays passed.
func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again []string) (string, error) {
inv := open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
return "", err
}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
var resent []string
for _, m := range p.Tiers[p.Tier] {
s := p.Modules[m]
if s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(again, m) ||
(s.Gate != nil && s.Gate.Verdict == inventory.GatePassed) {
continue
}
sendAgain(s)
resent = append(resent, m)
}
var asked []string
for _, m := range again {
sendAgain(p.Modules[m])
askModule(ctx, p, m, byName)
if s := p.Modules[m]; s.State == "asked" {
asked = append(asked, m+" as "+s.Build)
}
}
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
p.State = inventory.PlanBuilding
p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
}
if err := inv.SavePlan(ctx, p); err != nil {
return "", err
}
if p.State != inventory.PlanBuilding {
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
}
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; %s sent again with the tier, one send per machine, "+
"judged again", p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", "), orNone(strings.Join(resent, ", "))), nil
}
// sendAgain forgets a module's first send, so its plan sends it again.
func sendAgain(s *inventory.PlanModule) {
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
}
cmd/mesh-controller/plans_one_at_a_time_test.go
+1 -1
View File
@@ -18,7 +18,7 @@ func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
if err := open.inventory.SavePlan(ctx, plan); err != nil {
t.Fatal(err)
}
cmd/mesh-controller/probes.go
-1245
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/push.go
+198 -665
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/push_test.go
+2 -2
View File
@@ -76,7 +76,7 @@ func TestASkippedMachineIsStillAnError(t *testing.T) {
}
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
func numbered() func(string) (order, error) {
func numbered() func(string) (int64, error) {
var n int64
return func(string) (order, error) { n++; return order{sequence: n}, nil }
return func(string) (int64, error) { n++; return n, nil }
}
cmd/mesh-controller/queue.go
-709
View File
@@ -1,709 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The build queue, controlled by hand (novox/hq ADR 0219).
//
// Seen whole — what waits, what runs where and for how long, what was handed out as often as it
// may be and never settled — and changed through the controller: an ask cancelled, the queue
// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's
// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`).
//
// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that
// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
// rather than waiting for ever on an answer nobody will give.
// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
// than the holder's worst case (its two passes removing containers and its wait between, 50s).
const (
holderAsks = 15 * time.Second
killAnswer = 75 * time.Second
)
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
func dialTheBus() (*broker.JetStream, error) {
address, err := broker.BusAddress()
if err != nil {
return nil, err
}
js, err := broker.Dial(address)
if err != nil {
return nil, fmt.Errorf("cannot reach the bus: %w", err)
}
return js, nil
}
// queueCommand prints every ask in the build seat's work queue.
func queueCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("queue", flag.ContinueOnError)
asJSON := set.Bool("json", false, "the queue as JSON")
if _, err := parseAround(set, args); err != nil {
return err
}
js, err := dialTheBus()
if err != nil {
return err
}
defer js.Close()
seat := buildSeatHeld(ctx)
q, err := link.ReadQueue(ctx, js, seat)
if err != nil {
return err
}
if *asJSON {
body, err := json.MarshalIndent(q, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
fmt.Print(queueText(q, time.Now()))
return nil
}
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
// Never what an ask carries as `held` — that is every artifact the mesh has built.
func queueText(q link.Queue, now time.Time) string {
var b strings.Builder
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead))
ago := func(t time.Time) string {
if t.IsZero() {
return "asked at an unknown time"
}
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
}
what := func(a link.QueuedAsk) string {
s := a.Repository
if a.Path != "" {
s += " at " + a.Path
}
if a.Ref != "" {
s += " on " + a.Ref
}
return s
}
if len(running) > 0 {
fmt.Fprintln(&b, "\nin flight:")
for _, a := range running {
where := "taken, not yet said where"
switch {
case a.On != "":
where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
case a.Was != "":
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
}
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
}
}
if len(waiting) > 0 {
fmt.Fprintln(&b, "\nwaiting:")
for _, a := range waiting {
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
}
}
if len(dead) > 0 {
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
for _, a := range dead {
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
}
}
switch {
case len(q.Asks) == 0:
fmt.Fprintln(&b, "nothing is asked of it")
default:
fmt.Fprintln(&b, "\n`cancel <id>` drops a waiting or dead ask, `clear` every waiting one, `kill <id>` ends one in flight")
}
return b.String()
}
// cancelCommand drops one waiting or dead ask.
func cancelCommand(ctx context.Context, args []string) error {
if len(args) != 1 {
return errors.New("cancel <build id>")
}
js, err := dialTheBus()
if err != nil {
return err
}
defer js.Close()
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
seat := buildSeatHeld(ctx)
q, err := link.ReadQueue(ctx, js, seat)
if err != nil {
return err
}
ask, found := q.Find(args[0])
if !found {
return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+
"what came of it", args[0], seat)
}
said, err := cancelAsk(ctx, js, open, seat, ask)
if err != nil {
return err
}
fmt.Println(said)
return nil
}
// cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
//
// **In this order, and each step for a reason** (novox/hq ADR 0219):
// 1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
// is running — that is `kill`, on the machine running it;
// 2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
// 3. for a waiting ask, the worker is read again: one handed out since the queue was read was
// taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
// read the mark first and ends it as cancelled, or is building it;
// 4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
// handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
// at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
// that took it before the mark is building it, and only `kill` ends that;
// 5. the message is deleted by its sequence;
// 6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
if ask.State == link.AskInFlight && ask.On != "" {
return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
"ends the build where it runs", ask.ID, ask.On, ask.ID)
}
if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
return "", err
}
withdraw := func() {
if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
}
}
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
switch ask.State {
case link.AskWaiting:
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
withdraw()
return "", err
} else if taken {
withdraw()
return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
}
case link.AskInFlight:
if started, err := startedSince(ctx, js, seat, ask); err != nil {
withdraw()
return "", err
} else if started != "" {
withdraw()
return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
"ends it there", ask.ID, started, ask.ID)
}
}
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
}
recordCancelled(ctx, open, ask)
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
}
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
var holderLooks = 5 * time.Second
// startedSince waits out a holder's look at the cancelled set and says the machine that started the
// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
// a start of a build.
func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(holderLooks):
}
since := time.Now().Add(-7 * 24 * time.Hour)
if !ask.AskedAt.IsZero() {
since = ask.AskedAt.Add(-time.Minute)
}
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
if err != nil {
return "", err
}
s, ok := heard.Started[ask.ID]
if !ok || heard.Outcomes[ask.ID] {
return "", nil
}
at, _ := time.Parse(time.RFC3339Nano, s.At)
if ask.Started.IsZero() || at.After(ask.Started) {
return s.On, nil
}
return "", nil
}
// takenSince is whether the worker has handed out the ask at this sequence.
func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
if errors.Is(err, nats.ErrConsumerNotFound) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("cannot read the worker of the queue again: %w", err)
}
return info.Delivered.Stream >= seq, nil
}
// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded,
// then the plan that asked for it — by the id it asked with, or by repository and path.
func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) {
r := ask.Request
result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref,
Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand}
_ = builds{open.inventory, open}.Built(ctx, result)
}
// clearCommand cancels every waiting ask, and with --dead every dead one too.
func clearCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("clear", flag.ContinueOnError)
dead := set.Bool("dead", false, "the dead asks too")
if _, err := parseAround(set, args); err != nil {
return err
}
js, err := dialTheBus()
if err != nil {
return err
}
defer js.Close()
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
seat := buildSeatHeld(ctx)
said, err := clearQueue(ctx, js, open, seat, *dead)
fmt.Print(said)
return err
}
// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight.
func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) {
q, err := link.ReadQueue(ctx, js, seat)
if err != nil {
return "", err
}
var b strings.Builder
cancelled, refused := 0, 0
for _, a := range q.Asks {
if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) {
continue
}
said, err := cancelAsk(ctx, js, open, seat, a)
if err != nil {
refused++
fmt.Fprintf(&b, " %s: %v\n", a.ID, err)
continue
}
cancelled++
fmt.Fprintf(&b, " %s\n", said)
}
what := "waiting"
if dead {
what = "waiting and dead"
}
fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what)
if refused > 0 {
fmt.Fprintf(&b, ", %d could not be", refused)
}
fmt.Fprintln(&b)
if n := len(q.Of(link.AskInFlight)); n > 0 {
fmt.Fprintf(&b, "%d in flight, left running: `kill <id>` ends one where it runs\n", n)
}
if n := len(q.Of(link.AskDead)); n > 0 && !dead {
fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n)
}
if refused > 0 {
return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused)
}
return b.String(), nil
}
// rebuildCommand asks the module's current source again — or, given a build's id, that build's
// repository, path and ref — under a new id.
func rebuildCommand(ctx context.Context, args []string) error {
if len(args) != 1 {
return errors.New("rebuild <module | build id>")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var source buildSource
var path, ref, module string
if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
return err
} else if found {
source, module = sourceOfBuild(b, entries)
path, ref = b.Path, b.Ref
// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
// that commit out over everything since. Building that commit again is `replay`, which says
// what it would do and refuses to register it while anything newer is asked or registered.
if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
ref = followedBranch(e.Source.Ref)
follows := ref
if follows == "" {
follows = "the repository's default branch"
}
fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
}
} else if e, known := entryNamed(entries, args[0]); known {
// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
} else {
return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
}
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
if module != "" {
joined, said, err := joinAPlan(ctx, open, module)
if err != nil {
return err
}
if joined {
fmt.Println(said)
return nil
}
}
id, err := askABuild(ctx, source, path, ref)
if err != nil {
return err
}
fmt.Printf("rebuild asked as %s\n", id)
return nil
}
// entryNamed is the catalogue's entry for a module.
func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
for _, e := range entries {
if e.Manifest.Module == name {
return e, true
}
}
return inventory.Entry{}, false
}
// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
// (ADR 0111) — else the repository as it was cloned.
func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) {
for _, e := range entries {
if (b.Module != "" && e.Manifest.Module == b.Module) ||
(b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) {
return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module
}
}
return buildSource{Repository: b.Repository}, b.Module
}
// replayCommand asks a recorded build's repository and path again at the commit it built.
func replayCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("replay", flag.ContinueOnError)
register := set.Bool("register", false, "register what it builds, as any build is")
older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("replay <build id> [--register [--older]]")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
b, found, err := inv.BuildByID(ctx, positionals[0])
if err != nil {
return err
}
if !found {
return fmt.Errorf("no build %s is recorded", positionals[0])
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
source, module := sourceOfBuild(b, entries)
var history []inventory.Build
if module != "" {
if history, err = inv.Builds(ctx, module, 100); err != nil {
return err
}
}
// What is asked of the module and not yet answered, when the replay would be registered.
var outstanding []string
if *register {
js, err := dialTheBus()
if err != nil {
return err
}
q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
js.Close()
if err != nil {
return err
}
plans, err := inv.OpenPlans(ctx)
if err != nil {
return err
}
outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
}
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
return err
}
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
if err != nil {
return err
}
fmt.Println(replaySaid(b, module, id, *register))
return nil
}
// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
//
// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
// older commit is newer than everything since, and would roll that older commit out as the module's
// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
// --register says otherwise, and --register is refused when a newer build of a different commit is
// registered, unless --older says that is the point.
//
// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
// and their builds — made from newer source — would be recorded and never registered (issue 219
// orders by ask), the plan waiting on them sending the older commit instead.
func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
outstanding []string) error {
if b.Commit == "" {
return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
}
if older && !register {
return errors.New("--older only says what --register may do; a dry run registers nothing")
}
if register && len(outstanding) > 0 {
return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
orNone(module), strings.Join(outstanding, "; "), b.ID)
}
if !register || older {
return nil
}
for _, h := range history {
if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
continue
}
if h.AskedOrAt().After(b.AskedOrAt()) {
return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
"without --register looks at it; --register --older registers it anyway",
module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
}
}
return nil
}
// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome.
func replaySaid(b inventory.Build, module, id string, register bool) string {
what := b.Repository
if module != "" {
what = module
}
said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id)
if !register {
return said + "\n a dry run: the outcome is looked at and not taken in — nothing is recorded or " +
"registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it"
}
return said + "\n registered when it is built, as the module's current version — newer than every build " +
"asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it"
}
// killCommand finds the machine running a build and asks its holder to end it.
func killCommand(ctx context.Context, args []string) error {
if len(args) != 1 {
return errors.New("kill <build id>")
}
id := args[0]
js, err := dialTheBus()
if err != nil {
return err
}
defer js.Close()
seat := buildSeatHeld(ctx)
since := time.Now().Add(-7 * 24 * time.Hour)
if at, ok := link.BuildAskedAt(id); ok {
since = at.Add(-time.Minute)
}
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
if err != nil {
return err
}
started, ok := heard.Started[id]
if !ok {
return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id)
}
if heard.Outcomes[id] {
return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
}
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
if err != nil {
return err
}
return printHolderAnswer(started.On, answer)
}
// pauseCommand asks one machine's holder, or every holder's, to pause or resume.
func pauseCommand(ctx context.Context, verb string, args []string) error {
if len(args) > 1 {
return fmt.Errorf("%s [node]", verb)
}
js, err := dialTheBus()
if err != nil {
return err
}
defer js.Close()
seat := buildSeatHeld(ctx)
nodes := args
if len(nodes) == 0 {
if nodes, err = buildSeatHolders(ctx, seat); err != nil {
return err
}
if len(nodes) == 0 {
return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb)
}
}
failed := 0
for _, node := range nodes {
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks)
if err != nil {
fmt.Printf("%s: %v\n", node, err)
failed++
continue
}
if err := printHolderAnswer(node, answer); err != nil {
failed++
}
}
if failed > 0 {
return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb)
}
return nil
}
// printHolderAnswer prints what a holder said, or its refusal as the command's failure.
func printHolderAnswer(node string, answer link.Answer) error {
if answer.Error != "" {
fmt.Printf("%s: %s\n", node, answer.Error)
return errors.New(answer.Error)
}
var said struct {
Said string `json:"said"`
}
if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" {
fmt.Printf("%s: %s\n", node, said.Said)
return nil
}
fmt.Printf("%s: %s\n", node, string(answer.Result))
return nil
}
// buildSeatHolders is every machine an assigned module holding the build seat runs on.
func buildSeatHolders(ctx context.Context, seat string) ([]string, error) {
open, err := openStores(ctx)
if err != nil {
return nil, err
}
defer open.Close()
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
return nil, err
}
return holdersAmong(entries, seat), nil
}
// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each.
func holdersAmong(entries []inventory.Entry, seat string) []string {
seen := map[string]bool{}
var out []string
for _, e := range entries {
if !e.Manifest.ClaimsSeat(seat) {
continue
}
for _, n := range e.On {
if !seen[n] {
seen[n] = true
out = append(out, n)
}
}
}
sort.Strings(out)
return out
}
// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
// by repository and path, and every open plan holding it not yet built.
func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
var out []string
for _, a := range q.Asks {
if repositoryMatches(a.Repository, repository) && a.Path == path {
out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
}
}
if module == "" {
return out
}
for _, p := range plans {
if !p.Open() {
continue
}
for _, tier := range p.Tiers {
for _, m := range tier {
if m == module {
if st := p.Modules[m]; st == nil || st.State != "built" {
out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
}
}
}
}
}
return out
}
cmd/mesh-controller/queue_test.go
-772
View File
@@ -1,772 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
//
// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused' (each test on a bus of its own)
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
func asksRecorded(t *testing.T) *[]string {
t.Helper()
var asked []string
was := askABuild
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond))
asked = append(asked, source.Repository+"#"+id)
return id, nil
}
t.Cleanup(func() { askABuild = was })
return &asked
}
// twoTiers registers two modules, b standing on a, each with a source a plan asks.
func twoTiers(t *testing.T, open *stores) {
t.Helper()
for _, name := range []string{"a", "b"} {
if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
t.Fatal(err)
}
}
}
// A failed plan is retried: its failed module asked again under a new id, the plan building at that
// tier, and when that build comes in the plan goes on and asks its next tier.
func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
asked := asksRecorded(t)
twoTiers(t, open)
before := time.Now().UTC().Add(-time.Hour)
failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
Note: "a failed to build in tier 0",
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
Why: link.KilledByHand}}}
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
t.Fatal(err)
}
said, err := retryPlan(ctx, open, failed.ID)
if err != nil {
t.Fatal(err)
}
p, err := open.inventory.PlanByID(ctx, failed.ID)
if err != nil {
t.Fatal(err)
}
a := p.Modules["a"]
if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
}
if !strings.Contains(said, a.Build) {
t.Errorf("retry does not say the new id: %q", said)
}
if len(*asked) != 1 {
t.Fatalf("asked %v", *asked)
}
// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
}
asking, _ := link.BuildAskedAt(a.Build)
planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
p, err = open.inventory.PlanByID(ctx, failed.ID)
if err != nil {
t.Fatal(err)
}
if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
}
if len(*asked) != 2 {
t.Fatalf("asked %v", *asked)
}
}
// What retry refuses, and says why.
func TestRetryRefusesWhatItCannotResume(t *testing.T) {
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
plan := func(id, state string, created time.Time) inventory.Plan {
return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee",
Created: created, State: state, Tiers: [][]string{{"a"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
}
failed := plan("plan-1", inventory.PlanFailed, at)
for _, c := range []struct {
p inventory.Plan
others []inventory.Plan
says string
}{
{plan("plan-d", inventory.PlanDone, at), nil, "is done"},
{plan("plan-s", inventory.PlanSuperseded, at), nil, "was"},
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
} {
err := retryRefusal(c.p, c.others)
if err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
}
}
stopped := failed
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
stopped.Note = "a stopped at its first machine"
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
t.Errorf("a plan with nothing failed to build was retried: %v", err)
}
// Another branch's newer plan, an older one, and a failed one do not supersede it.
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
other.Branch = "release"
if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
t.Errorf("refused for a plan that does not supersede it: %v", err)
}
}
// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
asked := asksRecorded(t)
twoTiers(t, open)
before := time.Now().UTC().Add(-time.Hour)
failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
Note: "a failed to build in tier 0",
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
t.Fatal(err)
}
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
t.Fatal(err)
}
p, err := open.inventory.PlanByID(ctx, failed.ID)
if err != nil {
t.Fatal(err)
}
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
}
if len(*asked) != 1 {
t.Fatalf("asked %v", *asked)
}
// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
if err := rebuildCommand(ctx, []string{"b"}); err != nil {
t.Fatal(err)
}
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
}
if len(*asked) != 2 {
t.Fatalf("asked %v", *asked)
}
}
// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
t.Fatal("joined a failed plan a newer open one supersedes")
}
if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
}
built := failed
built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
t.Fatal("joined a plan that has the module built")
}
}
// replay is a dry run unless registered, and registering an older commit than one registered since
// is refused unless --older says it is meant (novox/hq issue 207).
func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
history := []inventory.Build{newer, old}
if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
t.Errorf("a dry run was refused: %v", err)
}
err := replayRefusal(old, "a", history, true, false, nil)
if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
t.Errorf("registering an older commit than the one registered was not refused: %v", err)
}
if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
t.Errorf("--register --older was refused: %v", err)
}
if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
t.Errorf("registering the newest build again was refused: %v", err)
}
// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
}
if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
t.Error("a build that recorded no commit was replayed")
}
if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
t.Error("--older without --register was taken")
}
// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
// holding it unbuilt, would be replaced by a replay asked now (issue 219).
q := link.Queue{Asks: []link.QueuedAsk{
{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
}}
plans := []inventory.Plan{
{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
}
outstanding := outstandingFor("a", "novox/a", "", q, plans)
if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
t.Fatalf("outstanding: %v", outstanding)
}
if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
t.Errorf("registered over an outstanding ask: %v", err)
}
if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
t.Errorf("a dry run was refused for what is outstanding: %v", err)
}
if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
t.Errorf("a dry replay does not say what it is: %q", said)
}
if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
t.Errorf("a registered replay does not say what it does: %q", said)
}
}
// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
// paused on some holders only is not a reason the plan is waiting.
func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
asked := now.Add(-12 * time.Minute)
p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding,
Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
line := planLineWith(p, now, all)
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
t.Errorf("a plan on a paused seat reads %q", line)
}
if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
t.Errorf("status --json says %+v", st)
}
if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
t.Errorf("a plan waiting on a paused seat counted late")
}
longAgo := now.Add(-25 * time.Hour)
forgotten := p
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
t.Errorf("a plan paused over a day reads %q", line)
}
some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
t.Fatalf("%+v", some)
}
if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
}
if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
t.Errorf("status --json: %+v", st)
}
// A plan rolling out, or with nothing asked, is not waiting on the seat.
rolling := p
rolling.State = inventory.PlanRolling
if _, paused := pausedWaiting(rolling, all, now); paused {
t.Error("a rolling plan reads as waiting on the build seat")
}
}
// The queue's verbs are the controller seat's, each to the command it names.
func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want []string
}{
{"queue", nil, []string{"queue"}},
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
{"clear", nil, []string{"clear"}},
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
{"pause", nil, []string{"pause"}},
{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
{"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}},
} {
got, err := argvFor(c.verb, c.args)
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
}
}
if _, err := argvFor("cancel", nil); err == nil {
t.Error("cancel without an id was taken")
}
declared := map[string]bool{}
for _, v := range catalogue.ControllerVerbs {
declared[v.Name] = true
}
for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} {
if !declared[v] {
t.Errorf("%s is not a verb of the controller seat", v)
}
}
}
// --- against a real bus -----------------------------------------------------------------------
// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller
// pointed at it, and a function that asks the seat one build.
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
t.Helper()
url := testbus.URL(t)
t.Setenv(broker.NATSVar, url)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
Emits: []string{"started", "built", "log.*", "paused.*"}}
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
t.Fatal(err)
}
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
_ = js.Context().PurgeStream(broker.EventsStream)
})
ask := func(id, repository string) link.BuildRequest {
r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
body, _ := json.Marshal(r)
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
t.Fatal(err)
}
return r
}
return js, ask
}
// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows.
func deadOne(t *testing.T, js *broker.JetStream) {
t.Helper()
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
for i := 0; i < worker.MaxDeliver; i++ {
msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
if err != nil {
t.Fatalf("delivery %d: %v", i+1, err)
}
_ = msgs[0].Nak()
}
// One more pull, as a holder always has one waiting: the server finds the ask past its deliveries
// then, and stops counting it pending.
if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 {
t.Fatalf("an ask past its deliveries was delivered again")
}
}
// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan
// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for
// is still found.
func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
js, ask := aBuildQueue(t)
open := aMesh(t)
ctx := t.Context()
twoTiers(t, open)
id := link.NewBuildID(time.Now())
ask(id, "https://forge.example/novox/a.git")
asked, _ := link.BuildAskedAt(id)
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
if err != nil {
t.Fatal(err)
}
if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
t.Fatalf("the queue reads %+v", q)
}
if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
strings.Contains(text, "secret-ish") {
t.Errorf("the queue says:\n%s", text)
}
if err := cancelCommand(ctx, []string{id}); err != nil {
t.Fatal(err)
}
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
t.Fatalf("the ask is still queued: %+v", q.Asks)
}
if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
}
b, found, err := open.inventory.BuildByID(ctx, id)
if err != nil || !found || b.Failed != link.CancelledByHand {
t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
}
p, err := open.inventory.PlanByID(ctx, plan.ID)
if err != nil {
t.Fatal(err)
}
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
}
if err := cancelCommand(ctx, []string{id}); err == nil {
t.Error("an ask cancelled already was cancelled again")
}
}
// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight.
func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
js, ask := aBuildQueue(t)
open := aMesh(t)
ctx := t.Context()
start := time.Now()
dead := link.NewBuildID(start)
ask(dead, "https://forge.example/novox/dead.git")
deadOne(t, js)
var waiting []string
for i := 1; i <= 2; i++ {
id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
waiting = append(waiting, id)
ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
}
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
if err != nil {
t.Fatal(err)
}
if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
t.Fatalf("the queue reads %+v", q)
}
said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
t.Errorf("clear said:\n%s", said)
}
q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
t.Fatalf("after clear the queue is %+v", q.Asks)
}
if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
t.Fatal(err)
}
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
t.Fatalf("clear --dead left %+v", q.Asks)
}
for _, id := range append(waiting, dead) {
if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
t.Errorf("%s is recorded as %+v", id, b)
}
}
}
// An ask in flight is not cancelled: kill ends it where it runs.
func TestCancelRefusesAnAskInFlight(t *testing.T) {
js, ask := aBuildQueue(t)
open := aMesh(t)
ctx := t.Context()
id := link.NewBuildID(time.Now())
ask(id, "https://forge.example/novox/a.git")
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
t.Fatal(err)
}
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
t.Fatal(err)
}
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
if err != nil {
t.Fatal(err)
}
a, _ := q.Find(id)
if a.State != link.AskInFlight || a.On != "ace" {
t.Fatalf("the taken ask reads %+v", a)
}
_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
if err == nil || !strings.Contains(err.Error(), "kill "+id) {
t.Fatalf("an ask in flight was cancelled: %v", err)
}
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
t.Error("a refused cancel recorded an outcome")
}
}
// kill finds the machine from the build's start and asks that machine's holder; pause asks the
// machine named. Each prints what the holder answered, and a refusal is the command's failure.
func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
js, _ := aBuildQueue(t)
ctx := t.Context()
asked := map[string]string{}
handlers := map[string]link.ToolHandler{
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
var args struct{ ID string }
_ = json.Unmarshal(raw, &args)
asked["kill"] = args.ID
if args.ID != "build-running" {
return nil, fmt.Errorf("ace is not building %s", args.ID)
}
return map[string]any{"said": "killed " + args.ID}, nil
},
"pause": func(context.Context, json.RawMessage) (any, error) {
asked["pause"] = "ace"
return map[string]any{"said": "ace is paused"}, nil
},
}
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
if err != nil {
t.Fatal(err)
}
defer stop()
for _, id := range []string{"build-running", "build-other"} {
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
t.Fatal(err)
}
}
if err := killCommand(ctx, []string{"build-running"}); err != nil {
t.Fatal(err)
}
if asked["kill"] != "build-running" {
t.Fatalf("the holder was asked %v", asked)
}
if err := killCommand(ctx, []string{"build-other"}); err == nil {
t.Error("the holder's refusal was not the command's")
}
if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
t.Errorf("a build nobody started: %v", err)
}
if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
t.Fatalf("pause: %v %v", err, asked)
}
if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
t.Error("a machine nothing answers on was resumed")
}
}
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
asked := asksRecorded(t)
twoTiers(t, open)
var sentTo [][]string
was := sendRollout
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
sentTo = append(sentTo, names)
return names, nil
}
t.Cleanup(func() { sendRollout = was })
// a records: a person's choice, since the default rolls out (novox/hq ADR 0236).
if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil {
t.Fatal(err)
}
long := time.Now().UTC().Add(-2 * time.Hour)
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
if err := open.inventory.SavePlan(ctx, &stopped); err != nil {
t.Fatal(err)
}
// A newer plan holding a refuses it: sending the older build would put it back.
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
t.Fatal(err)
}
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
t.Fatalf("retried under a newer plan: %v", err)
}
newer.State = inventory.PlanSuperseded
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
t.Fatal(err)
}
said, err := retryPlan(ctx, open, stopped.ID)
if err != nil {
t.Fatal(err)
}
if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
t.Fatalf("sent %v; said %q", sentTo, said)
}
p, err := open.inventory.PlanByID(ctx, stopped.ID)
if err != nil {
t.Fatal(err)
}
a := p.Modules["a"]
if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
}
// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
advancePlans(ctx, open)
if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
}
if len(*asked) != 1 {
t.Fatalf("asked %v", *asked)
}
}
// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
// the plan, asked later, never answers it (novox/hq ADR 0219).
func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
asked := time.Now().UTC().Add(-time.Minute)
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
p, _ := open.inventory.PlanByID(ctx, plan.ID)
if p.Modules["a"].State != "asked" {
t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
}
// From the records too: a later build of the module recorded is not the plan's.
recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
}
planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
}
}
// rebuild of a build made at a commit asks what the module follows now, never the commit.
func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
asked := asksRecorded(t)
twoTiers(t, open)
var refs []string
was := askABuild
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
refs = append(refs, ref)
return was(c, source, path, ref)
}
if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
t.Fatal(err)
}
if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
t.Fatal(err)
}
if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
t.Fatalf("asked %v at %v", *asked, refs)
}
}
// An ask the worker counts out that no machine said it started is cancelled only if no start comes
// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
js, ask := aBuildQueue(t)
open := aMesh(t)
ctx := t.Context()
was := holderLooks
holderLooks = 300 * time.Millisecond
t.Cleanup(func() { holderLooks = was })
id := link.NewBuildID(time.Now())
ask(id, "https://forge.example/novox/a.git")
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
t.Fatal(err)
}
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
if err != nil {
t.Fatal(err)
}
a, _ := q.Find(id)
if a.State != link.AskInFlight || a.On != "" {
t.Fatalf("the taken ask reads %+v", a)
}
// The holder that took it says it started, while the cancel waits.
go func() {
time.Sleep(100 * time.Millisecond)
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
_, _ = js.Context().Publish(link.BuildStarted(), started)
}()
if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
t.Fatalf("a build that started was cancelled: %v", err)
}
if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
t.Error("the withdrawn cancel is still marked")
}
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
t.Error("a withdrawn cancel recorded an outcome")
}
}
cmd/mesh-controller/readable.go
+1 -38
View File
@@ -3,8 +3,6 @@ package main
import (
"encoding/json"
"fmt"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"sort"
"time"
@@ -75,32 +73,6 @@ type meshStatus struct {
// alone. A document without this called a machine well while a predecessor's chain refused
// what the mesh declared open.
Filtered []machineFiltered `json:"filtered,omitempty"`
// Unheld is every module on a machine whose resources are applied through a seat nothing on
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
// dependency is met. Reported, not refused, until the switch.
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
// HandActsUnread says why when it could not be read, rather than reading as none.
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
HandActsUnread string `json:"handActsUnread,omitempty"`
// HealsThisWeek is what the healers did in the last seven days (novox/hq to-be 45 §7); HealsUnread
// why it could not be read.
HealsThisWeek *healsCount `json:"healsThisWeek,omitempty"`
HealsUnread string `json:"healsUnread,omitempty"`
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
Conditions []conditions.Condition `json:"conditions"`
ConditionsUnread string `json:"conditionsUnread,omitempty"`
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
// provider says so. A document without it called the mesh well while the identity provider
// refused every consumer for a day (04-ISSUES/179).
Failing []conditions.Condition `json:"failing,omitempty"`
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
}
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
@@ -199,7 +171,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now())}
// In a stated order, so two readings of an unchanged mesh are the same document.
untakenNodes := make([]string, 0, len(asked.untaken))
for name := range asked.untaken {
@@ -232,15 +204,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
}
}
out.Unheld = asked.unheld
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
if out.Conditions == nil {
out.Conditions = []conditions.Condition{}
}
out.Failing = providerStandings(asked.conditions)
out.Overflowing = asked.overflowing
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
cmd/mesh-controller/release.go
-667
View File
@@ -1,667 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// No build reaches a machine without a gate (novox/hq ADR 0236).
//
// **A send carries the machine's whole declaration.** A plan sending one module to its first machine
// sends every other module whose build moved there too; a cascade, a healer's resend and a whole-mesh
// push do the same. Under the old default (`record`) builds were registered and sent nowhere, so on the
// day the default became `roll` every machine was behind on dozens of builds no gate had seen — and the
// next send of anything would have restarted all of them at once, on every machine.
//
// So **a build moves on a machine only through a send that judges it there**, or a person's:
//
// - a gated send — a plan's first machine, a release plan's machine — carries every move waiting on that
// machine, and its gate judges each of them; a pass is each build's verdict, a failure puts back what
// failed;
// - a module a send exists for may move anywhere it is sent: a policy of *together*, a rollback, a build
// whose gate already passed;
// - a send naming a machine by a person (`push <node>`, the bus step) carries what it carries;
// - every other send — a plan's "rest", a cascade, a healer's, the bus's user list carried — is refused,
// or leaves the machine, while a move there waits for a gate. A rebuild that made the same artifacts
// from the same manifest is no move.
//
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
// sendScope is what a send may carry that no gate has seen.
type sendScope struct {
// judged are the machines whose every move this send's gate judges.
judged map[string]bool
// modules are those a send exists for, which may move wherever it goes.
modules map[string]bool
// person is a person's act: it carries what it carries.
person bool
}
type sendScopeKey struct{}
func withScope(ctx context.Context, s sendScope) context.Context {
return context.WithValue(ctx, sendScopeKey{}, s)
}
func scopeOf(ctx context.Context) sendScope {
s, _ := ctx.Value(sendScopeKey{}).(sendScope)
return s
}
// errUngated is a send refused because it would carry a build no gate has seen.
var errUngated = errors.New("a build waits there for its gate")
// errWalkedElsewhere is a gated send refused because a plan that has started walks a move there.
var errWalkedElsewhere = errors.New("a plan already walking a build there sends it")
// moveFacts is what tells a move from a rebuild, and a gated build from one no gate has seen.
type moveFacts struct {
current map[string]inventory.CurrentBuild
fps map[string]map[string]string
// srcs is, per module, per commit, its build's source fingerprint (novox/hq issue 280).
srcs map[string]map[string]string
passed map[string]map[string]bool
plans []inventory.Plan
}
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
var f moveFacts
var err error
if f.current, err = inv.CurrentBuilds(ctx); err != nil {
return f, err
}
if f.fps, err = inv.Fingerprints(ctx); err != nil {
return f, err
}
if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
return f, err
}
if f.passed, err = inv.PassedCommits(ctx); err != nil {
return f, err
}
f.plans, err = inv.OpenPlans(ctx)
return f, err
}
// identical is whether two builds of a module put the same thing on a machine: the same commit,
// builds made from the same source (novox/hq issue 280) — the module's tree, the contexts it read, its
// bases and toolchains, whatever digests an image rebuild made of them — or builds that made the same
// artifacts from the same manifest.
func (f moveFacts) identical(module, a, b string) bool {
if a == b || sameCommit(a, b) {
return true
}
if sa := f.srcs[module][a]; sa != "" && sa == f.srcs[module][b] {
return true
}
fa := f.fps[module][a]
return fa != "" && fa == f.fps[module][b]
}
// gated is whether a build of a module from this commit — or one identical to it — passed a gate.
func (f moveFacts) gated(module, commit string) bool {
for c := range f.passed[module] {
if f.identical(module, c, commit) {
return true
}
}
return false
}
// unchangedOnEvery is whether a plan's build of a module was made from the source of the build every
// machine running it was last sent (novox/hq issue 280): no move on any of them. False when no machine
// runs it, when what one was sent is not known, or when the build stands for itself.
func unchangedOnEvery(ctx context.Context, inv *inventory.Inventory, module, build string, running []string) (bool, error) {
if build == "" || len(running) == 0 {
return false, nil
}
same, err := inv.SameSourceCommits(ctx, module, build)
if err != nil || len(same) == 0 {
return false, err
}
for _, n := range running {
sent, known, err := inv.SentBuilds(ctx, n)
if err != nil {
return false, err
}
was, carried := sent[module]
if !known || !carried || !inRun(same, was) {
return false, nil
}
}
return true, nil
}
// inRun is whether a commit is one of a run's, however either is abbreviated.
func inRun(run map[string]bool, commit string) bool {
for c := range run {
if sameCommit(c, commit) {
return true
}
}
return false
}
// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out
// (a recorded one is a person's push), that the machine was sent before, moving to a build not identical
// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names
// none: the cascade holds it whole (ADR 0221).
//
// With all, a move to a build that passed a gate elsewhere counts too: what a gated send carries.
func (f moveFacts) moves(node string, modules []string, sent map[string]string, known, all bool) []inventory.CarriedMove {
if !known {
return nil
}
var out []inventory.CarriedMove
for _, m := range modules {
now := f.current[m]
was, carried := sent[m]
if !now.RollOut || !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) {
continue
}
out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit})
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// walkedBy is the open plan that has started walking a module's build — sent it to a first machine,
// not yet passed — other than to this machine; empty when none does.
func (f moveFacts) walkedBy(module, node string) string {
for _, p := range f.plans {
s, holds := p.Modules[module]
if !p.Open() || !holds || s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(s.First, node) {
continue
}
if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed {
continue
}
return p.ID
}
return ""
}
// machineMoves is the moves no gate has seen on one machine, read from what it would be sent now.
var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
plan, _, err := planFor(ctx, open, node)
if err != nil {
return nil, nil // it cannot be worked out: the send says why
}
modules := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
modules = append(modules, m.Module)
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the
// machines named, and why; the machine holding the bus, added only for its user list, is left instead.
func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) {
scope := scopeOf(ctx)
if scope.person {
return names, nil
}
f, err := readMoveFacts(ctx, open.inventory)
if err != nil {
return nil, err
}
var kept, refused []string
for _, n := range names {
moves, err := machineMoves(ctx, open, f, n, false)
if err != nil {
return nil, err
}
var waiting []string
for _, mv := range moves {
if !scope.judged[n] && !scope.modules[mv.Module] {
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
}
}
switch {
case len(waiting) == 0:
kept = append(kept, n)
case n == addedHolder:
fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s wait there for a gate "+
"— the bus may refuse what was newly granted until it is\n", n, strings.Join(waiting, ", "))
default:
refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", ")))
}
}
if len(refused) > 0 {
return nil, fmt.Errorf("%w: %s — a release plan sends them, one machine at a time, each judged (novox/hq ADR "+
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
}
return kept, nil
}
// gatedSend sends one machine everything waiting there, under a gate that judges it all: what moved is
// answered, with the build each moved to, for the gate to judge and to put back. Refused while a plan that
// has started walks one of those builds elsewhere: that plan sends it here once its gate passed.
//
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
// module was the same declaration sent again and again, each one setting aside the one before.
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) {
inv := open.inventory
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, nil, err
}
moves, err := machineMoves(ctx, open, f, node, true)
if err != nil {
return nil, nil, err
}
own := func(module string) bool {
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
}
for _, mv := range moves {
if own(mv.Module) {
continue
}
if id := f.walkedBy(mv.Module, node); id != "" {
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
mv.Module, short(mv.To), node, id)
}
}
for _, o := range owns {
i := slices.IndexFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == o.Module })
switch {
case i < 0:
moves = append(moves, o)
case moves[i].Build == "":
moves[i].Build = o.Build
}
}
if len(moves) == 0 && len(owns) == 0 {
return nil, nil, nil
}
for i := range moves {
if moves[i].Build == "" {
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
return nil, nil, err
}
}
}
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
if err != nil {
return nil, nil, err
}
return moves, sent, nil
}
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
seen := map[string]bool{}
for _, c := range g.Carried {
if c.Module == except || c.Build == "" || seen[c.Build] {
continue
}
seen[c.Build] = true
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why,
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err)
}
}
}
// failCarried puts back every build the gate carried that it found wanting, on the machines that were
// sent it, once each.
func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
var notes []string
if p.Note != "" {
notes = append(notes, p.Note)
}
done := map[string]bool{except: true}
for _, c := range g.Carried {
if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) {
continue
}
done[c.Module] = true
machines, err := sentTheBuild(ctx, open, c.Module, c.To)
if err != nil || len(machines) == 0 {
machines = []string{c.Node}
}
state := &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To}
// A module of the plan's tier sent in the same send (issue 281) keeps its own record of it.
if s := p.Modules[c.Module]; s != nil && except != "" && s.GatedBy == except && s.Build == c.Build {
state = s
}
p.Note = ""
gateFailed(ctx, open, p, c.Module, state, machines, g.Why)
notes = append(notes, p.Note)
}
p.State = inventory.PlanFailed
p.Note = strings.Join(notes, "; ")
}
// sentTheBuild is every machine running a module that was last sent this build of it.
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
running, err := open.inventory.Running(ctx, module)
if err != nil {
return nil, err
}
var out []string
for _, n := range running {
sent, known, err := open.inventory.SentBuilds(ctx, n)
if err != nil {
return nil, err
}
if known && sameCommit(sent[module], commit) {
out = append(out, n)
}
}
return out, nil
}
// releaseRepository is what a release plan says it is for, where a merge's says its repository.
const releaseRepository = "the builds waiting for a gate"
// releaseHeard is the machines a release plan may send: heard within their heartbeat's bound. A machine
// away is left, not judged against a bound it cannot meet. A variable so a test says who is heard.
var releaseHeard = func(ctx context.Context, open *stores) (map[string]bool, error) {
if d := doctorFrom; d != nil && d.watchdogs != nil {
return heardMachines(d), nil
}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return nil, err
}
out := map[string]bool{}
for _, r := range reports {
if r.At != nil && time.Since(*r.At) < 15*time.Minute {
out[r.Node] = true
}
}
return out, nil
}
// backlogNow is what the newest look found waiting, for `upgrade backlog` and the gate's probe.
var backlogNow struct {
held string
waiting map[string][]inventory.CarriedMove
}
// waitingMoves is every machine's moves no gate has seen and no started plan walks.
func waitingMoves(ctx context.Context, open *stores, all bool) (map[string][]inventory.CarriedMove, error) {
f, err := readMoveFacts(ctx, open.inventory)
if err != nil {
return nil, err
}
nodes, err := open.inventory.Nodes(ctx)
if err != nil {
return nil, err
}
out := map[string][]inventory.CarriedMove{}
for _, n := range nodes {
moves, err := machineMoves(ctx, open, f, n.Name, all)
if err != nil {
return nil, err
}
for _, mv := range moves {
if f.walkedBy(mv.Module, n.Name) == "" {
out[n.Name] = append(out[n.Name], mv)
}
}
}
return out, nil
}
// releaseBacklog opens a release plan when builds wait for a gate and none is open; not after a release
// plan failed, until a person releases one (by). Called with the plans held.
func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Plan, error) {
inv := open.inventory
plans, err := inv.OpenPlans(ctx)
if err != nil {
return nil, err
}
for _, p := range plans {
if p.Release != nil {
if by != "" {
return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID)
}
return nil, nil
}
}
waiting, err := waitingMoves(ctx, open, false)
if err != nil {
return nil, err
}
backlogNow.waiting, backlogNow.held = waiting, ""
if len(waiting) == 0 {
return nil, nil
}
// Opened by what no gate has seen; it walks every machine where anything of the release waits — a
// build that passed on the first machine still goes to the next one by this plan, judged there too.
if waiting, err = waitingMoves(ctx, open, true); err != nil {
return nil, err
}
if by == "" {
recent, err := inv.RecentPlans(ctx, 50)
if err != nil {
return nil, err
}
for _, p := range recent {
if p.Release == nil {
continue
}
if p.State == inventory.PlanFailed {
backlogNow.held = fmt.Sprintf("%s failed (%s); what waits is released again by a person", p.ID, p.Note)
return nil, nil
}
break
}
}
heard, err := releaseHeard(ctx, open)
if err != nil {
return nil, err
}
controllers, err := inv.Running(ctx, catalogue.ControllerSeatName)
if err != nil {
return nil, err
}
var order, last []string
modules := map[string]bool{}
for node, moves := range waiting {
if !heard[node] {
continue
}
for _, mv := range moves {
modules[mv.Module] = true
}
if slices.Contains(controllers, node) {
last = append(last, node)
} else {
order = append(order, node)
}
}
if len(order)+len(last) == 0 {
return nil, nil
}
sort.Strings(order)
sort.Strings(last)
order = append(order, last...)
names := make([]string, 0, len(modules))
for m := range modules {
names = append(names, m)
}
sort.Strings(names)
now := time.Now().UTC()
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
Release: &inventory.PlanRelease{Order: order, By: by},
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
strings.Join(order, ", "))}
if err := inv.SavePlan(ctx, &p); err != nil {
return nil, err
}
fmt.Printf("%s: %s\n", p.ID, p.Note)
return &p, nil
}
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
// gate, or the machine being judged judged once more; the plan done when every machine is.
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
r := p.Release
now := time.Now().UTC()
if r.Gate == nil {
heard, err := releaseHeard(ctx, open)
if err != nil {
return false, err
}
for r.Next < len(r.Order) {
node := r.Order[r.Next]
if !heard[node] {
r.Skipped = append(r.Skipped, node)
r.Next++
continue
}
moves, sent, err := gatedSend(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) {
note := fmt.Sprintf("waiting before %s: %v", node, err)
changed := p.Note != note
p.Note = note
return changed, nil
}
if err != nil {
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
}
if len(moves) == 0 {
r.Done = append(r.Done, node)
r.Next++
continue
}
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves}
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
if len(r.Skipped) > 0 {
p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
}
return true, nil
}
g := r.Gate
verdict, err := judgeMoves(ctx, open, g, nil, now)
if err != nil {
return false, err
}
switch verdict {
case "":
note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g))
changed := p.Note != note
p.Note = note
return changed, nil
case inventory.GatePassed:
passCarried(ctx, open, p, g, "")
r.Done = append(r.Done, firstOf(g.Machines))
r.Next++
r.Gate = nil
return true, nil
}
p.Note = ""
batched, back := batchingRollbacks(ctx)
failCarried(batched, open, p, g, "")
sendRollbacks(ctx, open, p, back)
p.Note = fmt.Sprintf("failed its gate on %s: %s — %s", strings.Join(g.Machines, ", "), g.Why, p.Note)
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
// backlogObservation is what the gate's probe says of a release held after a failure.
func backlogObservation() []conditions.Observation {
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
return nil
}
n := 0
var machines []string
for node, moves := range backlogNow.waiting {
n += len(moves)
machines = append(machines, node)
}
sort.Strings(machines)
return []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held",
Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%d build move(s) on %s wait for a gate and are not released: %s — `upgrade backlog` lists "+
"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}}
}
// backlogCommand is `upgrade backlog`, read-only, and `upgrade release-backlog --why`.
func backlogCommand(ctx context.Context, sub string, args []string) error {
set := flag.NewFlagSet("upgrade "+sub, flag.ContinueOnError)
why := addHandActFlags(set)
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("upgrade backlog | upgrade release-backlog --why <text>")
}
if sub == "release-backlog" {
if err := why.require("upgrade release-backlog"); err != nil {
return err
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if sub == "release-backlog" {
release, err := open.inventory.HoldPlans(ctx, true)
if err != nil {
return err
}
defer release()
why.record(ctx, "upgrade release-backlog", nil)
p, err := releaseBacklog(ctx, open, link.Caller())
if err != nil {
return err
}
if p == nil {
fmt.Println("nothing waits for a gate on any machine heard from: nothing to release")
return nil
}
fmt.Printf("%s releases it; `plans %s` says where it is\n", p.ID, p.ID)
return nil
}
waiting, err := waitingMoves(ctx, open, false)
if err != nil {
return err
}
if len(waiting) == 0 {
fmt.Println("no build waits for a gate on any machine")
return nil
}
nodes := make([]string, 0, len(waiting))
for n := range waiting {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
fmt.Printf("%s: %d build(s) wait for a gate\n", n, len(waiting[n]))
for _, mv := range waiting[n] {
fmt.Printf(" %-28s %s → %s\n", mv.Module, short(mv.From), short(mv.To))
}
}
return nil
}
cmd/mesh-controller/release_plan.go
+82 -774
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/release_plan_test.go
+4 -77
View File
@@ -145,7 +145,7 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
// Only a build from before the ask: not this ask's outcome.
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
}
if !settleFromRecords(&p, p.Tiers[0], records, nil) {
if !settleFromRecords(&p, p.Tiers[0], records) {
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
}
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
@@ -162,13 +162,13 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
late := map[string][]inventory.Build{"postgres": {
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
}}
if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
if settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "asked" {
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
}
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
if !settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "built" ||
r.Modules["postgres"].Commit != "4bcd5f73" {
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
}
@@ -176,81 +176,8 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
// A failure recorded after the ask fails the plan, as hearing it would have.
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}})
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
}
}
// The first machine's report, made between the send to it and the send to the rest, opens the gate for it:
// each machine is judged from its own send, not from the last one (novox/hq issue 256).
func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC)
firstSent := built.Add(31 * time.Second)
firstReported := built.Add(43 * time.Second)
restSent := built.Add(58 * time.Second)
restReported := built.Add(74 * time.Second)
reports := []inventory.Reported{
{Node: "ace", At: &firstReported},
{Node: "g14", At: &restReported},
}
since := func(node string) time.Time {
if node == "ace" {
return firstSent
}
return restSent
}
if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok {
t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting)
}
// The old reading, every machine from the last send, is what held the plan.
if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok {
t.Fatal("the single-moment reading should hold the first machine back")
}
early := built.Add(10 * time.Second)
if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" {
t.Fatal("a report from before the machine was sent opened the gate")
}
}
// novox/hq issue 278 and ADR 0236's open question: a change to the build agent — its manifest alone, as
// the catalogue's data sections were, or its program — rebuilds the build agent and nothing it builds.
// What it builds is ordered after it in a plan that holds both, never added to one for its sake. The
// merge that rebuilt 103 modules with the agent in tier 0 rebuilt them for a file read as shared code;
// the agent stood first only because everything else is built by it.
func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) {
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
agent := fromRepo("build-agent", repo, "modules/build-agent")
redis := fromRepo("redis", repo, "modules/redis")
postgres := fromRepo("postgres", repo, "modules/postgres")
entries := []inventory.Entry{agent, redis, postgres}
edges := []inventory.Edge{
{From: "redis", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
{From: "redis", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "postgres", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "mesh-tools", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
}
for _, paths := range [][]string{
{"modules/build-agent/module.json"}, // its manifest alone
{"modules/build-agent/cmd/agent/main.go", "modules/build-agent/module.json"}, // its program too
} {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", Paths: paths,
ModuleDirs: []string{"modules/build-agent"}, ModuleDirsSaid: true}
touched := whatTheMergeTouched(entries, entries, m)
if len(touched) != 1 || touched[0].Manifest.Module != "build-agent" {
t.Fatalf("%v touched %v", paths, touched)
}
p := planOfMerge(m, []string{"build-agent"}, edges)
if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "build-agent" || len(p.Modules) != 1 {
t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers)
}
}
// With what it builds moved beside it, the agent comes first and they follow: an order, not a widening.
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"},
[]string{"build-agent", "redis"}, edges)
if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 {
t.Fatalf("the agent, then what moved beside it: %v", p.Tiers)
}
}
cmd/mesh-controller/release_test.go
-257
View File
@@ -1,257 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The backlog the old default left — builds registered and sent nowhere — is released by a plan, one
// machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236).
type backlogMesh struct {
open *stores
sent [][]string
broken map[string]bool
}
// aBacklog is a mesh where `app` moved c1 → c2 on anchor and laptop, `late` moved on laptop only, and
// `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once.
func aBacklog(t *testing.T) *backlogMesh {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
b := &backlogMesh{open: open, broken: map[string]bool{}}
withConditionsInMemory(t)
was := doctorFrom
doctorFrom = nil
t.Cleanup(func() { doctorFrom = was })
build := func(module, commit, made string, asked time.Time) {
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}}
for _, m := range []string{"app", "late", "same"} {
build(m, "c1", "sha256:"+m+"-1", old)
for _, n := range on[m] {
if _, err := inv.Assign(ctx, n, m); err != nil {
t.Fatal(err)
}
}
}
for _, n := range []string{"anchor", "laptop"} {
sent := map[string]string{}
for m, nodes := range on {
for _, x := range nodes {
if x == n {
sent[m] = "c1"
}
}
}
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
t.Fatal(err)
}
}
build("app", "c2", "sha256:app-2", now)
build("late", "c2", "sha256:late-2", now)
build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes
assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
machineMoves = assigned
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
return map[string]bool{"anchor": true, "laptop": true}, nil
}
n := 0
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
b.sent = append(b.sent, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
modules, _ := open.inventory.Assigned(ctx, node)
carried := map[string]string{}
for _, m := range modules {
carried[m] = current[m].Commit
}
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
if b.broken[r.Node] {
r.Outcome = inventory.OutcomeFailed
}
f.reports[r.Node] = r
}
return f, nil
}
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
})
return b
}
func (b *backlogMesh) release(t *testing.T) inventory.Plan {
t.Helper()
plans, err := b.open.inventory.RecentPlans(t.Context(), 10)
if err != nil {
t.Fatal(err)
}
for _, p := range plans {
if p.Release != nil {
return p
}
}
t.Fatal("no release plan")
return inventory.Plan{}
}
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
f, err := readMoveFacts(ctx, inv)
if err != nil {
t.Fatal(err)
}
moves, _ := machineMoves(ctx, b.open, f, "laptop", false)
var names []string
for _, mv := range moves {
names = append(names, mv.Module)
}
if !reflect.DeepEqual(names, []string{"app", "late"}) {
t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names)
}
// Nothing else may carry them: a send that judges nothing is refused.
if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) {
t.Fatalf("a send that judges nothing would carry them: %v", err)
}
advancePlans(ctx, b.open)
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent)
}
p := b.release(t)
if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil {
t.Fatalf("the release plan is %+v", p.Release)
}
gateEvery = 0
for i := 0; i < 4; i++ {
advancePlans(ctx, b.open)
}
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("sent %v", b.sent)
}
p = b.release(t)
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) {
t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release)
}
for _, build := range []string{"build-app-c2", "build-late-c2"} {
if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err)
}
}
// Nothing waits now, and nothing opens again.
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
t.Fatalf("a release opened with nothing waiting: %+v %v", p, err)
}
}
// A release that fails on its first machine puts back what it carried there, goes no further, and the
// next one waits for a person, said as a condition; a person releases it with why.
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
gateEvery = 0
b.broken["anchor"] = true
advancePlans(ctx, b.open)
p := b.release(t)
if p.State != inventory.PlanFailed {
t.Fatalf("the release is %s: %s", p.State, p.Note)
}
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) {
t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("app is at %s, not put back", current["app"].Commit)
}
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
t.Fatal("app's build is not marked failed at its gate")
}
// late still waits on laptop, and is not released on its own after a failure.
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
t.Fatalf("a release opened after a failed one: %+v %v", p, err)
}
if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") {
t.Fatalf("the held release is not said: %+v", obs)
}
b.broken["anchor"] = false
if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
advancePlans(ctx, b.open)
}
if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" {
t.Fatalf("the person's release is %s (%+v)", p.State, p.Release)
}
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
t.Fatalf("late was not released to laptop: %v", sent)
}
}
cmd/mesh-controller/replays_test.go
-113
View File
@@ -1,113 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
// happened, asserting the rule's outcome rather than the fix's mechanism, so it can be run against the
// commit before the fix and fail there, and against the fix and pass. **Written only with what the
// controller had before each fix** — the stores, register, assign, planFor, declarationFor — so the
// prover (mesh-lab replays/cmd/prove) can lay this file over the older commit and run it there.
//
// Registered in mesh-lab's replays/register.go with the fix each one proves; run by this repository's
// merge check on every pull request with the rest of the suite.
// **R263 — a consumer's identity never refuses its provider's machine.** On 2026-10-06 the network
// manager came to require the mesh's resolver; on a machine whose name made its identity 23 to 26
// characters against the one global bound of 20, the anchor — the resolver's holder, carrying every
// consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the
// provider's machine composes whatever its consumers are called, and a provision that mints no
// credential holds no consumer to a key's length.
func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
Requires: []string{"wildcard-resolution"}})
for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
// The consumer's machine composes first, as a push does: what it is sent is what its provider grants.
for _, node := range []string{"laptop", "anchor"} {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatalf("%s does not resolve: %v", node, err)
}
if _, err := declarationFor(ctx, open, node, plan, settings); err != nil {
t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+
"mesh_laptop_networkmanager (26 characters), refused it: %v", node, err)
}
}
}
// **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for
// the resolver re-bound every database consumer on the home server — which runs its own store, beside
// its applications' data — to the store seat's holder on the control node, which made each a new empty
// database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a
// machine running its own store stays bound to it while another machine holds the store's seat; the
// resolver, which every holder answers alike, follows its seat.
func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
for _, m := range []catalogue.Manifest{
{Module: "store", Version: "1",
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
} {
register(t, open, m)
}
assignAll := func(pairs ...[2]string) {
for _, a := range pairs {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
}
// The control node holds the mesh's store and resolver seats; the home server runs its own of each.
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
t.Fatal(err)
}
}
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
[2]string{"laptop", "board"})
plan, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
var board, network string
for _, n := range plan.Needs {
switch {
case n.For == "board" && n.Name == "postgres-database":
board = n.From
case n.For == "network" && n.Name == "wildcard-resolution":
network = n.From
}
}
if board != "laptop" {
t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+
"database there (issue 273)", board)
}
if network != "anchor" {
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
}
}
cmd/mesh-controller/retire_verbs.go
-479
View File
@@ -1,479 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The verbs a person answers a provider's retirement with, and cleans up with (novox/hq ADR 0230).
//
// **The controller asks; the provider acts.** Approving, rejecting and deleting are each a question to
// the provider on the machine it runs on — its `provisioner_*` tools — because the provider owns its
// backend and the controller touches none. Every one says why, is written in the hand-act log before
// it is asked, and the provider says what it did as its `provisioner.retirement` event.
const retireUsage = "retire [--json] | retire approve <node> <module> --why <text> | retire reject <node> <module> --why <text>"
const cleanupUsage = "cleanup [list] [--json] | cleanup delete <node> <module> <consumer> --why <text> | " +
"cleanup delete --older-than <days> --why <text> [--confirm]"
func isNothingServes(err error) bool { return errors.Is(err, link.ErrNothingServes) }
func unmarshalAnswer(a link.Answer, v any) error {
if len(a.Result) == 0 {
return errors.New("an empty answer")
}
return json.Unmarshal(a.Result, v)
}
// retireCommand is `retire`, `retire approve` and `retire reject`.
func retireCommand(ctx context.Context, args []string) error {
sub := "list"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
switch sub {
case "list":
set := flag.NewFlagSet("retire", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(retireUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error { return listRetiring(ctx, open.inventory, conn, *asJSON) })
case "approve", "reject":
set := flag.NewFlagSet("retire "+sub, flag.ContinueOnError)
f := addHandActFlags(set)
rest, err := parseAround(set, args)
if err != nil {
return err
}
if len(rest) != 2 {
return errors.New(retireUsage)
}
if err := f.require("retire " + sub); err != nil {
return err
}
return onTheBus(func(conn *nats.Conn) error {
return answerRetirement(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, sub == "approve", f)
})
}
return errors.New(retireUsage)
}
// retiringRow is one provider's waiting or rejected set, as `retire` lists it.
type retiringRow struct {
Node string `json:"node"`
Module string `json:"module"`
Waiting []link.RetiredConsumer `json:"waiting,omitempty"`
Since string `json:"since,omitempty"`
Held int `json:"held,omitempty"`
Bound string `json:"bound,omitempty"`
Rejected []link.RetiredConsumer `json:"rejected,omitempty"`
RejectWhy string `json:"rejected-why,omitempty"`
Unasked string `json:"unasked,omitempty"`
}
func listRetiring(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, asJSON bool) error {
instances, err := providerInstances(ctx, inv)
if err != nil {
return err
}
var rows []retiringRow
for _, p := range instances {
row := retiringRow{Node: p.Node, Module: p.Module}
state, err := askRetirement(ctx, conn, p)
switch {
case isNothingServes(err):
row.Unasked = "answers no retirement question: it predates ADR 0230"
case err != nil:
row.Unasked = err.Error()
default:
if state.Waiting != nil {
row.Waiting, row.Since, row.Held = state.Waiting.Consumers, state.Waiting.Since, state.Waiting.Held
}
if state.Rejected != nil {
row.Rejected, row.RejectWhy = state.Rejected.Consumers, orWhy(state.Rejected.By, state.Rejected.Why)
}
row.Bound = state.Bound
if row.Waiting == nil && row.Rejected == nil {
continue
}
}
rows = append(rows, row)
}
if asJSON {
if rows == nil {
rows = []retiringRow{}
}
return printJSON(map[string]any{"providers": rows})
}
said := false
for _, r := range rows {
switch {
case r.Unasked != "":
fmt.Printf("%s on %s: not asked — %s\n", r.Module, r.Node, r.Unasked)
default:
if r.Waiting != nil {
said = true
fmt.Printf("%s on %s WAITS since %s to retire %d of the %d it holds (%s): %s\n"+
" retire approve %s %s --why … | retire reject %s %s --why …\n",
r.Module, r.Node, r.Since, len(r.Waiting), r.Held, orBound(r.Bound), consumerList(r.Waiting),
r.Node, r.Module, r.Node, r.Module)
}
if r.Rejected != nil {
said = true
fmt.Printf("%s on %s keeps active, by a rejection (%s): %s\n", r.Module, r.Node, r.RejectWhy,
consumerList(r.Rejected))
}
}
}
if !said {
fmt.Println("no provider waits for a person to approve a retirement")
}
return nil
}
// answerRetirement approves or rejects what one provider waits with. **The set sent is the set the
// provider says it waits with, read now**, and the provider refuses any other: a person approves what
// they were shown, never a set that moved since.
func answerRetirement(ctx context.Context, conn *nats.Conn, p providerInstance, approve bool, f handActFlags) error {
state, err := askRetirement(ctx, conn, p)
if err != nil {
return err
}
verb, tool := "retire reject", link.ToolRetireReject
var set []link.RetiredConsumer
if state.Waiting != nil {
set = state.Waiting.Consumers
}
if approve {
verb, tool = "retire approve", link.ToolRetireApprove
if set == nil && state.Rejected != nil {
// A rejection can be taken back: the consumers it kept are retired after all.
set = state.Rejected.Consumers
}
}
if len(set) == 0 {
return fmt.Errorf("%s on %s waits for nobody to approve or reject a retirement. Nothing was done", p.Module, p.Node)
}
names := make([]string, 0, len(set))
for _, c := range set {
names = append(names, c.Consumer)
}
if strings.TrimSpace(*f.cause) == "" {
*f.cause = kindRetireWaiting
}
if strings.TrimSpace(*f.condition) == "" {
*f.condition = retireWaitingKey(p.Module, p.Node)
}
f.record(ctx, verb, append([]string{p.Node, p.Module}, names...))
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, tool, p.Node, map[string]any{
"consumers": names, "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController,
}, retirementAsk)
if err != nil {
return err
}
if answer.Error != "" {
return fmt.Errorf("%s on %s refused: %s", p.Module, p.Node, answer.Error)
}
if approve && state.RetiresBy == "mark-only" {
fmt.Printf("%s on %s marked %s retired, MARK ONLY: this provider cannot disable a consumer, so they keep "+
"their access until `cleanup delete`; `cleanup list` shows them\n", p.Module, p.Node, strings.Join(names, ", "))
} else if approve {
fmt.Printf("%s on %s retired %s: access disabled, data kept; `cleanup list` shows them\n", p.Module, p.Node,
strings.Join(names, ", "))
} else {
fmt.Printf("%s on %s keeps %s active; the warning stays open until the mesh asks for them again or the "+
"retirement is approved\n", p.Module, p.Node, strings.Join(names, ", "))
}
return nil
}
// cleanupCommand is `cleanup list` and `cleanup delete`.
func cleanupCommand(ctx context.Context, args []string) error {
sub := "list"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
switch sub {
case "list":
set := flag.NewFlagSet("cleanup", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(cleanupUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error {
listing, err := gatherRetired(ctx, open.inventory, conn, time.Now())
if err != nil {
return err
}
return printRetired(listing, *asJSON)
})
case "delete":
set := flag.NewFlagSet("cleanup delete", flag.ContinueOnError)
f := addHandActFlags(set)
olderThan := set.Int("older-than", 0, "every retired consumer older than this many days")
confirm := set.Bool("confirm", false, "with --older-than: delete what is listed, rather than only list it")
rest, err := parseAround(set, args)
if err != nil {
return err
}
if err := f.require("cleanup delete"); err != nil {
return err
}
if strings.TrimSpace(*f.cause) == "" {
*f.cause = kindCleanupWaiting
}
switch {
case *olderThan > 0 && len(rest) == 0:
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error {
return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now())
})
case *olderThan == 0 && len(rest) == 3 && !*confirm:
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error {
// A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a
// provider's retired consumer otherwise.
if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil &&
r.RetiredAt != nil {
return deleteRetiredData(ctx, conn, open, r, f)
}
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
})
}
return errors.New(cleanupUsage)
}
return errors.New(cleanupUsage)
}
// retiredRow is one retired consumer, as `cleanup list` shows it.
type retiredRow struct {
Node string `json:"node"`
Module string `json:"module"`
Consumer string `json:"consumer"`
// ConsumerNode is where the consumer was, when the provider knows.
ConsumerNode string `json:"consumer-node,omitempty"`
Kind string `json:"kind,omitempty"`
RetiredAt string `json:"retired-at,omitempty"`
// AgeDays is whole days since it was retired; -1 when the provider could not say when.
AgeDays int `json:"age-days"`
SizeBytes *int64 `json:"size-bytes,omitempty"`
Why string `json:"why,omitempty"`
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
Access string `json:"access,omitempty"`
// Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is
// kept on its machine, and its class. Consumer is then the item.
Path string `json:"path,omitempty"`
Class string `json:"class,omitempty"`
}
// retiredListing is every provider's retired consumers, and the providers that could not say.
type retiredListing struct {
Retired []retiredRow `json:"retired"`
// Unasked names each provider not asked, and why: one older than the question, or one that failed.
Unasked []string `json:"unasked,omitempty"`
}
func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, now time.Time) (retiredListing, error) {
instances, err := providerInstances(ctx, inv)
if err != nil {
return retiredListing{}, err
}
listing := retiredOf(ctx, conn, instances, now)
// And every module's own data retired on its machine (novox/hq ADR 0233).
records, err := inv.Data(ctx)
if err != nil {
return retiredListing{}, err
}
listing.Retired = append(listing.Retired, retiredData(records, now)...)
sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays })
return listing, nil
}
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
out := retiredListing{Retired: []retiredRow{}}
for _, p := range instances {
state, err := askRetirement(ctx, conn, p)
if err != nil {
if isNothingServes(err) {
out.Unasked = append(out.Unasked, fmt.Sprintf("%s on %s answers no retirement question: it predates ADR 0230", p.Module, p.Node))
} else {
out.Unasked = append(out.Unasked, err.Error())
}
continue
}
for _, c := range state.Retired {
row := retiredRow{Node: p.Node, Module: p.Module, Consumer: c.Consumer, ConsumerNode: c.Node, Kind: c.Kind,
RetiredAt: c.RetiredAt, AgeDays: -1, SizeBytes: c.SizeBytes, Why: c.Why, Access: c.Access}
if at := retiredAt(c); !at.IsZero() {
row.AgeDays = int(now.Sub(at).Hours() / 24)
}
out.Retired = append(out.Retired, row)
}
}
sort.SliceStable(out.Retired, func(i, j int) bool { return out.Retired[i].AgeDays > out.Retired[j].AgeDays })
return out
}
func printRetired(l retiredListing, asJSON bool) error {
if asJSON {
return printJSON(l)
}
if len(l.Retired) == 0 {
fmt.Println("no provider holds a retired consumer, and no machine holds retired data")
}
for _, r := range l.Retired {
age := "age unknown"
if r.AgeDays >= 0 {
age = strconv.Itoa(r.AgeDays) + " day(s)"
}
kind := ""
if r.Kind != "" && r.Kind != "consumer" {
kind = " [" + r.Kind + "]"
}
if r.Access == "kept" {
kind += " [MARK ONLY: access kept until deleted]"
}
if r.Kind == retiredDataKind {
fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer,
r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
continue
}
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
}
for _, u := range l.Unasked {
fmt.Printf("not asked: %s\n", u)
}
return nil
}
// deleteRetired asks one provider to delete one consumer it holds retired — never an active one: the
// provider refuses that, and this refuses it first, from what the provider says it holds.
func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, consumer string, f handActFlags) error {
state, err := askRetirement(ctx, conn, p)
if err != nil {
return err
}
found := false
for _, c := range state.Retired {
found = found || c.Consumer == consumer
}
if !found {
return fmt.Errorf("%s on %s holds no retired consumer %s — only a retired consumer is deleted. Nothing was done",
p.Module, p.Node, consumer)
}
f.record(ctx, "cleanup delete", []string{p.Node, p.Module, consumer})
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetiredDelete, p.Node, map[string]any{
"consumer": consumer, "confirm": consumer, "why": strings.TrimSpace(*f.why), "by": link.Caller(),
"via": link.ViaController,
}, retirementAsk)
if err != nil {
return err
}
if answer.Error != "" {
return fmt.Errorf("%s on %s refused to delete %s: %s", p.Module, p.Node, consumer, answer.Error)
}
var done struct {
FreedBytes *int64 `json:"freed_bytes"`
}
_ = unmarshalAnswer(answer, &done)
fmt.Printf("%s on %s deleted %s (%s freed)\n", p.Module, p.Node, consumer, sizeWords(done.FreedBytes))
return nil
}
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
// One whose age the provider cannot say is never in it.
func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool,
f handActFlags, now time.Time) error {
listing, err := gatherRetired(ctx, open.inventory, conn, now)
if err != nil {
return err
}
return deleteFrom(ctx, conn, open, listing, days, confirm, f)
}
func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error {
var due []retiredRow
unknown := 0
for _, r := range listing.Retired {
switch {
case r.AgeDays < 0:
unknown++
case r.AgeDays > days:
due = append(due, r)
}
}
for _, u := range listing.Unasked {
fmt.Printf("not asked: %s\n", u)
}
if unknown > 0 {
fmt.Printf("%d retired consumer(s) whose age their provider cannot say are left out\n", unknown)
}
if len(due) == 0 {
fmt.Printf("nothing has been retired more than %d day(s)\n", days)
return nil
}
fmt.Printf("retired more than %d day(s):\n", days)
for _, r := range due {
fmt.Printf(" %s on %s: %s — %d day(s), %s\n", r.Module, r.Node, r.Consumer, r.AgeDays, sizeWords(r.SizeBytes))
}
if !confirm {
fmt.Printf("nothing was deleted: add --confirm to delete these %d\n", len(due))
return nil
}
var failed []string
for _, r := range due {
var err error
if r.Kind == retiredDataKind {
var rec inventory.DataRecord
if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil {
err = deleteRetiredData(ctx, conn, open, rec, f)
}
} else {
err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f)
}
if err != nil {
failed = append(failed, err.Error())
}
}
if len(failed) > 0 {
return fmt.Errorf("%d of %d not deleted: %s", len(failed), len(due), strings.Join(failed, "; "))
}
return nil
}
cmd/mesh-controller/retirement.go
-343
View File
@@ -1,343 +0,0 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
// (novox/hq ADR 0230, the operator's decision of 2026-10-06; it replaces ADR 0229's withdrawal brake).
//
// A provider retires a consumer — disables its access, keeps its data, marks it with when and why —
// once it has seen the same consumers go unasked for in five passes. More than three at once, or more
// than half of those it holds where it holds more than one, is a person actively working on the mesh,
// so the provider retires nothing and waits: the controller keeps that as an urgent condition naming
// what would go and the two verbs that answer it. A retired consumer is deleted only by `cleanup
// delete`, which the provider executes on its own backend — the controller never touches one — and
// one retired more than thirty days is a warning that cleanup is waiting (D11).
// The kinds a provider's retirement word raises.
const (
kindRetireWaiting = "retire-waiting"
kindRetireRejected = "retire-rejected"
kindCleanupWaiting = "cleanup-waiting"
// sourceRetirement is what raised a retirement condition: the provider's own event.
sourceRetirement = "provisioner.retirement"
)
// cleanupAfter is how long a consumer may stay retired before cleanup is said to be waiting (D11).
const cleanupAfter = 30 * 24 * time.Hour
// retirementAsk is how long one provider is given to answer one question about its retired consumers.
var retirementAsk = 20 * time.Second
func retireWaitingKey(module, node string) string {
return conditions.Key(conditions.ScopeProvider, module+"."+node, "retire")
}
func retireRejectedKey(module, node string) string {
return conditions.Key(conditions.ScopeProvider, module+"."+node, "retire-rejected")
}
// retirements keeps what providers say about consumers they retire, as conditions.
type retirements struct {
keeper func() *conditions.Keeper
// record writes an act done by hand that reached a provider some other way than this controller's
// verbs; nil records nothing (a test).
record func(ctx context.Context, act link.HandAct) error
}
func consumerList(cs []link.RetiredConsumer) string {
parts := make([]string, 0, len(cs))
for _, c := range cs {
p := c.Consumer
if c.Node != "" {
p += " (" + c.Node + ")"
}
parts = append(parts, p)
}
return strings.Join(parts, ", ")
}
// waitingObservation is a provider waiting for a person to approve or reject a retirement.
func waitingObservation(r link.Retirement) conditions.Observation {
since := r.Since
if since.IsZero() {
since = r.At
}
summary := fmt.Sprintf("%s on %s would retire %d consumer(s) the mesh no longer asks for — %s — more than its "+
"bound (%s), so it retires nothing until a person answers: `retire approve %s %s --why …` or `retire reject "+
"%s %s --why …`", r.Module, r.ProviderNode, len(r.Consumers), consumerList(r.Consumers), orBound(r.Bound),
r.ProviderNode, r.Module, r.ProviderNode, r.Module)
said := fmt.Sprintf("waiting since %s, %d held: %s", since.UTC().Format("2006-01-02 15:04 MST"), r.Held,
consumerList(r.Consumers))
return conditions.Observation{Scope: conditions.ScopeProvider, ID: r.Module + "." + r.ProviderNode,
Token: "retire", Kind: kindRetireWaiting, Machine: r.ProviderNode, Also: consumerNodes(r),
Severity: conditions.Urgent, Summary: summary, Said: said, Source: sourceRetirement,
Resolver: conditions.ResolverOperator}
}
// rejectedObservation is consumers kept active by a person's rejection though the mesh asks for them no more.
func rejectedObservation(r link.Retirement) conditions.Observation {
summary := fmt.Sprintf("%s on %s keeps %s active although the mesh no longer asks for them: a person rejected "+
"their retirement (%s). Assign them again, or `retire approve %s %s --why …`", r.Module, r.ProviderNode,
consumerList(r.Consumers), orWhy(r.By, r.Why), r.ProviderNode, r.Module)
return conditions.Observation{Scope: conditions.ScopeProvider, ID: r.Module + "." + r.ProviderNode,
Token: "retire-rejected", Kind: kindRetireRejected, Machine: r.ProviderNode, Also: consumerNodes(r),
Severity: conditions.Warning, Summary: summary, Said: "rejected: " + orWhy(r.By, r.Why),
Source: sourceRetirement, Resolver: conditions.ResolverOperator}
}
func orBound(b string) string {
if b == "" {
return "more than 3, or more than half of those held"
}
return b
}
func orWhy(by, why string) string {
switch {
case by != "" && why != "":
return by + ": " + why
case why != "":
return why
case by != "":
return "by " + by
}
return "no reason given"
}
// consumerNodes are the other machines a retirement concerns: where its consumers are.
func consumerNodes(r link.Retirement) []string {
seen := map[string]bool{r.ProviderNode: true}
var out []string
for _, c := range r.Consumers {
if c.Node != "" && !seen[c.Node] {
seen[c.Node] = true
out = append(out, c.Node)
}
}
sort.Strings(out)
return out
}
// Retired keeps one word. Waiting raises the urgent condition; a rejection turns it into a warning; a
// retirement, an approval or the set settling clears both. An approval, a rejection or a deletion that
// did not come through this controller's verbs is recorded in the hand-act log here, so every one is.
func (s retirements) Retired(ctx context.Context, r link.Retirement) error {
k := s.keeper()
if k == nil {
return fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
}
waiting, rejected := retireWaitingKey(r.Module, r.ProviderNode), retireRejectedKey(r.Module, r.ProviderNode)
clear := func(keys ...string) error {
for _, key := range keys {
if _, err := k.Clear(ctx, key, fmt.Sprintf("%s on %s says %s", r.Module, r.ProviderNode, r.Change)); err != nil {
return storeAway(err)
}
}
return nil
}
var err error
switch r.Change {
case link.RetireWaiting:
if _, err = k.Observe(ctx, waitingObservation(r)); err != nil {
return storeAway(err)
}
case link.RetireRejected:
if err = clear(waiting); err != nil {
return err
}
if _, err = k.Observe(ctx, rejectedObservation(r)); err != nil {
return storeAway(err)
}
case link.RetireApproved, link.RetireRetired, link.RetireSettled:
if err = clear(waiting, rejected); err != nil {
return err
}
}
switch r.Change {
case link.RetireApproved, link.RetireRejected, link.RetireDeleted:
if r.Via != link.ViaController && s.record != nil {
verb := map[string]string{link.RetireApproved: "retire approve", link.RetireRejected: "retire reject",
link.RetireDeleted: "cleanup delete"}[r.Change]
cause := kindRetireWaiting
if r.Change == link.RetireDeleted {
cause = kindCleanupWaiting
}
why := r.Why
if strings.TrimSpace(why) == "" {
why = "no reason given to the provider"
}
act := link.HandAct{Verb: verb, Args: append([]string{r.ProviderNode, r.Module}, r.Names()...),
Why: why + " (asked of the provider directly, not through the controller)", Cause: cause,
By: r.By, At: r.At}
if act.By == "" {
act.By = "unknown, asked of " + r.Module + " on " + r.ProviderNode + " directly"
}
if err := s.record(ctx, act); err != nil {
return fmt.Errorf("%s on %s %s by hand, and it could not be recorded: %v: %w", r.Module,
r.ProviderNode, r.Change, err, link.ErrTryAgain)
}
}
}
return nil
}
// recordHandActOnTheBus writes an act through the serving controller's connection.
func recordHandActOnTheBus(ctx context.Context, act link.HandAct) error {
return onTheBus(func(conn *nats.Conn) error {
_, err := link.RecordHandAct(ctx, conn, act)
return err
})
}
// providerInstance is one provider module on one machine.
type providerInstance struct {
Node, Module string
}
// providerInstances are every module assigned on every machine whose manifest receives contributions:
// every provider, which serves the retirement tools (ADR 0230) — or is older than them.
func providerInstances(ctx context.Context, inv *inventory.Inventory) ([]providerInstance, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
}
var out []providerInstance
for _, n := range nodes {
modules, err := inv.Assigned(ctx, n.Name)
if err != nil {
return nil, fmt.Errorf("what %s is assigned cannot be read: %w", n.Name, err)
}
for _, m := range modules {
if man, ok := shelf[m]; ok && len(man.Receives) > 0 {
out = append(out, providerInstance{Node: n.Name, Module: m})
}
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Node != out[j].Node {
return out[i].Node < out[j].Node
}
return out[i].Module < out[j].Module
})
return out, nil
}
// askRetirement asks one provider for what it holds retired and what waits.
func askRetirement(ctx context.Context, conn *nats.Conn, p providerInstance) (link.RetirementState, error) {
var state link.RetirementState
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetirement, p.Node, map[string]any{}, retirementAsk)
if err != nil {
return state, err
}
if answer.Error != "" {
return state, fmt.Errorf("%s on %s answered %s with an error: %s", p.Module, p.Node, link.ToolRetirement, answer.Error)
}
if err := unmarshalAnswer(answer, &state); err != nil {
return state, fmt.Errorf("%s on %s answered %s with something unreadable: %w", p.Module, p.Node, link.ToolRetirement, err)
}
return state, nil
}
// retiredAt reads a retired consumer's moment; zero when the provider could not say.
func retiredAt(c link.RetiredConsumer) time.Time {
t, err := time.Parse(time.RFC3339, c.RetiredAt)
if err != nil {
return time.Time{}
}
return t
}
// cleanupObservation is a provider holding consumers retired longer than cleanupAfter.
func cleanupObservation(p providerInstance, old []link.RetiredConsumer, now time.Time) conditions.Observation {
parts := make([]string, 0, len(old))
for _, c := range old {
parts = append(parts, fmt.Sprintf("%s (%d days, %s)", c.Consumer, int(now.Sub(retiredAt(c)).Hours()/24),
sizeWords(c.SizeBytes)))
}
return conditions.Observation{Scope: conditions.ScopeProvider, ID: p.Module + "." + p.Node, Token: "cleanup",
Kind: kindCleanupWaiting, Machine: p.Node, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s on %s holds %d consumer(s) retired more than %d days, waiting for a person to "+
"delete or bring them back: %s — `cleanup list`, then `cleanup delete %s %s <consumer> --why …`",
p.Module, p.Node, len(old), int(cleanupAfter.Hours()/24), strings.Join(parts, ", "), p.Node, p.Module),
Resolver: conditions.ResolverOperator}
}
func sizeWords(size *int64) string {
if size == nil || *size < 0 {
return "size unknown"
}
b := float64(*size)
for _, unit := range []string{"B", "KB", "MB", "GB"} {
if b < 1024 || unit == "GB" {
if unit == "B" {
return fmt.Sprintf("%d B", *size)
}
return fmt.Sprintf("%.1f %s", b, unit)
}
b /= 1024
}
return ""
}
// probeRetired is D11: no provider holds a consumer retired more than thirty days. Every provider
// assigned is asked what it holds retired; one that does not serve the question — an older build, a
// TypeScript provider not yet on the SDK that retires — has nothing it can say and is passed over,
// which is not a failure of the probe. A provider that cannot be asked otherwise is.
func probeRetired(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
if d.js == nil {
return nil, fmt.Errorf("no bus to ask the providers over")
}
instances, err := providerInstances(ctx, d.open.inventory)
if err != nil {
return nil, err
}
return retiredTooLong(ctx, d.js.Conn(), instances, time.Now())
}
// retiredTooLong asks each provider and answers one observation per provider holding anything retired
// longer than cleanupAfter.
func retiredTooLong(ctx context.Context, conn *nats.Conn, instances []providerInstance,
now time.Time) ([]conditions.Observation, error) {
var out []conditions.Observation
var problems []string
for _, p := range instances {
state, err := askRetirement(ctx, conn, p)
if err != nil {
if isNothingServes(err) {
continue
}
problems = append(problems, err.Error())
continue
}
var old []link.RetiredConsumer
for _, c := range state.Retired {
if at := retiredAt(c); !at.IsZero() && now.Sub(at) > cleanupAfter {
old = append(old, c)
}
}
if len(old) > 0 {
out = append(out, cleanupObservation(p, old, now))
}
}
if len(problems) > 0 {
// Not "nothing retired": a provider that could not be asked may hold the oldest of all.
return nil, fmt.Errorf("%s", strings.Join(problems, "; "))
}
return out, nil
}
cmd/mesh-controller/retirement_test.go
-481
View File
@@ -1,481 +0,0 @@
package main
import (
"context"
"encoding/json"
"flag"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
// (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that
// answer it ask the provider — never anything else — for exactly what it said.
func waitingWord(module, node string, names ...string) link.Retirement {
r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting,
Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()}
for _, n := range names {
r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"})
}
return r
}
func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition {
t.Helper()
all, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range all {
out[c.Key] = c
}
return out
}
func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) {
k, _ := withConditionsInMemory(t)
var recorded []link.HandAct
r := retirements{keeper: func() *conditions.Keeper { return k },
record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }}
ctx := t.Context()
waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d")
if err := r.Retired(ctx, waiting); err != nil {
t.Fatal(err)
}
open := openKeys(t, k)
c, ok := open["provider.postgres.anchor.retire"]
if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent {
t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open)
}
for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres",
"retire reject anchor postgres", "more than 3"} {
if !strings.Contains(c.Summary, want) {
t.Errorf("the condition does not say %q: %s", want, c.Summary)
}
}
// Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here —
// the verb recorded it before it asked.
rejected := waiting
rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController
if err := r.Retired(ctx, rejected); err != nil {
t.Fatal(err)
}
open = openKeys(t, k)
if _, still := open["provider.postgres.anchor.retire"]; still {
t.Fatal("a rejection left the provider waiting")
}
if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning ||
c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") {
t.Fatalf("a rejection is not a warning saying why: %+v", open)
}
if len(recorded) != 0 {
t.Fatalf("an act through the controller was recorded twice: %+v", recorded)
}
// Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here.
approved := waiting
approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", ""
if err := r.Retired(ctx, approved); err != nil {
t.Fatal(err)
}
if open := openKeys(t, k); len(open) != 0 {
t.Fatalf("an approval left conditions open: %+v", open)
}
if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" ||
!strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") {
t.Fatalf("an approval outside the controller was not recorded: %+v", recorded)
}
// Waiting again, then the set settles (asked for again): cleared. And retired clears too.
for _, change := range []string{link.RetireSettled, link.RetireRetired} {
if err := r.Retired(ctx, waiting); err != nil {
t.Fatal(err)
}
done := waiting
done.Change = change
if err := r.Retired(ctx, done); err != nil {
t.Fatal(err)
}
if open := openKeys(t, k); len(open) != 0 {
t.Fatalf("%s left conditions open: %+v", change, open)
}
}
// A deletion asked of the provider directly is recorded too.
deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x",
Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}}
if err := r.Retired(ctx, deleted); err != nil {
t.Fatal(err)
}
if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting {
t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded)
}
}
func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) {
body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting",
"held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"})
r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body)
if err != nil || r.Module != "idp" || len(r.Consumers) != 2 {
t.Fatalf("%+v %v", r, err)
}
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement",
[]byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil {
t.Fatal("a change the mesh has no name for was read")
}
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil {
t.Fatal("a failing word was read as a retirement")
}
k, _ := withConditionsInMemory(t)
if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil {
t.Fatal(err)
}
if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok {
t.Fatal("not kept under the emitter the subject names")
}
}
func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "pg", Version: "1",
Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}})
if _, err := assign(ctx, open, "anchor", "pg"); err != nil {
t.Fatal(err)
}
r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }}
for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} {
if err := r.Retired(ctx, w); err != nil {
t.Fatal(err)
}
}
if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{
cleanupObservation(providerInstance{Node: "anchor", Module: "gone"},
[]link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()),
}); err != nil {
t.Fatal(err)
}
all, _ := conditionsFrom.Open(ctx)
if len(all) != 3 {
t.Fatalf("%+v", all)
}
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil {
t.Fatal(err)
}
left := openKeys(t, conditionsFrom)
if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" {
t.Fatalf("only the assigned provider's waiting should stay: %+v", left)
}
}
// fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked.
type fakeProvider struct {
mu sync.Mutex
state link.RetirementState
asked []map[string]any
deleted []string
approved []string
rejected []string
}
func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) {
t.Helper()
answer := func(m *nats.Msg, result any, refusal string) {
body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node})
_ = m.Respond(body)
}
names := func(args map[string]any) []string {
var out []string
for _, v := range args["consumers"].([]any) {
out = append(out, v.(string))
}
sort.Strings(out)
return out
}
set := func(cs []link.RetiredConsumer) []string {
var out []string
for _, c := range cs {
out = append(out, c.Consumer)
}
sort.Strings(out)
return out
}
for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} {
tool := tool
sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) {
f.mu.Lock()
defer f.mu.Unlock()
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
f.asked = append(f.asked, map[string]any{"tool": tool, "args": args})
switch tool {
case link.ToolRetirement:
answer(m, f.state, "")
case link.ToolRetireApprove:
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
answer(m, nil, "not the set I wait with")
return
}
f.approved = names(args)
answer(m, map[string]any{"retired": f.approved}, "")
case link.ToolRetireReject:
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
answer(m, nil, "not the set I wait with")
return
}
f.rejected = names(args)
answer(m, map[string]any{"kept": f.rejected}, "")
case link.ToolRetiredDelete:
if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController {
answer(m, nil, "confirm, why and via")
return
}
f.deleted = append(f.deleted, args["consumer"].(string))
answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "")
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
}
func onATestBus(t *testing.T) *nats.Conn {
t.Helper()
url := testbus.URL(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
return js.Conn()
}
func whyFlags(t *testing.T, why string) handActFlags {
t.Helper()
set := flag.NewFlagSet("t", flag.ContinueOnError)
f := addHandActFlags(set)
if err := set.Parse([]string{"--why", why}); err != nil {
t.Fatal(err)
}
return f
}
func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct {
t.Helper()
acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute))
if err != nil {
t.Fatal(err)
}
var out []link.HandAct
for _, a := range acts {
if a.Verb == verb {
out = append(out, a)
}
}
return out
}
func retiredDaysAgo(name string, days int) link.RetiredConsumer {
size := int64(4096)
return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it",
RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size}
}
func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6}
fake.serve(t, conn, "pg-approve", "anchor")
p := providerInstance{Node: "anchor", Module: "pg-approve"}
said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) })
if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") {
t.Fatalf("approved %v; said %s", fake.approved, said)
}
acts := handActsBy(t, conn, "retire approve")
if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting ||
acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" {
t.Fatalf("the approval is not in the hand-act log: %+v", acts)
}
// Reject, on another provider: the same set, and nothing approved.
other := &fakeProvider{state: fake.state}
other.serve(t, conn, "pg-reject", "anchor")
printed(t, func() error {
return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false,
whyFlags(t, "still moving"))
})
if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil {
t.Fatalf("rejected %v approved %v", other.rejected, other.approved)
}
// Nothing waiting: refused, nothing asked but the question.
idle := &fakeProvider{}
idle.serve(t, conn, "pg-idle", "anchor")
if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true,
whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") {
t.Fatalf("%v", err)
}
if len(idle.asked) != 1 {
t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked)
}
}
func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
fake.state.Held = []string{"active"}
fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)}
fake.serve(t, conn, "pg-clean", "anchor")
p := providerInstance{Node: "anchor", Module: "pg-clean"}
said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) })
if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") {
t.Fatalf("deleted %v; said %s", fake.deleted, said)
}
// An active consumer, or one it does not hold, is refused before the provider is asked to delete.
for _, name := range []string{"active", "nobody"} {
if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil ||
!strings.Contains(err.Error(), "only a retired consumer is deleted") {
t.Fatalf("%s: %v", name, err)
}
}
if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("more was deleted: %v", fake.deleted)
}
if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" {
t.Fatalf("the deletion is not in the hand-act log: %+v", acts)
}
// Older than: listed, and nothing deleted without confirm; with it, only the old one.
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 {
t.Fatalf("%+v", listing)
}
fake.deleted = nil
said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
strings.Contains(said, "young") {
t.Fatalf("deleted %v; said %s", fake.deleted, said)
}
printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("confirmed, deleted %v", fake.deleted)
}
}
func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) {
conn := onATestBus(t)
old := &fakeProvider{}
old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)}
old.serve(t, conn, "pg-d11-old", "anchor")
young := &fakeProvider{}
young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)}
young.serve(t, conn, "pg-d11-young", "anchor")
instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"},
// Nothing serves this one: a provider older than the question is passed over, not a failure.
{Node: "anchor", Module: "pg-d11-predates"}}
found, err := retiredTooLong(t.Context(), conn, instances, time.Now())
if err != nil {
t.Fatal(err)
}
if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting ||
found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") ||
strings.Contains(found[0].Summary, "fresh") {
t.Fatalf("%+v", found)
}
}
func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want string
}{
{"retire", map[string]any{}, "retire --json"},
{"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"},
"retire approve anchor postgres --why moved"},
{"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"},
"retire reject anchor postgres --why no"},
{"cleanup", map[string]any{}, "cleanup list --json"},
{"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"},
"cleanup delete anchor postgres x --why gone"},
{"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"},
{"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"},
"cleanup delete --older-than 30 --why tidy --confirm"},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
}
}
for _, args := range []map[string]any{
{"answer": "approve", "node": "anchor", "module": "postgres"}, // no why
{"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"},
} {
if _, err := argvFor("retire", args); err == nil {
t.Errorf("retire %v was composed", args)
}
}
for _, args := range []map[string]any{
{"node": "anchor", "module": "postgres", "consumer": "x"}, // no why
{"older-than": "30"},
{"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"},
} {
if _, err := argvFor("cleanup", args); err == nil {
t.Errorf("cleanup %v was composed", args)
}
}
if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" ||
repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" {
t.Error("the generic verb would let a retirement or a deletion through without a why")
}
}
// A mark-only provider's retired consumer keeps its access; the listing and the approval say so
// rather than claiming it was disabled (ADR 0230).
func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
kept := retiredDaysAgo("ledger", 3)
kept.Access = "kept"
fake.state.Retired = []link.RetiredConsumer{kept}
fake.state.RetiresBy = "mark-only"
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2}
fake.serve(t, conn, "vault-mark", "anchor")
p := providerInstance{Node: "anchor", Module: "vault-mark"}
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
said := printed(t, func() error { return printRetired(listing, false) })
if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" {
t.Fatalf("%s %+v", said, listing)
}
said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) })
if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") {
t.Fatal(said)
}
}
cmd/mesh-controller/rollout.go
+9 -2
View File
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
// would bury the ones that matter under a list nobody can act on.
func speaksOnTheBus(m catalogue.Manifest) bool {
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
}
@@ -388,7 +388,14 @@ func rolloutMint(ctx context.Context, again bool) error {
}
// providesBus is whether a manifest provides the mesh's bus.
func providesBus(m catalogue.Manifest) bool { return catalogue.ProvidesBus(m) }
func providesBus(m catalogue.Manifest) bool {
for _, o := range m.Provides {
if o.Name == "mesh-bus" {
return true
}
}
return false
}
// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
cmd/mesh-controller/rollout_first_test.go
-152
View File
@@ -1,152 +0,0 @@
package main
import (
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only
// once that machine has reported it applied; a module whose policy says together goes everywhere at
// once, as before.
func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) {
running := []string{"novox", "ace", "g14"}
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
now := sentAt.Add(5 * time.Minute)
bound := 30 * time.Minute
after := sentAt.Add(time.Minute)
next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep {
return nextRollout(s, running, together, reports, now, bound)
}
// Together: every machine at once.
if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first {
t.Fatalf("a together policy did not send every machine at once: %+v", step)
}
// Otherwise the first by name, alone, when none has reported lately.
step := next(inventory.PlanModule{}, running, false, nil)
if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) {
t.Fatalf("the first send was %+v, wanted ace alone", step)
}
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
report := func(outcome string, current bool) []inventory.Reported {
return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current},
{Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}}
}
// No report yet, or one about an older declaration than it was last sent: wait.
for what, reports := range map[string][]inventory.Reported{
"no report": nil,
"a report about older": report(inventory.OutcomeApplied, false),
} {
step := next(state, running, false, reports)
if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" {
t.Errorf("%s: %+v, wanted to wait for ace", what, step)
}
}
// Applied what it was last sent: the rest, and only the rest.
step = next(state, running, false, report(inventory.OutcomeApplied, true))
if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) {
t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step)
}
// Failed or refused: stop, the rest untouched.
for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} {
step := next(state, running, false, report(outcome, true))
if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) ||
!reflect.DeepEqual(step.rest, []string{"novox", "g14"}) {
t.Errorf("a first machine that %s it: %+v", outcome, step)
}
}
// The machine holding the bus went with the first send: the rest wait for it too, and it is
// not sent again.
both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt}
half := report(inventory.OutcomeApplied, true)
if step := next(both, running, false, half); step.waiting != "novox" {
t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step)
}
all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true})
if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) {
t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step)
}
// One machine, or none: nothing is waited for that cannot come.
if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first {
t.Fatalf("a module nothing runs was sent: %+v", step)
}
if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" {
t.Fatalf("a module on one machine waited for more: %+v", step)
}
}
// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there,
// naming the machine and the bound; the rest are left alone.
func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) {
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute)
if !strings.Contains(step.failed, "ace did not report it applied within 30m") ||
!reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 {
t.Fatalf("a silent first machine: %+v", step)
}
}
// The first machine is the first by name among those heard from lately: a laptop that is away is
// not the one the rest wait on. When none has been heard from, the first by name.
func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) {
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour)
reports := []inventory.Reported{
{Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately},
}
step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute)
if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) {
t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step)
}
}
// An announced move of a module an open plan is still rolling out is left to the plan: sending it
// here as well put the bundle on every machine at once (novox/hq issue 249).
func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) {
sent := time.Now()
plans := []inventory.Plan{
{ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}},
{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}},
}
if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" {
t.Fatalf("a module the plan is rolling out was not left to it: %q", got)
}
for _, m := range []string{"gitea", "keycloak"} {
if got := rolledOutByAPlan(plans, m); got != "" {
t.Errorf("%s, which no plan will send, was left to %s", m, got)
}
}
}
// A plan that ends without sending what it built says so, with the remedy; a module whose rollout
// stopped at its first machine is not among them.
func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) {
at := time.Now()
p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1",
Modules: map[string]*inventory.PlanModule{
"agent": {State: "built"},
"stopped": {State: "built", First: []string{"ace"}, FirstAt: &at},
"sent": {State: "built", SentAt: &at},
"notes": {State: "built"},
"later": {},
}}
rollsOut := func(m string) bool { return m != "notes" }
sayUnsent(&p, rollsOut)
sayUnsent(&p, rollsOut)
if p.Note != "closed by hand at tier 1; built and never sent: agent — `push --behind` sends them" {
t.Fatalf("the note reads %q", p.Note)
}
}
cmd/mesh-controller/rotate.go
+1 -34
View File
@@ -6,8 +6,6 @@ import (
"flag"
"fmt"
"sort"
"github.com/novox/mesh-controller/internal/inventory"
)
// rotateCommand replaces a credential and moves both ends together.
@@ -35,16 +33,12 @@ func rotateCommand(ctx context.Context, args []string) error {
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
// and rotating the other nine would be a great deal of disruption for one suspicion.
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
// One consuming module rather than every module on the machine. A machine runs many consumers
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
// issue 268) is no reason to restart every other one on the machine.
module := set.String("module", "", "only this consuming module's credential")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
return errors.New("rotate <provision> [--consumer <machine>]")
}
provision := positionals[0]
@@ -59,12 +53,6 @@ func rotateCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
holders = ofModule(holders, *module)
if len(holders) == 0 && *module != "" {
return fmt.Errorf(
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
"says what a machine holds", *module, onMachine(*only), provision)
}
if len(holders) == 0 {
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
// and a rotation somebody believes happened is worse than one they know did not.
@@ -128,27 +116,6 @@ func rotateCommand(ctx context.Context, args []string) error {
return nil
}
// ofModule is the holders whose consuming module is this one; all of them when none is named.
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
if module == "" {
return holders
}
var out []inventory.Holder
for _, h := range holders {
if h.ConsumerModule == module {
out = append(out, h)
}
}
return out
}
func onMachine(machine string) string {
if machine == "" {
return ""
}
return " on " + machine
}
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
func asLocal(local string) string {
if local == "" {
cmd/mesh-controller/rotate_test.go
-27
View File
@@ -1,27 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
// 268): a module that leaked its database password is no reason to restart every other consumer.
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
holders := []inventory.Holder{
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
}
got := ofModule(holders, "letta")
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
t.Fatalf("narrowed to letta: %+v", got)
}
if len(ofModule(holders, "")) != 3 {
t.Fatal("no module named narrowed anyway")
}
if len(ofModule(holders, "absent")) != 0 {
t.Fatal("a module holding nothing matched")
}
}
cmd/mesh-controller/same_source_test.go
-209
View File
@@ -1,209 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A build whose source is unchanged is never a move (novox/hq issue 280): an image is not
// byte-reproducible, so two builds of one source make two digests, and the rule that a rebuild with
// identical artifacts is no move (ADR 0236) never held for one.
// anImageBuild is a build outcome of an image module: its manifest names the image by the digest made.
func anImageBuild(t *testing.T, module, id, commit, digest, source string, asked time.Time) link.BuildResult {
t.Helper()
image := "registry.invalid:5000/" + module + "/server@sha256:" + digest
manifest, _ := json.Marshal(map[string]any{"module": module, "version": "1",
"resources": []any{map[string]any{"id": "svc", "type": "container", "name": module, "image": image}}})
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module,
On: "anchor", Module: module, Commit: commit, Manifest: manifest, SourceFingerprint: source,
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: image}},
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
}
// shelfNames is whether the module the mesh holds names this digest.
func shelfNames(t *testing.T, inv *inventory.Inventory, module, digest string) bool {
t.Helper()
shelf, err := inv.Catalogue(t.Context())
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(shelf[module])
return strings.Contains(string(raw), digest)
}
// A rebuild of an unchanged source is registered with the artifacts of the build it was first made
// as: no machine is sent a new digest, a module standing on it is handed the same base, and the two
// builds are one to every rule that asks whether a send moves something. A real source change moves.
func TestARebuildOfAnUnchangedSourceKeepsItsArtifacts(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
for i, b := range []link.BuildResult{
anImageBuild(t, "app", "", "c1aaaaaa", strings.Repeat("a", 64), "src1:same", start),
// Another module's merge rebuilt it: a new commit, a new image digest, the same source.
anImageBuild(t, "app", "", "c2bbbbbb", strings.Repeat("b", 64), "src1:same", start.Add(time.Minute)),
} {
if _, _, err := takeIn(ctx, inv, b); err != nil {
t.Fatalf("build %d: %v", i, err)
}
}
if !shelfNames(t, inv, "app", strings.Repeat("a", 64)) || shelfNames(t, inv, "app", strings.Repeat("b", 64)) {
t.Fatal("a rebuild of an unchanged source registered the digest it made, not the one the mesh holds")
}
if src, err := inv.SourceOf(ctx, "app"); err != nil || src.BuiltFrom != "c2bbbbbb" {
t.Fatalf("the module is not at the commit it was rebuilt from: %+v %v", src, err)
}
held, err := inv.Held(ctx)
if err != nil || !strings.HasSuffix(held["app/server"], strings.Repeat("a", 64)) {
t.Fatalf("a module standing on it is handed %q, not the build the mesh holds (%v)", held["app/server"], err)
}
f, err := readMoveFacts(ctx, inv)
if err != nil {
t.Fatal(err)
}
if !f.identical("app", "c1aaaaaa", "c2bbbbbb") {
t.Fatal("two builds of one source are not one build")
}
// A real change to the source moves: its own digest registered, and not identical.
if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c3cccccc", strings.Repeat("c", 64), "src1:changed",
start.Add(2*time.Minute))); err != nil {
t.Fatal(err)
}
if !shelfNames(t, inv, "app", strings.Repeat("c", 64)) {
t.Fatal("a changed source did not register what it made")
}
if f, _ = readMoveFacts(ctx, inv); f.identical("app", "c2bbbbbb", "c3cccccc") {
t.Fatal("a changed source is read as the same build")
}
// And a builder that says no fingerprint is told apart by its artifacts alone, as before.
if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c4dddddd", strings.Repeat("d", 64), "",
start.Add(3*time.Minute))); err != nil {
t.Fatal(err)
}
if !shelfNames(t, inv, "app", strings.Repeat("d", 64)) {
t.Fatal("a build with no fingerprint did not register what it made")
}
}
// The bus rebuilt for another module's merge, its source untouched and its image digest new: no move —
// no push to its machine is held, no bus step is demanded, and `bus upgrade` has nothing to do. A real
// change to the bus's source is the planned step again.
func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
bus := func(id, commit, digest, source string, at time.Time) link.BuildResult {
b := anImageBuild(t, "nats", id, commit, digest, source, at)
manifest, _ := json.Marshal(map[string]any{"module": "nats", "version": "2",
"provides": []any{map[string]any{"name": "mesh-bus"}},
"resources": []any{map[string]any{"id": "svc", "type": "container", "name": "nats",
"image": b.Made[0].Reference}}})
b.Manifest = manifest
return b
}
if _, _, err := takeIn(ctx, inv, bus("", "n1111111", strings.Repeat("a", 64), "src1:bus", start)); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1111111"}); err != nil {
t.Fatal(err)
}
// The wide rebuild: a new commit, a new image digest, the same source.
if _, _, err := takeIn(ctx, inv, bus("", "n2222222", strings.Repeat("b", 64), "src1:bus", start.Add(time.Minute))); err != nil {
t.Fatal(err)
}
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
t.Fatalf("a bus rebuilt from an unchanged source held its machine: %v %v", held, err)
}
wasSnapshot := takeBusSnapshot
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
takeBusSnapshot = func(context.Context, string, string) (string, error) {
return "", errors.New("no snapshot is taken for a bus step nobody needs")
}
var sent [][]string
wasSend := sendRollout
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
sent = append(sent, names)
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
if err := busCommand(ctx, []string{"upgrade", "--why", "nothing changed", "--reversible"}); err != nil || len(sent) != 0 {
t.Fatalf("a bus step ran for a bus whose source is unchanged: %v, sent %v", err, sent)
}
if !shelfNames(t, inv, "nats", strings.Repeat("a", 64)) {
t.Fatal("the bus the mesh holds is not the image its machine runs")
}
// A real change to the bus's source: the planned step.
if _, _, err := takeIn(ctx, inv, bus("", "n3333333", strings.Repeat("c", 64), "src1:bus-2.12", start.Add(2*time.Minute))); err != nil {
t.Fatal(err)
}
held, err := busHeld(ctx, inv, []string{"anchor"})
if err != nil || !strings.Contains(held["anchor"], "planned step") {
t.Fatalf("a changed bus did not demand its planned step: %v %v", held, err)
}
}
// A plan's build made from the source every machine running the module runs is not sent and not
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
for _, tc := range []struct {
name string
source string
sent [][]string
}{
{"unchanged", "src1:app", nil},
{"changed", "src1:app-changed", [][]string{{"anchor"}}},
} {
t.Run(tc.name, func(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// What anchor and laptop run (c1) was made from src1:app; the plan's build of c2 from the source
// the case says.
for _, b := range []inventory.Build{
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: tc.source,
Asked: time.Now(), At: time.Now()},
} {
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
p := g.plan(t)
p.Modules["app"].Build = "build-2s"
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
if !reflect.DeepEqual(g.sent, tc.sent) {
t.Fatalf("sent %v, want %v", g.sent, tc.sent)
}
p = g.plan(t)
s := p.Modules["app"]
if tc.sent == nil && (s.SentAt == nil || s.Gate != nil || !strings.Contains(s.Why, "no move")) {
t.Fatalf("an unchanged source was not read as no move: %+v", s)
}
if tc.sent != nil && (s.Gate == nil || len(s.First) == 0) {
t.Fatalf("a changed source was not sent to its first machine, gated: %+v", s)
}
})
}
}
cmd/mesh-controller/sayable_test.go
-287
View File
@@ -1,287 +0,0 @@
package main
import (
"context"
"fmt"
"net"
"os"
"sort"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"golang.org/x/net/dns/dnsmessage"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/outward"
)
// **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277,
// ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row
// suppressed past its bound, every probe's finding, every event's condition — is held to the operator
// channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's
// shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
// this is where the producer is made to say it rightly in the first place.
func TestMain(m *testing.M) {
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
unsaid.note(o, field, r)
}
code := m.Run()
if said := unsaid.all(); len(said) > 0 {
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+
"(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n",
len(said), strings.Join(said, "\n "))
if code == 0 {
code = 1
}
}
os.Exit(code)
}
// unsaid collects, across the suite, every finding whose words the operator's channel would withhold.
var unsaid unsayable
type unsayable struct {
mu sync.Mutex
seen map[string]bool
}
func (u *unsayable) note(o conditions.Observation, field string, r outward.Refusal) {
text := o.Summary
if field == "key" {
text = o.Key()
}
u.mu.Lock()
defer u.mu.Unlock()
if u.seen == nil {
u.seen = map[string]bool{}
}
u.seen[fmt.Sprintf("%s (source %s, kind %s): its %s carries %s — %q", o.Key(), o.Source, o.Kind, field, r.What,
text)] = true
}
func (u *unsayable) all() []string {
u.mu.Lock()
defer u.mu.Unlock()
var out []string
for s := range u.seen {
out = append(out, s)
}
sort.Strings(out)
return out
}
// linted is a producer's findings, held to the content rule as a keeper would hold them: for a test
// that reads a producer's findings without raising them.
func linted(obs []conditions.Observation) []conditions.Observation {
for _, o := range obs {
machines := append([]string{o.Machine}, o.Also...)
if r, ok := outward.Check(o.Key(), machines...); !ok {
unsaid.note(o, "key", r)
}
if r, ok := outward.Check(o.Summary, machines...); !ok {
unsaid.note(o, "summary", r)
}
}
return obs
}
// resolverStandIn is a resolver on loopback that answers as answer says; nil answers nothing.
func resolverStandIn(t *testing.T, answer func(q dnsmessage.Question, n int64) *dnsmessage.Message) (port string, asked *atomic.Int64) {
t.Helper()
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = conn.Close() })
asked = &atomic.Int64{}
go func() {
buf := make([]byte, 1500)
for {
n, from, err := conn.ReadFrom(buf)
if err != nil {
return
}
var q dnsmessage.Message
if q.Unpack(buf[:n]) != nil || len(q.Questions) == 0 {
continue
}
reply := answer(q.Questions[0], asked.Add(1))
if reply == nil {
continue
}
reply.Header.ID, reply.Header.Response, reply.Questions = q.ID, true, q.Questions
packed, _ := reply.Pack()
_, _ = conn.WriteTo(packed, from)
}
}()
_, port, _ = net.SplitHostPort(conn.LocalAddr().String())
return port, asked
}
// answersRightly answers A with the address the test's machine has, and NODATA for AAAA.
func answersRightly(q dnsmessage.Question) *dnsmessage.Message {
reply := &dnsmessage.Message{}
if q.Type == dnsmessage.TypeA {
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Name, Type: dnsmessage.TypeA,
Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
}
return reply
}
// quickResolvers makes D2's patience short for a test.
func quickResolvers(t *testing.T, port string) {
t.Helper()
before, within, pause := resolverPort, resolverWithin, resolverPause
resolverPort, resolverWithin, resolverPause = port, 150*time.Millisecond, 10*time.Millisecond
t.Cleanup(func() { resolverPort, resolverWithin, resolverPause = before, within, pause })
}
var anchorOnTheNetwork = []inventory.Overlay{{Name: "anchor", Address: "10.77.0.1"}}
// **D2 asks again before it says anything** (novox/hq issue 277): a resolver that misses a question on
// a loaded machine and answers the next try is a resolver that answers.
func TestAResolverThatMissesOneTryAndAnswersTheNextIsWell(t *testing.T) {
port, asked := resolverStandIn(t, func(q dnsmessage.Question, n int64) *dnsmessage.Message {
if n <= 2 {
return nil // the first question of each kind lost, as under a push and a build starting
}
return answersRightly(q)
})
quickResolvers(t, port)
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
if len(got) != 0 {
t.Fatalf("a resolver that answered its second try was said: %+v", got)
}
if asked.Load() < 3 {
t.Fatalf("asked %d times", asked.Load())
}
}
// **A resolver that answers nothing is held for the next run, and said in machine names**: the address
// it was asked at and the socket's words are the evidence, never the summary the operator reads.
func TestAResolverThatAnswersNothingIsHeldAndSaidInMachineNames(t *testing.T) {
port, asked := resolverStandIn(t, func(dnsmessage.Question, int64) *dnsmessage.Message { return nil })
quickResolvers(t, port)
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
if len(got) != 1 {
t.Fatalf("%+v", got)
}
o := got[0]
if !o.Confirm || o.Key() != "seat.mesh-dns-resolver.anchor.wrong" || o.Severity != conditions.Urgent {
t.Fatalf("not held for a second look, or not the resolver's condition: %+v", o)
}
if r, ok := outward.Check(o.Summary, "anchor"); !ok {
t.Fatalf("the summary carries %s: %q", r, o.Summary)
}
if !strings.Contains(o.Summary, "anchor's") || !strings.Contains(o.Said, "127.0.0.1") ||
!strings.Contains(o.Said, "anchor.internal") {
t.Fatalf("summary %q, evidence %q", o.Summary, o.Said)
}
if want := int64(2 * resolverTries); asked.Load() != want {
t.Fatalf("asked %d times, want %d: each question %d times", asked.Load(), want, resolverTries)
}
}
// **A resolver that answers wrongly is said at once**, in machine names: an answer is not the absence
// of one (issue 262's NXDOMAIN for IPv6).
func TestAResolverAnsweringWronglyIsSaidAtOnceInMachineNames(t *testing.T) {
port, _ := resolverStandIn(t, func(q dnsmessage.Question, _ int64) *dnsmessage.Message {
if q.Type == dnsmessage.TypeAAAA {
return &dnsmessage.Message{Header: dnsmessage.Header{RCode: dnsmessage.RCodeNameError}}
}
return answersRightly(q)
})
quickResolvers(t, port)
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
if len(got) != 1 || got[0].Confirm {
t.Fatalf("%+v", got)
}
if r, ok := outward.Check(got[0].Summary, "anchor"); !ok || !strings.Contains(got[0].Summary, "NXDOMAIN") ||
!strings.Contains(got[0].Summary, "anchor's IPv6 address") {
t.Fatalf("summary %q (%v)", got[0].Summary, r)
}
}
// **The self-check raises a finding one look can be wrong about on the second run in a row**, keeps it
// open while it is seen, and clears it when it is not — and a probe that cannot run is said as a
// condition only when the next run cannot run it either. Neither is ever a pass in the verdict.
func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) {
var unanswered, broken atomic.Bool
held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong",
Machine: "anchor", Severity: conditions.Urgent, Confirm: true,
Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"}
withProbes(t,
probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
if unanswered.Load() {
return []conditions.Observation{held}, nil
}
return nil, nil
}},
probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
if broken.Load() {
return nil, fmt.Errorf("a question timed out")
}
return nil, nil
}},
)
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
d := &doctor{keeper: k, host: "anchor"}
keys := func() []string {
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
var out []string
for _, c := range open {
out = append(out, c.Key)
}
sort.Strings(out)
return out
}
unanswered.Store(true)
broken.Store(true)
run := d.runOnce(t.Context(), "a test")
if open := keys(); len(open) != 0 {
t.Fatalf("one look raised %v", open)
}
if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun {
t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes)
}
// Seen twice in a row: raised, and so is the probe that could not run twice.
run = d.runOnce(t.Context(), "a test")
if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" {
t.Fatalf("two looks in a row left open %v", open)
}
if run.Probes[0].Verdict != verdictFail {
t.Fatalf("%+v", run.Probes[0])
}
// Open, and seen again: kept, never cleared and raised again.
d.runOnce(t.Context(), "a test")
if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 {
t.Fatalf("the open condition was not kept as it was: %+v", c)
}
// Answered again: cleared. Then one look alone raises nothing again.
unanswered.Store(false)
broken.Store(false)
d.runOnce(t.Context(), "a test")
if open := keys(); len(open) != 0 {
t.Fatalf("a passing run left open %v", open)
}
unanswered.Store(true)
d.runOnce(t.Context(), "a test")
if open := keys(); len(open) != 0 {
t.Fatalf("one look after a pass raised %v", open)
}
}
cmd/mesh-controller/seat_dependencies_test.go
-149
View File
@@ -1,149 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// Defends novox/hq ADR 0207 at the controller's acts: `assign` refuses a module whose resources a
// seat nothing on the node holds applies, `unassign` refuses taking the last holder from under its
// dependents, and `status` reports what composition does not yet refuse.
func serviceManagerHolder() catalogue.Manifest {
return catalogue.Manifest{Module: "systemd", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}},
Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
}
func packageManagerHolder() catalogue.Manifest {
return catalogue.Manifest{Module: "pacman", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.PackageManagerSeat, Scope: catalogue.ScopeNode}},
Resources: []map[string]any{{"id": "refresh", "type": "service", "unit": "pacman-refresh.timer"}}}
}
func aDaemon() catalogue.Manifest {
return catalogue.Manifest{Module: "sshd", Version: "1",
Resources: []map[string]any{{"id": "sshd", "type": "service", "unit": "sshd.service"}}}
}
func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, serviceManagerHolder())
register(t, open, packageManagerHolder())
register(t, open, aDaemon())
_, err := assign(ctx, open, "laptop", "sshd")
if err == nil {
t.Fatal("sshd went onto a machine nothing holds the service manager of")
}
for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not say %q:\n%v", want, err)
}
}
assigned, err := open.inventory.Assigned(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if contains(assigned, "sshd") {
t.Fatalf("a refused assignment was kept: %v", assigned)
}
// The holders depend on each other, so neither goes on alone — and both go on in one act.
if _, err := assign(ctx, open, "laptop", "systemd"); err == nil {
t.Fatal("systemd went on alone though its package needs a package manager")
}
if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil {
t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said)
}
if said, err := assign(ctx, open, "laptop", "sshd"); err != nil {
t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said)
}
}
func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) {
argv, err := argvFor("assign", map[string]any{"node": "laptop", "module": "systemd, pacman"})
if err != nil {
t.Fatal(err)
}
if strings.Join(argv, " ") != "assign laptop systemd pacman" {
t.Errorf("the seat's assign became %v", argv)
}
}
func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, serviceManagerHolder())
register(t, open, packageManagerHolder())
register(t, open, aDaemon())
if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil {
t.Fatal(err)
}
_, err := unassign(ctx, open, "laptop", "systemd")
if err == nil {
t.Fatal("the service manager came off a machine still running services")
}
for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not name %q:\n%v", want, err)
}
}
assigned, _ := open.inventory.Assigned(ctx, "laptop")
if !contains(assigned, "systemd") {
t.Fatalf("a refused unassignment took the module off anyway: %v", assigned)
}
if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil {
t.Fatalf("a dependent could not come off: %v", err)
}
}
func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) {
// The mesh as it ran before the switch (novox/hq ADR 0207 §4).
defer catalogue.EnforcingSeatDependencies(false)()
open := aMesh(t)
ctx := t.Context()
register(t, open, serviceManagerHolder())
register(t, open, aDaemon())
// Assigned straight into the store: a machine whose modules predate the rule, which is every
// machine on the day it ships.
if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil {
t.Fatal(err)
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if _, refused := asked.refused["laptop"]; refused {
t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"])
}
if asked.well() {
t.Error("a mesh with an unheld dependency reads as all well")
}
got := printed(t, func() error { return printStatus(asked) })
for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} {
if !strings.Contains(got, want) {
t.Errorf("status does not say %q:\n%s", want, got)
}
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
Unheld []catalogue.Unheld `json:"unheld"`
}
if err := json.Unmarshal(body, &doc); err != nil {
t.Fatal(err)
}
if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat {
t.Errorf("the document's unheld is %+v", doc.Unheld)
}
}
cmd/mesh-controller/seats.go
+9 -35
View File
@@ -29,13 +29,11 @@ type seatHolder struct {
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
type seatRow struct {
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
Replicated bool `json:"replicated,omitempty"`
Holders []seatHolder `json:"holders"`
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
Holders []seatHolder `json:"holders"`
}
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -49,7 +47,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
rows := make([]seatRow, 0, len(seats))
for _, s := range seats {
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
Replicated: s.Replicated, Holders: []seatHolder{}}
Holders: []seatHolder{}}
seen := map[seatHolder]bool{}
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -106,13 +104,9 @@ func seatCommand(ctx context.Context, args []string) error {
return nil
}
if len(args) == 3 && args[1] == "--to" {
return handOver(ctx, args[0], args[2], false)
return handOver(ctx, args[0], args[2])
}
if len(args) == 3 && args[1] == "--add" {
return handOver(ctx, args[0], args[2], true)
}
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
"seat <name> --add <node>/<module>")
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
}
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -125,12 +119,7 @@ func seatCommand(ctx context.Context, args []string) error {
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
// and refusing to record a handover to a module the node has not started would make the handover
// impossible to do before the switch instead of as the switch.
//
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
// records the named assignment as a further holder and leaves every holder on record as it is. A
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
// the one named, as it always did.
func handOver(ctx context.Context, seatName, to string, adding bool) error {
func handOver(ctx context.Context, seatName, to string) error {
nodeName, module, ok := strings.Cut(to, "/")
if !ok || nodeName == "" || module == "" {
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -146,10 +135,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
if !known {
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
}
if adding && !seat.Replicated {
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
}
assigned, err := inv.Assigned(ctx, nodeName)
if err != nil {
return err
@@ -172,7 +157,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
}
var was string
var held []string
holdings, err := inv.Holdings(ctx)
if err != nil {
return err
@@ -180,7 +164,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
for _, h := range holdings {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
was = h.Node
held = append(held, h.Node)
}
}
@@ -204,15 +187,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
} else if err := catalogue.CanHold(*m, seat); err != nil {
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
}
if adding {
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
strings.Join(held, ", "))
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
return nil
}
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
cmd/mesh-controller/seatverbs.go
+33 -681
View File
@@ -9,11 +9,9 @@ import (
"github.com/nats-io/nats.go/micro"
"os"
"os/exec"
"slices"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
@@ -38,194 +36,19 @@ type verbAnswer struct {
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
//
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
// the verb declares but did not use for the command line it composed — given beside another that
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
// not take that" would have stopped it before anything was sent.
func argvFor(verb string, args map[string]any) ([]string, error) {
a, err := readArguments(verb, args)
if err != nil {
return nil, err
str := func(key string) string {
v, _ := args[key].(string)
return strings.TrimSpace(v)
}
argv, err := a.commandLine()
if len(a.misread) > 0 {
// The table and the command line disagree: the verb reads an argument no caller can see
// in its schema, so no caller could ever pass it.
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
"table and its command lines disagree", verb, quoteAll(a.misread))
}
if err != nil {
return nil, err
}
if unused := a.unused(); len(unused) > 0 {
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
}
return argv, nil
}
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
// command line was composed from.
type verbArguments struct {
verb string
given map[string]string
used map[string]bool
declared map[string]bool
misread []string // arguments the command line read that the schema does not declare: a bug here
}
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
// wrote.
func controllerVerb(name string) (catalogue.Verb, bool) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == name {
return v, true
}
}
return catalogue.Verb{}, false
}
// declaredArguments are a schema's properties, and which of them are switches.
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
switches = map[string]bool{}
props, _ := v.Input["properties"].(map[string]any)
for name, p := range props {
names = append(names, name)
desc, _ := p.(map[string]any)
switch enum := desc["enum"].(type) {
case []string:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
case []any:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
}
}
sort.Strings(names)
return names, switches
}
// readArguments refuses what the verb does not take, before anything is composed.
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
v, known := controllerVerb(verb)
if !known {
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
names, switches := declaredArguments(v)
declared := map[string]bool{}
for _, n := range names {
declared[n] = true
}
takes := "none"
if len(names) > 0 {
takes = quoteAll(names)
}
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
keys := make([]string, 0, len(args))
for k := range args {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if !declared[k] {
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
}
var value string
switch x := args[k].(type) {
case nil:
continue
case string:
value = strings.TrimSpace(x)
case bool:
if !switches[k] {
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
}
value = fmt.Sprint(x)
default:
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
}
if switches[k] {
switch value {
case "true":
case "false", "":
continue // said and off: the same as not given, and nothing passed over
default:
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
need := func(keys ...string) error {
for _, k := range keys {
if str(k) == "" {
return fmt.Errorf("%s needs %q", verb, k)
}
}
if value != "" {
a.given[k] = value
}
return nil
}
return a, nil
}
// str is one argument's value, marked as used.
func (a *verbArguments) str(key string) string {
if !a.declared[key] {
a.misread = append(a.misread, key)
}
a.used[key] = true
return a.given[key]
}
// on is a switch, marked as used.
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
// need refuses a call missing a required argument, in the verb's own words.
func (a *verbArguments) need(keys ...string) error {
for _, k := range keys {
if a.str(k) == "" {
return fmt.Errorf("%s needs %q", a.verb, k)
}
}
return nil
}
// unused are the arguments given that the command line was not composed from.
func (a *verbArguments) unused() []string {
var out []string
for k := range a.given {
if !a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
return out
}
func (a *verbArguments) usedGiven() []string {
var out []string
for k := range a.given {
if a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
if len(out) == 0 {
return []string{"nothing"}
}
return out
}
func quoteAll(xs []string) string {
q := make([]string, len(xs))
for i, x := range xs {
if x == "nothing" {
q[i] = x
continue
}
q[i] = fmt.Sprintf("%q", x)
}
return strings.Join(q, ", ")
}
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
func (a *verbArguments) commandLine() ([]string, error) {
verb, str, on, need := a.verb, a.str, a.on, a.need
switch verb {
case "command":
// The generic verb: the command line as given, split as a shell would split it, with
@@ -241,15 +64,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
if len(argv) == 0 {
return nil, errors.New("command names no command")
}
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
// it says why, as it would through its own verb.
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
"line (recorded in the hand-act log). Nothing was done", repair)
}
return argv, nil
case "tools":
return nil, errors.New("tools is answered from the records, not by a command")
case "status":
return []string{"status", "--json"}, nil
case "nodes":
@@ -267,14 +82,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
if id := str("log"); id != "" {
return []string{"builds", "--log", id}, nil
}
argv := []string{"builds"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
if m := str("module"); m != "" {
argv = append(argv, m)
return []string{"builds", m}, nil
}
return argv, nil
return []string{"builds"}, nil
case "plans":
if r := str("repository"); r != "" {
argv := []string{"plans", "--what-if", r}
@@ -284,137 +95,25 @@ func (a *verbArguments) commandLine() ([]string, error) {
if m := str("modules"); m != "" {
argv = append(argv, "--modules", m)
}
if d := str("module-dirs"); d != "" {
argv = append(argv, "--module-dirs", d)
}
return argv, nil
}
for _, act := range []string{"stop", "close", "retry", "go"} {
if id := str(act); id != "" {
argv := []string{"plans", act, id}
if act == "retry" {
return argv, nil
}
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
if id := str("stop"); id != "" {
return []string{"plans", "stop", id}, nil
}
if id := str("id"); id != "" {
return []string{"plans", id}, nil
}
argv := []string{"plans"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
return argv, nil
// The delivery's owner's verbs (novox/hq ADR 0239).
case "delivery-plan":
if err := need("repository", "paths"); err != nil {
return nil, err
}
argv := []string{"delivery", "plan", "--repository", str("repository"), "--paths", str("paths")}
for _, flag := range []string{"head", "base", "module-dirs", "removed"} {
if v := str(flag); v != "" {
argv = append(argv, "--"+flag, v)
}
}
return argv, nil
case "delivery-order":
if err := need("members"); err != nil {
return nil, err
}
return []string{"delivery", "order", "--members", str("members")}, nil
case "delivery-check":
if err := need("members"); err != nil {
return nil, err
}
if g := str("group"); g != "" {
return []string{"delivery", "check", "--group", g, "--members", str("members")}, nil
}
return []string{"delivery", "check", "--members", str("members")}, nil
case "deliver":
if err := need("plan"); err != nil {
return nil, err
}
argv := []string{"delivery", "go", str("plan"), "--by", catalogue.DeliverySeat}
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
}
return argv, nil
case "delivery-stop":
if err := need("plan", "why"); err != nil {
return nil, err
}
argv := []string{"delivery", "stop", str("plan"), "--why", str("why")}
if b := str("by"); b != "" {
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
}
return argv, nil
case "delivery-walks":
argv := []string{"delivery", "walks"}
if id := str("plan"); id != "" {
return append(argv, "--plan", id), nil
}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
return argv, nil
// The build queue (novox/hq ADR 0219).
case "queue":
return []string{"queue"}, nil
case "cancel", "kill":
if err := need("id"); err != nil {
return nil, err
}
return []string{verb, str("id")}, nil
case "clear":
if on("dead") {
return []string{"clear", "--dead"}, nil
}
return []string{"clear"}, nil
case "rebuild":
if err := need("what"); err != nil {
return nil, err
}
return []string{"rebuild", str("what")}, nil
case "replay":
if err := need("id"); err != nil {
return nil, err
}
argv := []string{"replay", str("id")}
if on("register") {
argv = append(argv, "--register")
}
if on("older") {
argv = append(argv, "--older")
}
return argv, nil
case "pause", "resume":
if n := str("node"); n != "" {
return []string{verb, n}, nil
}
return []string{verb}, nil
return []string{"plans"}, nil
case "plan":
if err := need("node"); err != nil {
return nil, err
}
if on("files") {
return []string{"plan", str("node"), "--files"}, nil
}
return []string{"plan", str("node"), "--json"}, nil
case "assign", "unassign":
if err := need("node", "module"); err != nil {
return nil, err
}
// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
// the seats that apply resources depend on each other and go on together.
return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
return []string{verb, str("node"), str("module")}, nil
case "pin":
if err := need("node", "provision", "from", "module"); err != nil {
return nil, err
@@ -429,232 +128,23 @@ func (a *verbArguments) commandLine() ([]string, error) {
// Sent and not waited for: the asker reads `status` for what the machine did, which is
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
// time out on every machine that takes a minute, and say nothing about the ones that did not.
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
if err := need("why"); err != nil {
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
}
why := []string{"--why", str("why")}
if c := str("cause"); c != "" {
why = append(why, "--cause", c)
}
if n := str("node"); n != "" {
// behind is not read here: given with a machine, it is refused as passed over — naming
// a machine and asking for every machine behind are two requests, and guessing one
// would push a machine nobody named, or not push one somebody did.
return append([]string{"push", n, "--wait", "0"}, why...), nil
return []string{"push", n, "--wait", "0"}, nil
}
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
// first, so a caller who meant one machine reads that it was not one.
on("behind")
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
case "hand-act":
if err := need("what", "why", "cause"); err != nil {
return nil, err
}
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
if c := str("condition"); c != "" {
argv = append(argv, "--condition", c)
}
return argv, nil
case "hand-acts":
argv := []string{"hand-acts", "--json"}
if d := str("days"); d != "" {
argv = append(argv, "--days", d)
}
return argv, nil
case "healers":
argv := []string{"healers", "--json"}
if d := str("days"); d != "" {
argv = append(argv, "--days", d)
}
return argv, nil
case "durations":
argv := []string{"durations", "--json"}
if k := str("kind"); k != "" {
argv = append(argv, "--kind", k)
}
if d := str("days"); d != "" {
argv = append(argv, "--days", d)
}
return argv, nil
case "conditions":
// One verb, four shapes, as `plans` (novox/hq to-be 45 §2): a silence, one condition, the
// history, or the open ones filtered.
if key := str("silence"); key != "" {
if err := need("for", "why"); err != nil {
return nil, err
}
argv := []string{"conditions", "silence", key, "--for", str("for"), "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
if on("history") {
argv := []string{"conditions", "history", "--json"}
if d := str("days"); d != "" {
argv = append(argv, "--days", d)
}
if k := str("key"); k != "" {
argv = append(argv, "--key", k)
}
return argv, nil
}
if k := str("key"); k != "" {
return []string{"conditions", "show", k, "--json"}, nil
}
argv := []string{"conditions", "--json"}
for _, filter := range []string{"scope", "severity", "machine"} {
if v := str(filter); v != "" {
argv = append(argv, "--"+filter, v)
}
}
return argv, nil
case "retire":
answer := str("answer")
if answer == "" {
return []string{"retire", "--json"}, nil
}
if answer != "approve" && answer != "reject" {
return nil, fmt.Errorf("retire answers approve or reject, not %q", answer)
}
if err := need("node", "module", "why"); err != nil {
return nil, err
}
argv := []string{"retire", answer, str("node"), str("module"), "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
case "cleanup":
consumer, older := str("consumer"), str("older-than")
switch {
case consumer != "" && older != "":
return nil, errors.New("cleanup deletes one consumer or those older than some days, not both")
case consumer != "":
if err := need("node", "module", "why"); err != nil {
return nil, err
}
argv := []string{"cleanup", "delete", str("node"), str("module"), consumer, "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
case older != "":
if err := need("why"); err != nil {
return nil, err
}
argv := []string{"cleanup", "delete", "--older-than", older, "--why", str("why")}
if on("confirm") {
argv = append(argv, "--confirm")
}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
return []string{"cleanup", "list", "--json"}, nil
case "data":
argv := []string{"data", "--json"}
if m := str("machine"); m != "" {
argv = append(argv, "--machine", m)
}
if on("retired") {
argv = append(argv, "--retired")
}
return argv, nil
case "upgrade":
if on("backlog") {
return []string{"upgrade", "backlog"}, nil
}
if on("release-backlog") {
argv := []string{"upgrade", "release-backlog", "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
m, policy := str("module"), str("policy")
if m == "" {
if policy != "" {
return nil, errors.New("upgrade takes a policy only for a module")
}
return []string{"upgrade"}, nil
}
argv := []string{"upgrade", m}
if policy != "" {
argv = append(argv, policy)
if on("together") {
argv = append(argv, "--together")
}
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
}
}
return argv, nil
case "bus":
if !on("upgrade") {
return []string{"bus"}, nil
}
argv := []string{"bus", "upgrade", "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
if on("reversible") {
argv = append(argv, "--reversible")
}
if on("irreversible") {
argv = append(argv, "--irreversible")
}
if w := str("snapshot-taken"); w != "" {
argv = append(argv, "--snapshot-taken", w)
}
return argv, nil
case "doctor":
which := 0
argv := []string{"doctor"}
for _, sub := range []string{"run", "probes", "signals"} {
if on(sub) {
which++
argv = append(argv, sub)
}
}
if which > 1 {
return nil, errors.New("doctor answers one of run, probes or signals at a time")
}
return append(argv, "--json"), nil
return []string{"push", "--behind", "--wait", "0"}, nil
case "rotate":
if p := str("provision"); p != "" {
argv := []string{"rotate", p}
if c := str("consumer"); c != "" {
argv = append(argv, "--consumer", c)
}
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
// and secret stay the other shape's, and are refused as passed over.
if m := str("module"); m != "" {
argv = append(argv, "--module", m)
}
return argv, nil
}
_, node := a.given["node"]
_, module := a.given["module"]
_, secret := a.given["secret"]
if node || module || secret {
if err := need("node", "module", "secret"); err != nil {
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
}
argv := []string{"secret", "rotate", str("node"), str("module"), str("secret")}
// Why, recorded in the hand-act log (novox/hq ADR 0228); a cause only beside a why.
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
}
return argv, nil
if str("node") != "" && str("module") != "" && str("secret") != "" {
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
}
// Neither shape: the command says its usage, which names both, and that is the answer the
// caller needs.
// Half of either shape: the command says its usage, which names both shapes, and that is
// the answer the caller needs.
return []string{"rotate"}, nil
case "settings":
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
@@ -662,16 +152,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err := need("module"); err != nil {
return nil, err
}
var argv []string
if on("clear") {
argv := []string{"settings", "set", str("module")}
switch {
case str("clear") == "true":
argv = []string{"settings", "clear", str("module")}
} else {
argv = []string{"settings", "set", str("module")}
// Neither values nor clear: the command says its usage, which names both.
if v := str("values"); v != "" {
argv = append(argv, v)
}
case str("values") != "":
argv = append(argv, str("values"))
}
// Neither values nor clear: the command says its usage, which names both, and that is the
// answer the caller needs — the same as `rotate` given half of either shape.
if n := str("node"); n != "" {
argv = append(argv, "--node", n)
}
@@ -703,41 +192,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
}
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
verb, catalogue.ControllerSeatName)
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true,
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
"delivery": true}
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
func repairingCommand(argv []string) string {
switch {
case argv[0] == "push":
return "push"
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close" || argv[1] == "go"):
return "plans " + argv[1]
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
return "broker consumer-reset"
case argv[0] == "hand-act":
return "hand-act record"
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
return "conditions silence"
case argv[0] == "retire" && len(argv) > 1 && (argv[1] == "approve" || argv[1] == "reject"):
return "retire " + argv[1]
case argv[0] == "cleanup" && len(argv) > 1 && argv[1] == "delete":
return "cleanup delete"
}
return ""
}
func isWhyFlag(word string) bool {
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
}
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true}
// runVerb runs this binary with the given command line and gathers what it said.
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
@@ -749,12 +208,6 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
// The same environment: the stores' credentials, the bus, the broker — everything a command run
// from a shell in this container would have, because it is that.
cmd.Env = os.Environ()
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
caller := link.CallerIn(ctx)
if caller == "" {
caller = "a seat call whose caller the bus did not name"
}
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
// prints its warnings beside the document, and a JSON parsed from the two together parsed
@@ -791,35 +244,8 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
handlers := map[string]link.ToolHandler{}
for _, v := range seat.Serves {
verb := v.Name
if inProcess[verb] {
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
args := map[string]any{}
if len(bytes.TrimSpace(raw)) > 0 {
if err := json.Unmarshal(raw, &args); err != nil {
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
}
}
// Refused like any verb's: what a verb does not take is not ignored.
a, err := readArguments(verb, args)
if err != nil {
return nil, err
}
if verb == "calls" {
return callsAnswer(link.Calls, a.given["call"])
}
if verb == "doctor" {
// From the serving controller, which runs the self-check and hears the signals
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
argv, err := a.commandLine()
if err != nil {
return nil, err
}
sub := ""
if len(argv) > 2 {
sub = argv[1]
}
return doctorAnswer(ctx, sub)
}
if verb == "tools" {
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
return seatTools(), nil
}
continue
@@ -858,85 +284,12 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
if err != nil {
return nil, err
}
if verb == "status" && statusFrom != nil {
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
return statusFrom.answer(ctx)
}
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
// Whatever it did, `status` is composed again once it has.
defer statusFrom.nudge()
}
if answersFirst(argv) {
// Before anything is sent: a push sends the bus's own machine first, and a broker
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
link.Acknowledge(ctx)
}
return runVerb(ctx, argv)
}
}
return handlers, behind, nil
}
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
func actsOnAPlan(args map[string]any) bool {
for _, act := range []string{"stop", "close", "retry"} {
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
return true
}
}
return false
}
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
// the records, `calls` from what this process served.
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
// through `command`. A push sends the machine holding the bus first when its user list changed, the
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
func answersFirst(argv []string) bool {
return len(argv) > 0 && argv[0] == "push"
}
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
// the reason said beside it.
func callsAnswer(log *link.CallLog, id string) (any, error) {
if id != "" {
c, ok, err := log.Get(id)
if err != nil {
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
"bus could not be read: %w", id, err)
}
if !ok {
if log.IsDurable() {
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
}
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
}
return c, nil
}
recent, err := log.Recent()
for i := range recent {
recent[i].Answer = nil
}
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
if log.IsDurable() {
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
broker.KeptCallsDurably, broker.CallsKeptFor)
} else {
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
}
if err != nil {
answer["unread"] = err.Error()
}
return answer, nil
}
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
func seatTools() map[string]any {
@@ -957,10 +310,9 @@ func seatTools() map[string]any {
}
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
// more, since an argument a verb does not declare is refused.
// verb runnable rather than that it happens to want the arguments the check guessed.
func sampleArguments(v catalogue.Verb) map[string]any {
sample := map[string]any{}
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
switch required := v.Input["required"].(type) {
case []string:
for _, k := range required {

Some files were not shown because too many files have changed in this diff Show More