novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Compare commits

merge into: novox/mesh-controller:fix/gate-baseline-composes
Branches Tags
novox/mesh-controller:main novox/mesh-controller:fix/gate-baseline-composes novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module novox/mesh-controller:fix/the-runtime-file-is-the-runtimes novox/mesh-controller:feat/no-change-takes-effect-unseen novox/mesh-controller:feat/build-edges-are-recorded novox/mesh-controller:fix/several-contributions-collide-in-grants-v2
..
pull from: novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module
Branches Tags
novox/mesh-controller:main novox/mesh-controller:fix/gate-baseline-composes novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module novox/mesh-controller:fix/the-runtime-file-is-the-runtimes novox/mesh-controller:feat/no-change-takes-effect-unseen novox/mesh-controller:feat/build-edges-are-recorded novox/mesh-controller:fix/several-contributions-collide-in-grants-v2
1 Commits
fix/gate-baseline-composes .. fix/the-registry-trust-test-reads-the-runtimes-module
Author SHA1 Message Date
jochen 92442d377c The registry-trust test reads the runtime's module, which now writes the trust (ADR 0222) 2026-10-05 22:46:57 +02:00
1370 changed files with 1187 additions and 659440 deletions
Expand all files Collapse all files
Dockerfile
+3 -5
View File
@@ -21,15 +21,13 @@ ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7
FROM ${GO_BASE} AS build
WORKDIR /src
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
# the image builds from this repository alone — the host's validator among them, whose module no
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
# Dependencies first, so a change to the source does not refetch them.
COPY go.mod go.sum ./
COPY vendor/ vendor/
RUN go mod download
COPY . .
ARG VERSION=development
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
RUN CGO_ENABLED=0 go build -trimpath \
-ldflags "-s -w -X main.version=${VERSION}" \
-o /mesh-controller ./cmd/mesh-controller
Makefile
+9 -9
View File
@@ -102,18 +102,18 @@ proxy-image:
# The whole gate. Raises a database, runs everything against it, and takes it down again --
# including when the tests fail, which is why the teardown is not conditional.
#
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
# It was one package at a time against one shared bus, because the live tests assert, read and remove
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
# was reading, as "no response from stream". That reads as a bug in the code under test.
check: fmt vet postgres
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
# the gate.
# Without a database the live tests skip rather than fail, so this is the honest subset and not
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
test:
go test -timeout 15m ./...
go test -p 1 ./...
vet:
go vet ./...
cmd/mesh-builder/main.go
+1 -44
View File
@@ -237,21 +237,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
npmrc, err := packagesFrom()
var built builder.Result
if request.Check != nil {
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
// published or registered; the verdict is the outcome.
result.Checked = request.Check
registry := ""
if r, ok := publisher.(builder.Registry); ok {
registry = r.Address
}
var v builder.CheckVerdict
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
if err == nil {
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
}
} else if err == nil {
if err == nil {
// The package-registry credential is a build input, so it is resolved before the clone: a
// build that could not have resolved its dependencies is refused in front of the reason, not
// after a clone that then fails at npm ci.
@@ -277,8 +263,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
// builder that is not running, and those want completely different responses.
result.Failed = err.Error()
say("failed", err.Error())
} else if request.Check != nil {
say("checked", result.Check.Verdict+": "+result.Check.Summary)
} else {
manifest, marshalErr := json.Marshal(built.Manifest)
if marshalErr != nil {
@@ -292,8 +276,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
})
}
result.Against = built.Against
result.SourceFingerprint = built.Source
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
for _, r := range built.Read {
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
}
@@ -330,31 +312,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
}
}
// checkSpecOf is a check request as the builder runs it.
func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
c := request.Check
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
for dir, b := range c.Beside {
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
}
for _, m := range c.Members {
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
Ref: m.Ref, Paths: m.Paths})
}
return spec
}
// layerOf is one layer of a check as the outcome carries it.
func layerOf(l *builder.Layer) *link.CheckLayer {
if l == nil {
return nil
}
return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules}
}
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
// (novox/hq ADR 0076, issue 053).
//
cmd/mesh-builder/once.go
-2
View File
@@ -105,7 +105,6 @@ func buildOnce(ctx context.Context, args []string) error {
Ref: *ref,
Manifest: built.Manifest,
Against: built.Against,
Source: built.Source,
}
for _, r := range built.Read {
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
@@ -136,7 +135,6 @@ type onceResult struct {
Made []madeArtifact `json:"made"`
Against []string `json:"against,omitempty"`
Read []readRepository `json:"read,omitempty"`
Source string `json:"source-fingerprint,omitempty"`
}
// readRepository is a repository this build read source from besides the module's own.
cmd/mesh-controller/acting.go
-353
View File
@@ -1,353 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
//
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
// not happen. Three ways a process stands to the lease:
//
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
// objects, which are the controller's to write — waiting while another holds it, and renews it.
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
// its service manager restarts it as a candidate (serve, in push.go).
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
// between makes it stale and refused like any other. **When nobody holds the lease, the command
// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
// it, as it would for another controller.
// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
// refused nothing: there is nothing to order against, and nothing it does reaches a machine.
//
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
// instead would be a controller that can never send the user list that lets it act.
// actor is this process's standing to the lease.
type actor struct {
mu sync.Mutex
// held is the lease this process holds: the serving controller's, or a command's own.
held *lease.Lease
// unleased is why a serving controller acts without the lease; empty while it holds it or is not
// serving.
unleased string
// serving is a serving controller, which never borrows another's epoch.
serving bool
// kv is the lease bucket, for a command to read the holder's epoch from.
kv jetstream.KeyValue
close func()
// noBus is a process with no bus configured.
noBus bool
// reset is when the lease bucket was found raised again from nothing and its revisions moved past
// the highest epoch issued, and what was said of it; zero when it was not (S12).
reset time.Time
resetSaid string
}
// theLease is this process's standing to the lease.
var theLease = &actor{}
// instance names this process among controller instances: its machine, its process and when it
// started. The lease's holder and every call this process keeps carry it.
var instance = func() string {
host, _ := os.Hostname()
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
}()
// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
func (a *actor) epoch(ctx context.Context) (uint64, error) {
a.mu.Lock()
held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
a.mu.Unlock()
switch {
case held != nil:
return held.Epoch()
case serving && unleased != "":
return 0, nil
case serving:
return 0, lease.ErrNotHeld
case noBus:
return 0, nil
}
return a.forACommand(ctx)
}
// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
a.mu.Lock()
defer a.mu.Unlock()
if a.held != nil {
return a.held.Epoch()
}
if a.kv == nil {
address, err := broker.BusAddress()
if err != nil {
// No bus: this process reaches no machine, and has nothing to order against.
a.noBus = true
return 0, nil
}
js, err := broker.Dial(address)
if err != nil {
return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
"read and nothing is done: %w", err)
}
api, err := jetstream.New(js.Conn())
if err != nil {
js.Close()
return 0, err
}
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := broker.EnsureLeaseBucket(reading, api); err != nil {
js.Close()
return 0, err
}
kv, err := api.KeyValue(reading, broker.LeaseBucket)
if err != nil {
js.Close()
return 0, err
}
a.kv, a.close = kv, js.Close
if _, found, err := lease.Current(reading, kv); err == nil && !found {
// Nobody: this command takes it for as long as it runs.
l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
if err != nil {
return 0, err
}
epoch, err := l.TryTake(reading)
if err != nil {
return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
}
keeping, stop := context.WithCancel(context.Background())
go l.Keep(keeping)
a.held = l
closeBus := a.close
a.close = func() {
stop()
l.Release(context.Background())
closeBus()
}
return epoch, nil
}
}
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
holder, found, err := lease.Current(reading, a.kv)
if err != nil {
return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
}
if !found {
return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
"more is done under an epoch nobody holds — run the command again")
}
return holder.Epoch, nil
}
// release gives back what this process holds, at its end.
func (a *actor) release() {
a.mu.Lock()
closing := a.close
a.close = nil
a.mu.Unlock()
if closing != nil {
closing()
}
}
// holderOf is this process as the lease's holder.
func holderOf(instance string) lease.Holder {
host, _ := os.Hostname()
return lease.Holder{Instance: instance, Host: host, Build: version}
}
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
a.mu.Lock()
a.serving = true
a.mu.Unlock()
js, err := broker.Dial(address)
if err != nil {
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
"lease: %w", broker.BareAddress(address), err)
}
api, err := jetstream.New(js.Conn())
if err != nil {
js.Close()
return nil, err
}
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
err = broker.EnsureLeaseBucket(asserting, api)
cancel()
if err != nil {
js.Close()
return nil, err
}
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
a.mu.Lock()
defer a.mu.Unlock()
a.reset = time.Now()
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
"no machine refuses the next epoch", was, floor, floor)
}})
if err != nil {
js.Close()
return nil, err
}
gone := make(chan struct{})
epoch, err := l.Take(ctx)
switch {
case ctx.Err() != nil:
js.Close()
return nil, ctx.Err()
case err != nil && !errors.Is(err, lease.ErrUnwritable):
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
js.Close()
return nil, err
case err != nil:
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
a.mu.Lock()
a.unleased = err.Error()
a.mu.Unlock()
say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
go a.takeWhenGranted(ctx, l, inv, gone)
default:
a.took(ctx, l, inv, epoch, gone)
}
a.mu.Lock()
a.close = func() {
if held, err := l.Epoch(); err == nil {
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
say("how epoch %d ended could not be recorded: %v", held, err)
}
cancel()
}
l.Release(context.Background())
js.Close()
}
a.mu.Unlock()
return gone, nil
}
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
a.mu.Lock()
a.held, a.unleased = l, ""
a.mu.Unlock()
h := holderOf(instance)
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
Build: h.Build, Taken: time.Now()})
cancel()
if err != nil {
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
}
for _, e := range expired {
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
}
go l.Keep(ctx)
go func() {
<-l.Lost()
if ctx.Err() == nil {
why := l.LostWhy()
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
cancel()
fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
"be started again as a candidate\n", epoch, why)
}
close(gone)
}()
}
// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
// controller if another took it meanwhile: two serving at once is what the lease is for.
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
tick := time.NewTicker(lease.RenewEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
}
epoch, err := l.TryTake(ctx)
if errors.Is(err, lease.ErrTaken) {
fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
"stops and exits\n", err)
close(gone)
return
}
if err != nil {
// Still not written, or not readable this time: unleased, said by S12, tried again.
a.mu.Lock()
a.unleased = err.Error()
a.mu.Unlock()
continue
}
a.took(ctx, l, inv, epoch, gone)
return
}
}
// standing is what `status` and the self-check say of this process and the lease.
type standing struct {
Epoch uint64
Held bool
Renewed time.Time
Unleased string
// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
Reset time.Time
ResetSaid string
}
func (a *actor) standing() standing {
a.mu.Lock()
held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
a.mu.Unlock()
st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
if held == nil {
return st
}
epoch, err := held.Epoch()
st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
return st
}
// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
func init() {
link.ActingGate = func(ctx context.Context) error {
_, err := theLease.epoch(ctx)
if err != nil {
return fmt.Errorf("this controller may not send: %w", err)
}
return nil
}
}
cmd/mesh-controller/acts.go
-4
View File
@@ -224,10 +224,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
for _, line := range unheldChange(shelf, node, assigned, left) {
answer += "\n " + line
}
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
answer += "\n " + line
}
return answer + blockedElsewhere(ctx, open, node), nil
}
cmd/mesh-controller/awaiting_push_test.go
-281
View File
@@ -1,281 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
func aKeeper() catalogue.Manifest {
return catalogue.Manifest{Module: "keeper", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
Resources: []map[string]any{
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
}}
}
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
// are made — without a bus to carry it.
func pushedBy(t *testing.T, open *stores, node string) string {
t.Helper()
ctx := t.Context()
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, node, plan, settings)
if err != nil {
t.Fatal(err)
}
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
digest := digestOf(body)
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
t.Fatal(err)
}
return digest
}
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
func pushedAndApplied(t *testing.T, open *stores, node string) {
t.Helper()
digest := pushedBy(t, open, node)
record, err := open.inventory.NodeByName(t.Context(), node)
if err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
t.Fatal(err)
}
}
// d1 runs D1 once.
func d1(t *testing.T, open *stores) []conditions.Observation {
t.Helper()
got, err := probeDeclarations(t.Context(), &doctor{open: open})
if err != nil {
t.Fatal(err)
}
return got
}
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
// can be compared about a secret that does not exist (status), and nothing here string-matches.
plan, settings, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
var notMade *catalogue.NotMadeError
if !errors.As(err, &notMade) || notMade.Module != "keeper" || notMade.Name != "repository" {
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
}
// Foreseen, it composes, names what the push makes, and made nothing.
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
}
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
}
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
if got := d1(t, open); len(got) != 0 {
t.Fatalf("a machine waiting for a push raised %+v", got)
}
// Past the bound: a warning, not urgent, saying what the push will make.
before := awaitingPushBound
awaitingPushBound = -time.Minute
t.Cleanup(func() { awaitingPushBound = before })
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
}
// Pushed: the secret is made and D1 says nothing.
pushedBy(t, open, "laptop")
if got := d1(t, open); len(got) != 0 {
t.Fatalf("a pushed machine still raised %+v", got)
}
}
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aTalker())
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
t.Fatal(err)
}
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
}
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
// uncomposable for that — the stand-in hides nothing.
register(t, open, aKeeper())
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
t.Fatal(err)
}
one, two := rivals()
register(t, open, one)
register(t, open, two)
_, _ = assign(ctx, open, "anchor", "rival-one")
_, _ = assign(ctx, open, "anchor", "rival-two")
keys := map[string]conditions.Severity{}
for _, o := range d1(t, open) {
keys[o.Key()] = o.Severity
}
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
t.Fatalf("a real failure beside a waiting secret: %v", keys)
}
}
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
// refused on it, so D1 is too.
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
t.Fatalf("a given secret under an old key: %+v", got)
}
}
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop")
sent := time.Now()
since, err := open.inventory.AwaitingSince(ctx, "laptop")
if err != nil || since.After(sent) {
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
}
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
since, err = open.inventory.AwaitingSince(ctx, "laptop")
if err != nil || since.Before(sent.Add(-time.Second)) {
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
}
}
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
// it and had time to report — never between assign and push.
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
now := time.Now()
sent := now.Add(-time.Minute)
long := now.Add(-time.Hour)
cases := []struct {
name string
send lastSend
want bool
}{
{"never sent", lastSend{}, false},
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
}
for _, c := range cases {
if got := c.send.settled("keeper", now); got != c.want {
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
}
}
}
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop")
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
sends, err := readDeliveries(ctx, open.inventory)
if err != nil {
t.Fatal(err)
}
if sends["laptop"].settled("keeper", time.Now()) {
t.Fatal("a holder assigned and not pushed is expected to answer")
}
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
t.Fatal("a module the machine was sent long ago is not expected to answer")
}
pushedBy(t, open, "laptop")
sends, err = readDeliveries(ctx, open.inventory)
if err != nil {
t.Fatal(err)
}
if sends["laptop"].settled("keeper", time.Now()) {
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
}
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
t.Fatal("pushed past the grace, a holder is not expected")
}
if _, ok := sends["laptop"].carried["keeper"]; !ok {
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
}
pushedAndApplied(t, open, "laptop")
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
}
}
cmd/mesh-controller/bindings.go
-140
View File
@@ -1,140 +0,0 @@
package main
import (
"context"
"fmt"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
)
// A binding to data moves only by a person (novox/hq ADR 0232, issue 273).
//
// The resolver keeps each consumer of a provision that keeps its data at the provider it was last
// sent (catalogue/bound.go) and says what it would have moved. This is where that is said: on the
// push that composed it, and by the self-check's D12 every run, as an urgent condition naming the
// consumer, both providers and the pin that confirms the move.
// The condition kinds of D12.
const (
// kindBindingKept is a move the resolver refused: the consumer is still where its data is.
kindBindingKept = "binding-kept"
// kindBindingMoved is a consumer about to be sent another provider than the one on record with
// no pin naming it — what the resolver exists to make impossible, said if it ever is not.
kindBindingMoved = "binding-moved"
// kindBindingMoving is a move a pin asked for, not yet sent: a person's act, said so that the
// data is moved before the push that carries it.
kindBindingMoving = "binding-moving"
)
// probeBindingsID is the self-check's id for this probe, and the source of what it raises.
const probeBindingsID = "D12"
// keptObservation is the urgent condition for one refused move.
func keptObservation(k catalogue.KeptBinding) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeMachine, ID: bindingID(k.Machine, k.Consumer, k.Provision),
Token: kindBindingKept, Kind: kindBindingKept, Machine: k.Machine, Also: otherMachines(k.Machine, k.Bound.Node, k.Would.Node),
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("on %s, the mesh %s", k.Machine, k.String())}
}
func bindingID(machine, consumer, provision string) string {
return machine + "." + consumer + "." + provision
}
func otherMachines(machine string, nodes ...string) []string {
var out []string
seen := map[string]bool{machine: true}
for _, n := range nodes {
if n != "" && !seen[n] {
seen[n] = true
out = append(out, n)
}
}
return out
}
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
// condition at once where this process keeps the conditions: a push is when a person is looking.
func reportKept(ctx context.Context, plan catalogue.Resolution) {
for _, k := range plan.Kept {
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
if conditionsFrom != nil {
o := keptObservation(k)
o.Source = probeBindingsID
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
}
}
}
}
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
// moves it (said, so the data goes first), and never anything else.
func probeBindings(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
inv := d.open.inventory
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
}
var out []conditions.Observation
for _, n := range nodes {
plan, _, err := planFor(ctx, d.open, n.Name)
if err != nil {
if unresolvable(err) {
// D1 says it, with the binding that refused it when that is why.
continue
}
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
}
bound, err := inv.BindingsFor(ctx, n.Name)
if err != nil {
return nil, err
}
pins, err := inv.PinsFor(ctx, n.Name)
if err != nil {
return nil, err
}
out = append(out, bindingFindings(plan, bound, pins)...)
}
return out, nil
}
// bindingFindings is what one machine's resolution says against its record.
func bindingFindings(plan catalogue.Resolution, bound map[string]map[string]catalogue.Chosen,
pins map[string]catalogue.Chosen) []conditions.Observation {
var out []conditions.Observation
for _, k := range plan.Kept {
out = append(out, keptObservation(k))
}
said := map[string]bool{}
for _, need := range plan.Needs {
if !need.KeepsData || need.ByRecord {
continue
}
was, recorded := bound[need.For][need.Name]
now := catalogue.Chosen{Node: need.From, Module: need.Module}
if !recorded || was == now || (was.Module == "" && was.Node == now.Node) {
continue
}
id := bindingID(plan.Node, need.For, need.Name)
if said[id] {
continue
}
said[id] = true
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Machine: plan.Node,
Also: otherMachines(plan.Node, was.Node, now.Node), Resolver: conditions.ResolverOperator}
if pin, pinned := pins[need.Name]; pinned && pin.Node == now.Node && (pin.Module == "" || pin.Module == now.Module) {
o.Token, o.Kind, o.Severity = kindBindingMoving, kindBindingMoving, conditions.Warning
o.Summary = fmt.Sprintf("on %s, %s's %s moves from %s to %s at the next push, by the pin — its data "+
"is on %s: move it first", plan.Node, need.For, need.Name, was, now, was)
} else {
o.Token, o.Kind, o.Severity = kindBindingMoved, kindBindingMoved, conditions.Urgent
o.Summary = fmt.Sprintf("on %s, %s's %s would be sent %s, and it is bound to %s, where its data is, "+
"with no pin naming %s — a move nothing asked for", plan.Node, need.For, need.Name, now, was, now)
}
out = append(out, o)
}
return out
}
cmd/mesh-controller/bindings_test.go
-183
View File
@@ -1,183 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
)
// novox/hq issue 273, ADR 0232: a consumer of a provision that keeps its data moves only by a pin.
func storeManifests() []catalogue.Manifest {
return []catalogue.Manifest{
{Module: "store", Version: "1",
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
}
}
func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) catalogue.Needed {
t.Helper()
for _, n := range plan.Needs {
if n.For == consumer && n.Name == provision {
return n
}
}
t.Fatalf("no %s for %s: %+v", provision, consumer, plan.Needs)
return catalogue.Needed{}
}
// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
for _, m := range storeManifests() {
register(t, open, m)
}
assignAll := func(pairs ...[2]string) {
for _, a := range pairs {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
}
// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
t.Fatal(err)
}
}
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
[2]string{"laptop", "board"})
plan, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
}
if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
}
// Sent, and recorded: only the binding to data.
bindings := boundToData(plan, nil)
if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
t.Fatalf("recorded %+v", bindings)
}
if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
t.Fatal(err)
}
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
}
// The laptop's store taken away: refused, not moved to the anchor's empty one.
if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
t.Fatal(err)
}
if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
}
// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
t.Fatal(err)
}
plan, _, err = planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
t.Fatalf("pinned to the anchor and bound to %s", n.From)
}
found, err := probeBindings(ctx, &doctor{open: open})
if err != nil {
t.Fatal(err)
}
if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
t.Fatalf("a pinned move is not said before it is sent: %+v", found)
}
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
t.Fatal(err)
}
all, err := inv.Bindings(ctx)
if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
t.Fatalf("%+v, %v", all, err)
}
if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
t.Fatalf("a move sent is still said: %+v, %v", found, err)
}
}
// What one machine's resolution says against its record.
func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
home, anchor := catalogue.Chosen{Node: "home", Module: "store"}, catalogue.Chosen{Node: "anchor", Module: "store"}
plan := catalogue.Resolution{Node: "laptop",
Kept: []catalogue.KeptBinding{{Machine: "laptop", Consumer: "board", Provision: "postgres-database",
Bound: home, Would: anchor}},
Needs: []catalogue.Needed{
{Name: "postgres-database", For: "board", From: "home", Module: "store", KeepsData: true},
{Name: "postgres-database", For: "game", From: "anchor", Module: "store", KeepsData: true},
{Name: "wildcard-resolution", For: "network", From: "anchor", Module: "resolver"},
}}
bound := map[string]map[string]catalogue.Chosen{
"board": {"postgres-database": home},
"game": {"postgres-database": home},
"network": {"wildcard-resolution": {Node: "home", Module: "resolver"}},
}
found := linted(bindingFindings(plan, bound, nil))
if len(found) != 2 {
t.Fatalf("found %+v", found)
}
kept, moved := found[0], found[1]
if kept.Kind != kindBindingKept || kept.Severity != conditions.Urgent || kept.Machine != "laptop" ||
!strings.Contains(kept.Summary, "would move board's postgres-database from home/store to anchor/store") ||
!strings.Contains(kept.Summary, "its data is on home/store") ||
!strings.Contains(kept.Summary, "`pin laptop postgres-database anchor store` to confirm a move (and move the data first)") {
t.Errorf("kept: %+v", kept)
}
if moved.Kind != kindBindingMoved || moved.Severity != conditions.Urgent ||
!strings.Contains(moved.Summary, "game's postgres-database would be sent anchor/store") {
t.Errorf("moved: %+v", moved)
}
if kept.Key() == moved.Key() {
t.Error("two consumers, one condition")
}
}
// A push says the move it refused, and raises its condition at once.
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
k, _ := withConditionsInMemory(t)
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
open[0].Source != probeBindingsID {
t.Fatalf("raised %+v", open)
}
}
cmd/mesh-controller/build.go
-92
View File
@@ -6,9 +6,7 @@ import (
"errors"
"flag"
"fmt"
"github.com/novox/mesh-controller/internal/conditions"
"os"
"slices"
"strings"
"time"
@@ -149,8 +147,6 @@ func buildFrom(result link.BuildResult) inventory.Build {
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
// rebuild the graph rather than a list of names.
Path: result.Path,
// What it was made from (novox/hq issue 280): two builds with one are one build.
SourceFingerprint: result.SourceFingerprint,
}
// When it was asked, which is what orders it against another build of the same module
// (novox/hq 04-ISSUES/219) — not when it was heard.
@@ -573,46 +569,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
result.On, result.Repository, short(result.Commit), err)
}
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
// The branch the module already follows is its trunk — never the branch a build was asked at, or a
// build of a feature branch by name would make that branch its trunk.
follows := ""
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
follows = followedBranch(was.Ref)
}
if err := publishable(result, follows); err != nil {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
manifest.Module, short(result.Commit), err)
}
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
// and the next push would send it.
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
return manifest, kept, err
} else if failed {
return manifest, kept, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
short(result.Commit))
}
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
// what the build the mesh stands on was made from registers that build's artifacts at the new
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
// byte-reproducible, and the bus rebuilt for another module's merge demanded a planned upgrade.
if kept.SourceFingerprint != "" {
stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID)
if err != nil {
return manifest, kept, err
}
if stands != "" && stands != kept.ID && len(raw) > 0 {
if same, err := catalogue.ParseManifest(raw); err == nil && same.Module == manifest.Module {
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
manifest.Module, short(result.Commit), stands)
manifest = same
}
}
}
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
if errors.Is(err, inventory.ErrSuperseded) {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
@@ -627,34 +583,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return manifest, kept, nil
}
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
"never published (ADR 0238) — merge it, and the merge builds it")
// publishable says whether a build's outcome may become a module's version: its commit on the module's
// trunk, as the build seat read the forge at the build — the branch the module follows when its source
// names one, else its repository's default branch. A build seat that could not say — one older than the
// rule, building its own successor — is let through and said, so the rule can reach the mesh.
func publishable(result link.BuildResult, follows string) error {
if result.Check != nil || result.Checked != nil || result.DryRun {
return errors.New("a check or a dry run is never published")
}
if result.Trunk == "" {
fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+
"registered as before\n", result.ID, short(result.Commit))
return nil
}
trunk := result.Trunk
if follows != "" {
trunk = follows
}
on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk)
if !on {
return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk)
}
return nil
}
// buildAndShow builds and prints the manifest without recording anything.
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
repository, err := cloneFrom(ctx, source)
@@ -748,24 +676,6 @@ type answers struct {
// refused, until the switch — and while there is any, the mesh is not all well: the order the
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
unheld []catalogue.Unheld
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
// not be read when they could not — never read as none.
conditions []conditions.Condition
conditionsUnread string
// overflowing is every module whose identity overflows the bound of a provision it requires
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
// this is the one place it is said across the mesh. Not well while there is any.
overflowing []catalogue.Overflow
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
// where the log is not on hand; handActsUnread why it could not be read when it could not.
handActs *int
handActsUnread string
// heals is what the healers did in the last seven days (novox/hq to-be 45 §7): acts, and conditions
// handed to the operator; healsUnread why it could not be read.
heals *healsCount
healsUnread string
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
@@ -782,14 +692,12 @@ func heldBy(ctx context.Context) map[string]string {
if err != nil {
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
"base will be told that base is missing: %v\n", err)
// empty-on-error: said above; a build that names a base is refused by name for want of it
return nil
}
defer open.Close()
held, err := open.inventory.Held(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
// empty-on-error: said above; a build that names a base is refused by name for want of it
return nil
}
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
cmd/mesh-controller/bus_step.go
-410
View File
@@ -1,410 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236).
//
// **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the
// controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything,
// and a version whose data format moved (2.10 → 2.11) cannot be undone by putting the old one back.
// So nothing sends a new bus build on its own: its policy is `record` whatever anyone says (the
// catalogue's DerivedUpgrade, and `upgrade` refuses a roll-out), a plan builds it and sends nothing, a
// cascade holds the machine (ADR 0221), and a push naming that machine is refused while a new bus build
// waits for it (busHeld). The one way is this verb: a person, with why, the streams snapshotted first,
// whether it can be reverted said before it starts, the step said as `bus-maintenance` while it runs,
// and every stream, durable consumer and a round trip checked after (H-bus) — or the step said failed,
// with its snapshot as the way back.
// busStepProbe is the registry's row for the step; its kinds.
const (
busStepProbe = "DB"
kindBusMaintenance = "bus-maintenance"
kindBusUpgradeFailed = "bus-upgrade-failed"
)
// busStepBound is how long after its start a bus upgrade must be followed by a healthy bus.
var busStepBound = 15 * time.Minute
// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is: the bus
// machine's backup holder backs the bus module up now, whose dump is the streams' snapshot (novox/hq ADR
// 0235). A variable so a test takes none. A person who took one by hand says where with --snapshot-taken,
// and then none is taken — for a bus whose module does not yet carry the snapshot program.
var takeBusSnapshot = snapshotTheBusNow
// busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per
// machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds
// no bus module.
type busPending struct {
module string
machines []string
from map[string]string
to string
// same are the commits whose build was made from the same source as the build the mesh holds, or
// made the same artifacts and manifest (novox/hq issue 280).
same map[string]bool
}
// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the
// two builds were made from the same source, or made the same artifacts from the same manifest — a
// rebuild of the same source for another module's merge changes nothing the machine runs, whatever
// image digest it made (novox/hq issue 280).
func (b busPending) moves(machine string) bool {
from, known := b.from[machine]
if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) {
return false
}
return !known || !b.same[from]
}
// pendingBus reads what a bus upgrade would do.
func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, error) {
var b busPending
shelf, err := inv.Catalogue(ctx)
if err != nil {
return b, err
}
for name, m := range shelf {
if catalogue.ProvidesBus(m) {
b.module = name
}
}
if b.module == "" {
return b, nil
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
return b, err
}
b.to = current[b.module].Commit
if b.machines, err = inv.Running(ctx, b.module); err != nil {
return b, err
}
b.from, b.same = map[string]string{}, map[string]bool{}
made, err := inv.BuildFingerprints(ctx, b.module)
if err != nil {
return b, err
}
for commit, refs := range made {
b.same[commit] = refs != "" && refs == made[b.to]
}
sources, err := inv.BuildSourceFingerprints(ctx, b.module)
if err != nil {
return b, err
}
for commit, src := range sources {
if src != "" && src == sources[b.to] {
b.same[commit] = true
}
}
for _, n := range b.machines {
sent, known, err := inv.SentBuilds(ctx, n)
if err != nil {
return b, err
}
if known {
if c, carried := sent[b.module]; carried {
b.from[n] = c
}
}
}
return b, nil
}
// busHeld names the machines a push may not send because sending them would replace the bus: the
// planned step's, not a push's (ADR 0236). Said with the remedy.
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
b, err := pendingBus(ctx, inv)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, n := range machines {
for _, holder := range b.machines {
if n == holder && b.moves(n) {
out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+
"`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)",
n, b.module, short(orNotKnown(b.from[n])), short(b.to))
}
}
}
return out, nil
}
// busCommand is `bus` — what a bus upgrade would do and how the last went — and `bus upgrade`.
func busCommand(ctx context.Context, args []string) error {
sub := ""
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
set := flag.NewFlagSet("bus", flag.ContinueOnError)
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
irreversible := set.Bool("irreversible", false, "the new version cannot be undone by putting the old one back, "+
"and this is the person's explicit word that it runs anyway")
why := addHandActFlags(set)
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>]]")
}
switch sub {
case "":
return busStatus(ctx)
case "upgrade":
default:
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub)
}
// Everything refused before anything is done.
if err := why.require("bus upgrade"); err != nil {
return err
}
if *reversible == *irreversible {
return errors.New("bus upgrade says, before it starts, whether the new version can be undone by putting the " +
"old one back: --reversible, or --irreversible as your explicit word that it runs anyway (to-be 45 §8). " +
"Nothing was done")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
b, err := pendingBus(ctx, inv)
if err != nil {
return err
}
if b.module == "" {
return errors.New("the mesh holds no module that provides its bus: there is nothing to upgrade")
}
var moving []string
for _, n := range b.machines {
if b.moves(n) {
moving = append(moving, n)
}
}
if len(moving) == 0 {
fmt.Printf("every machine running %s runs the build the mesh holds (%s): nothing to upgrade\n", b.module, short(b.to))
return nil
}
where := strings.TrimSpace(*snapshot)
if where == "" {
for _, n := range moving {
fmt.Printf("snapshotting the bus's streams on %s first (its backup holder, ADR 0235)…\n", n)
if where, err = takeBusSnapshot(ctx, b.module, n); err != nil {
return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w — a snapshot "+
"taken by hand is said with --snapshot-taken <where>", err)
}
}
}
from := map[string]bool{}
for _, n := range moving {
from[orNotKnown(b.from[n])] = true
}
step, err := inv.StartBusStep(ctx, inventory.BusStep{Module: b.module, Machines: moving,
From: strings.Join(sortedKeys(from), ", "), To: b.to, Snapshot: where, Reversible: *reversible,
By: link.Caller(), Why: strings.TrimSpace(*why.why)})
if err != nil {
return err
}
cause := "bus-upgrade"
if strings.TrimSpace(*why.cause) == "" {
why.cause = &cause
}
why.record(ctx, "bus upgrade", append([]string{b.module}, moving...))
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
if err != nil {
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
}
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
return nil
}
// busStatus is `bus`: what an upgrade would do, and the last step.
func busStatus(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
b, err := pendingBus(ctx, open.inventory)
if err != nil {
return err
}
if b.module == "" {
fmt.Println("the mesh holds no module that provides its bus")
} else {
fmt.Printf("the bus is %s, built %s, on %s; never rolled out — a planned step (`bus upgrade`)\n", b.module,
short(b.to), orNone(strings.Join(b.machines, ", ")))
for _, n := range b.machines {
state := "runs it"
if b.moves(n) {
state = "runs " + short(orNotKnown(b.from[n])) + ": `bus upgrade` replaces it"
}
fmt.Printf(" %-10s %s\n", n, state)
}
}
fmt.Println(" `bus upgrade` has the bus machine's backup holder snapshot the streams first (ADR 0235)")
s, found, err := open.inventory.LatestBusStep(ctx)
if err != nil || !found {
return err
}
state := "running since " + s.Started.Local().Format("2006-01-02 15:04")
if s.Ended != nil {
state = s.Outcome + " at " + s.Ended.Local().Format("2006-01-02 15:04")
}
fmt.Printf("last step %d: %s → %s on %s by %s (%s): %s; snapshot %s\n", s.ID, s.From, short(s.To),
strings.Join(s.Machines, ", "), orNone(s.By), s.Why, state, s.Snapshot)
if s.Found != "" {
fmt.Printf(" %s\n", s.Found)
}
return nil
}
// probeBusStep is DB: a bus upgrade running is said as `bus-maintenance`; the bus healthy again after
// the machines reported the new build ends it done; past its bound, unhealthy, it ends failed and is
// said — urgent, with its snapshot — while the bus is still not healthy.
func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
inv := d.open.inventory
s, found, err := inv.LatestBusStep(ctx)
if err != nil || !found {
return nil, err
}
if s.Ended != nil && s.Outcome != "failed" {
return nil, nil
}
problems, err := busHealth(ctx, d)
if err != nil {
return nil, err
}
reports, err := inv.LastReports(ctx)
if err != nil {
return nil, err
}
applied := true
for _, r := range reports {
for _, n := range s.Machines {
if r.Node == n && (!r.Current || r.Outcome != inventory.OutcomeApplied || r.At == nil || r.At.Before(s.Started)) {
applied = false
problems = append(problems, n+" has not reported the new bus applied")
}
}
}
id := s.Module
if s.Ended == nil {
switch {
case applied && len(problems) == 0:
return nil, inv.EndBusStep(ctx, s.ID, "done", "the bus answered healthy after the upgrade")
case time.Since(s.Started) > busStepBound:
found := strings.Join(problems, "; ")
if err := inv.EndBusStep(ctx, s.ID, "failed", found); err != nil {
return nil, err
}
s.Found = found
default:
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "maintenance",
Kind: kindBusMaintenance, Severity: conditions.Warning,
Summary: fmt.Sprintf("the bus is being upgraded (step %d, %s → %s on %s, by %s: %s); its snapshot is %s",
s.ID, s.From, short(s.To), strings.Join(s.Machines, ", "), orNone(s.By), s.Why, s.Snapshot),
Said: orNone(strings.Join(problems, "; "))}}, nil
}
}
if len(problems) == 0 {
return nil, nil // failed, and healthy since: nothing wrong now
}
way := "put the old build back"
if !s.Reversible {
way = "restore the snapshot"
}
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "upgrade-failed",
Kind: kindBusUpgradeFailed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the bus upgrade (step %d, %s → %s) did not end healthy within %s: %s — the way back is to %s (%s)",
s.ID, s.From, short(s.To), busStepBound, strings.Join(problems, "; "), way, s.Snapshot)}}, nil
}
func sortedKeys(set map[string]bool) []string {
out := make([]string, 0, len(set))
for k := range set {
out = append(out, k)
}
sort.Strings(out)
return out
}
// busSnapshotWithin is how long the bus machine's backup holder is given to take the bus's snapshot.
var busSnapshotWithin = 15 * time.Minute
// snapshotTheBusNow asks the bus machine's backup holder to back the bus module up now — its dump is
// the streams' snapshot (novox/hq ADR 0235) — and waits until it says a backup newer than the ask:
// where the snapshot is, as a person reads it. The serving controller's connection, or one of its own.
func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) {
var where string
err := onTheBus(func(conn *nats.Conn) error {
asked := time.Now()
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", node,
map[string]any{"module": module}, 30*time.Second)
if err != nil {
return fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
}
if answer.Error != "" {
return fmt.Errorf("%s's backup holder would not take the bus's snapshot: %s", node, answer.Error)
}
deadline := time.Now().Add(busSnapshotWithin)
for {
answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", node, map[string]any{}, 10*time.Second)
if err == nil && answer.Error == "" {
if measured, err := readHolder(answer.Result); err == nil {
for item, m := range measured[module] {
if m.LastBackup != nil && m.LastBackup.After(asked) && m.Error == "" {
where = fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, item,
m.LastBackup.UTC().Format(time.RFC3339))
return nil
}
}
}
}
if time.Now().After(deadline) {
return fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s; the bus is "+
"not replaced", node, busSnapshotWithin)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(10 * time.Second):
}
}
})
return where, err
}
// errBusWaits is a send refused because it would replace the bus outside its planned step.
var errBusWaits = errors.New("a new bus build waits for its planned step")
type busStepKey struct{}
// withBusStep marks a send as the bus's planned step: the one send that may replace the bus.
func withBusStep(ctx context.Context) context.Context {
return context.WithValue(ctx, busStepKey{}, true)
}
func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on }
cmd/mesh-controller/busobjects.go
-162
View File
@@ -1,162 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
// D6 and D7).
//
// **What the controller asserts at its start and what the self-check expects to find are one
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
// the drift rather than the fault.
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
// can now hear a declaration.
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
}
names := make([]string, 0, len(nodes))
for _, n := range nodes {
names = append(names, n.Name)
}
if err := broker.Raise(r, names); err != nil {
return nil, err
}
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
// after something started asking for builds flushes the backlog instead of having lost it.
// With the seats' holders, so each role's work queue gets the consumer its holder takes
// work from. Passed as nil until the first live raise, which left the build machine bound to a
// consumer nothing had created (2026-09-28).
holders, err := seatHolders(ctx, inv)
if err != nil {
return nil, err
}
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
return nil, err
}
// And how every module hears what it consumes. Derived from the same records the user list is
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
// Done on every raise, not only when a credential is issued: every module moved onto this bus
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
consumers, err := moduleConsumers(ctx, inv)
if err != nil {
return nil, err
}
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
var failed []error
for _, c := range consumers {
if err := r.EnsureConsumer(c.Consumer); err != nil {
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
}
}
if len(failed) > 0 {
return nil, errors.Join(failed...)
}
return names, nil
}
// What raises the condition a send says when the objects it implies could not be asserted, and its kind.
const (
sourceBusObjects = "bus-objects"
kindBusObjectsUnasserted = "bus-objects-unasserted"
)
// assertOnSend asserts, on the bus a send is about to use, every object assertBusObjects derives —
// **whenever a declaration is sent, not only when the controller starts** (novox/hq issue 208).
//
// A module assigned after the controller started was sent its declaration and found no consumer to
// bind (`consumer not found`, messenger on 2026-10-06), and a seat holder assigned after it found no
// worker: the objects a declaration implies were asserted at start and nowhere else, so they existed
// only for what was assigned before the last restart. The same derivation, not a second list of what
// a send needs: what start asserts, the self-check expects and a send asserts are one answer. Every
// part is idempotent, so asserting the whole of it again is the no-op a restart already relies on.
//
// **A failure is said and raised, and the send goes on.** The objects are the mesh's, not the
// machines' being sent: holding every machine back for one consumer that none of them may use would
// turn one fault into all of them, and the declarations are not what is wrong. It is never silent —
// said in the send's own output and raised as a condition, which the next send that asserts them
// clears — and the start-time raise still refuses to serve without them.
func assertOnSend(ctx context.Context, inv *inventory.Inventory, r broker.Raiser, indent string) error {
_, err := assertBusObjects(ctx, inv, r)
var observed []conditions.Observation
if err != nil {
fmt.Printf("%sTHE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES: %v\n", indent, err)
fmt.Printf("%s a module may find no consumer to bind, or a holder no worker; sent anyway, raised as "+
"condition %s, and asserted again by the next send\n", indent, unassertedObservation(err).Key())
observed = append(observed, unassertedObservation(err))
}
// Observed when it failed, cleared when it did not: a send that asserted everything is the
// observation that the bus holds what it should.
if kerr := withKeeper(ctx, func(k *conditions.Keeper) error {
return k.Reconcile(ctx, sourceBusObjects, observed)
}); kerr != nil {
fmt.Printf("%sand whether the bus holds what this send implies could not be kept as a condition: %v\n",
indent, kerr)
}
return err
}
// unassertedObservation is a send's failure to assert the bus's objects, as a condition.
func unassertedObservation(err error) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeBus, ID: "objects", Token: "unasserted",
Kind: kindBusObjectsUnasserted, Severity: conditions.Warning, Source: sourceBusObjects,
Summary: "the bus's streams and consumers could not be asserted when a declaration was sent: " +
"a module may find no consumer to bind, or a seat's holder no worker",
Said: err.Error()}
}
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
records, err := inv.BusRecords(ctx)
if err != nil {
return nil, err
}
users, err := broker.Users(records)
if err != nil {
return nil, err
}
return broker.ConsumersOf(users), nil
}
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
consumers, err := moduleConsumers(ctx, inv)
return len(consumers), err
}
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
var rec recordingRaiser
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
}
return rec.streams, rec.consumers, nil
}
// recordingRaiser keeps what it was asked to assert and asserts nothing.
type recordingRaiser struct {
streams []broker.Stream
consumers []broker.Consumer
}
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
r.streams = append(r.streams, s)
return nil
}
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
r.consumers = append(r.consumers, c)
return nil
}
cmd/mesh-controller/busobjects_test.go
-197
View File
@@ -1,197 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
// only when the controller starts.
// aCarriedConsumer is the runtime on laptop and, carried by it, a module that consumes an event: the
// shape of messenger on 2026-10-06, assigned after the controller started.
func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
t.Helper()
register(t, open, catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1",
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}})
register(t, open, catalogue.Manifest{Module: "messenger", Version: "1",
Consumes: []string{"billing.order.placed"}})
for _, m := range []string{catalogue.RuntimeModule, "messenger"} {
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
t.Fatal(err)
}
}
consumers, err := moduleConsumers(t.Context(), open.inventory)
if err != nil {
t.Fatal(err)
}
for _, c := range consumers {
if c.Module == "messenger" && c.Node == "laptop" {
return c.Consumer
}
}
t.Fatalf("messenger on laptop is derived no consumer: %+v", consumers)
return broker.Consumer{}
}
// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
// started, and the worker its seat's queue should have for it.
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
t.Helper()
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
t.Fatal(err)
}
for _, s := range inventory.MeshSeats() {
if s.Name == "node-build-agent" {
c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
if !needed {
t.Fatal("the build agent's seat needs no worker")
}
return c
}
}
t.Fatal("the mesh declares no build agent's seat")
return broker.Consumer{}
}
// asserted says whether a recording holds a consumer by stream and name.
func asserted(rec *recordingRaiser, want broker.Consumer) bool {
for _, c := range rec.consumers {
if c.Stream == want.Stream && c.Name == want.Name {
return true
}
}
return false
}
// What a send asserts includes what was assigned after the start's assertion: a carried module's
// consumer and a late holder's worker. The derivation is the start's own, so this holds without a bus.
func TestASendAssertsWhatWasAssignedAfterStart(t *testing.T) {
open := aMesh(t)
var atStart recordingRaiser
if _, err := assertBusObjects(t.Context(), open.inventory, &atStart); err != nil {
t.Fatal(err)
}
consumer := aCarriedConsumer(t, open)
worker := aLateHolder(t, open)
if asserted(&atStart, consumer) || asserted(&atStart, worker) {
t.Fatal("the start asserted what was not yet assigned; the test proves nothing")
}
var onSend recordingRaiser
if err := assertOnSend(t.Context(), open.inventory, &onSend, ""); err != nil {
t.Fatal(err)
}
if !asserted(&onSend, consumer) {
t.Fatalf("messenger's consumer %s on %s is not asserted by the send: %+v", consumer.Name, consumer.Stream, onSend.consumers)
}
if !asserted(&onSend, worker) {
t.Fatalf("the late holder's worker %s on %s is not asserted by the send: %+v", worker.Name, worker.Stream, onSend.consumers)
}
}
// failingRaiser refuses one consumer by name and records everything it was asked.
type failingRaiser struct {
recordingRaiser
refuse string
}
func (f *failingRaiser) EnsureConsumer(c broker.Consumer) error {
f.consumers = append(f.consumers, c)
if c.Name == f.refuse {
return errors.New("nats: API error: code=503 description=insufficient resources")
}
return nil
}
// A send whose objects cannot be asserted says so in its output and raises a condition — and the next
// send that asserts them clears it. The consumers after the refused one are still asked for.
func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
open := aMesh(t)
consumer := aCarriedConsumer(t, open)
worker := aLateHolder(t, open)
failing := &failingRaiser{refuse: consumer.Name}
var sendErr error
out := stdoutOf(t, func() error {
sendErr = assertOnSend(t.Context(), open.inventory, failing, " ")
return nil
})
if sendErr == nil || !strings.Contains(sendErr.Error(), "how messenger on laptop hears what it consumes") {
t.Fatalf("the failure is not answered: %v", sendErr)
}
if !strings.Contains(out, "THE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES") ||
!strings.Contains(out, "insufficient resources") || !strings.Contains(out, "bus.objects.unasserted") {
t.Fatalf("the failure is not said in the send's output:\n%s", out)
}
if !asserted(&failing.recordingRaiser, worker) {
t.Fatal("the seat's worker was not asked for")
}
c, open1, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted")
if err != nil || !open1 {
t.Fatalf("no condition raised: %v", err)
}
if c.Kind != kindBusObjectsUnasserted || c.Source != sourceBusObjects ||
len(c.Evidence) == 0 || !strings.Contains(c.Evidence[0].Said, "insufficient resources") {
t.Fatalf("the condition does not say what failed: %+v", c)
}
// The next send asserts them, and that observation clears it.
if err := assertOnSend(t.Context(), open.inventory, &recordingRaiser{}, ""); err != nil {
t.Fatal(err)
}
if _, stillOpen, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); err != nil || stillOpen {
t.Fatalf("a send that asserted everything left the condition open: %v", err)
}
}
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
// once a declaration is sent, and a holder assigned after start its seat's worker.
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
url := testbus.URL(t)
open := aMesh(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
_ = js.Context().DeleteStream(s)
}
// The controller starting, before either was assigned.
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
t.Fatal(err)
}
consumer := aCarriedConsumer(t, open)
worker := aLateHolder(t, open)
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
for _, c := range []broker.Consumer{consumer, worker} {
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
}
}
server := link.ConnectNats(js, nil, nil)
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
t.Fatal(err)
}
for _, c := range []broker.Consumer{consumer, worker} {
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
}
}
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
t.Fatal("a send that asserted everything raised a condition")
}
}
cmd/mesh-controller/calls_verb_test.go
-81
View File
@@ -1,81 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
// RequestTimeout) — the shortest wait of a caller the mesh ships.
const consoleWaits = 30 * time.Second
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
if link.AnswerWithin >= consoleWaits/2 {
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
}
if link.AnswerWithin >= broker.ResponseTTL {
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
}
}
// A push — named or through command — answers before it runs: it sends the machine holding the bus
// first, and the broker reloading its user list forgets the answer it was about to permit.
func TestAPushAnswersBeforeItSends(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want bool
}{
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
{"push", map[string]any{"why": "w"}, true},
{"command", map[string]any{"command": "push anchor --why w"}, true},
{"command", map[string]any{"command": "push --behind --why=w"}, true},
{"command", map[string]any{"command": "builds"}, false},
{"status", map[string]any{}, false},
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil {
t.Fatalf("%s %v: %v", c.verb, c.args, err)
}
if got := answersFirst(argv); got != c.want {
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
}
}
}
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
handlers, behind, err := seatToolHandlers()
if err != nil || len(behind) != 0 {
t.Fatalf("%v %v", behind, err)
}
calls, ok := handlers["calls"]
if !ok {
t.Fatal("calls is not served")
}
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
!strings.Contains(err.Error(), `"node"`) {
t.Errorf("calls took an argument it does not declare: %v", err)
}
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
!strings.Contains(err.Error(), "not across a restart") {
t.Errorf("an unknown call was not said plainly: %v", err)
}
got, err := calls(context.Background(), json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
if _, listed := got.(map[string]any)["calls"]; !listed {
t.Errorf("calls answered %v", got)
}
}
cmd/mesh-controller/changeplan.go
-155
View File
@@ -1,155 +0,0 @@
package main
import (
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
// planner, so the two can be compared.
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
type policyOf func(module string) (inventory.Upgrade, bool)
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
machines := map[string]*link.MachinePlan{}
machine := func(name string) *link.MachinePlan {
if machines[name] == nil {
machines[name] = &link.MachinePlan{Machine: name}
}
return machines[name]
}
for _, tier := range r.Plan.Tiers {
for _, name := range tier {
e, held := byName[name]
if !held {
continue
}
u, known := policy(name)
waits := known && !u.RollOut
for _, on := range e.On {
if waits {
machine(on).Waits = append(machine(on).Waits, name)
} else {
machine(on).Receives = append(machine(on).Receives, name)
}
}
switch {
case name == "nats":
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
case waits && len(e.On) > 0:
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
orNone(u.From)))
}
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
var offers []string
for _, o := range e.Manifest.Provides {
offers = append(offers, o.Name)
}
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
name, strings.Join(offers, ", ")))
}
if e.Manifest.Data != nil && len(e.On) > 0 {
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
}
}
}
var names []string
for name := range machines {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
p.Machines = append(p.Machines, *machines[name])
}
p.Summary = summaryOf(p)
return p
}
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
func summaryOf(p link.ChangePlan) string {
if len(p.Moved) == 0 && len(p.New) == 0 {
return "builds nothing: the change touches no module of the mesh's graph"
}
var parts []string
what := strings.Join(p.Moved, ", ")
if len(p.Dependents) > 0 {
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
}
if len(p.New) > 0 {
if what != "" {
what += ", "
}
what += "new: " + strings.Join(p.New, ", ")
}
var to []string
for _, m := range p.Machines {
if len(m.Receives) > 0 {
to = append(to, m.Machine)
}
}
if len(to) > 0 {
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
} else {
parts = append(parts, "builds "+what+", sent nowhere")
}
bus := "no bus step"
for _, s := range p.Steps {
if strings.HasPrefix(s, "a planned bus step") {
bus = "a bus step"
}
}
parts = append(parts, bus)
waiting := 0
for _, m := range p.Machines {
waiting += len(m.Waits)
}
if waiting > 0 {
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
}
return strings.Join(parts, "; ")
}
// planText is a change plan as a person reads it, for the pull request's comment.
func planText(p link.ChangePlan) string {
var b strings.Builder
fmt.Fprintf(&b, "change plan of %s at %.8s into %s: %s\n", p.Repository, p.Head, p.Base, p.Summary)
for i, tier := range p.Tiers {
fmt.Fprintf(&b, " tier %d: %s\n", i, strings.Join(tier, ", "))
}
for _, m := range p.Machines {
line := " " + m.Machine + ": "
if len(m.Receives) > 0 {
line += "receives " + strings.Join(m.Receives, ", ")
}
if len(m.Waits) > 0 {
if len(m.Receives) > 0 {
line += "; "
}
line += "waits for a person: " + strings.Join(m.Waits, ", ")
}
b.WriteString(line + "\n")
}
for _, s := range p.Steps {
b.WriteString(" - " + s + "\n")
}
if len(p.Unread) > 0 {
fmt.Fprintf(&b, " read by no module's build: %s\n", strings.Join(p.Unread, ", "))
}
return b.String()
}
cmd/mesh-controller/changeplan_test.go
-74
View File
@@ -1,74 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
// person, a provider whose consumers follow it.
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
entry := func(name, path string, on ...string) inventory.Entry {
e := fromRepo(name, catalogue_, path)
e.Source.BuiltFrom = "old"
e.On = on
return e
}
nats := entry("nats", "modules/nats", "anchor")
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
gitea := entry("gitea", "modules/gitea", "anchor")
held := entry("photos", "modules/photos", "anchor", "laptop")
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
tools.On = []string{"anchor", "laptop"}
entries := []inventory.Entry{nats, gitea, held, tools}
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
policy := func(module string) (inventory.Upgrade, bool) {
if module == "photos" {
return inventory.Upgrade{RollOut: false, From: "a person"}, true
}
return inventory.Upgrade{RollOut: true}, true
}
plan := func(paths ...string) link.ChangePlan {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
}
p := plan("modules/gitea/index.ts")
if p.Summary != "builds gitea → anchor; no bus step" {
t.Errorf("one module's change reads %q", p.Summary)
}
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
if !strings.Contains(p.Summary, "a bus step") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
t.Errorf("the bus and a held module read %q", p.Summary)
}
got := map[string]string{}
for _, m := range p.Machines {
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
}
// The bus first, what stands on it after: the build plan's order, per machine.
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
t.Errorf("the deploy plan reads %v", got)
}
text := strings.Join(p.Steps, "\n")
for _, want := range []string{"a planned bus step", "photos waits for a person", "nats provides mesh-bus"} {
if !strings.Contains(text, want) {
t.Errorf("the steps do not say %q:\n%s", want, text)
}
}
p = plan("merge-check.sh")
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
t.Errorf("a root file's plan reads %+v", p)
}
if !strings.Contains(planText(p), "read by no module's build: merge-check.sh") {
t.Errorf("the plan's text does not say why nothing is built:\n%s", planText(p))
}
}
cmd/mesh-controller/check.go
+1 -38
View File
@@ -25,17 +25,7 @@ import (
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
// refused what it could not see would teach people to ignore it.
//
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
// provider's machine when a real machine's name first meets the module's.
func moduleCheck(paths []string, out io.Writer) error {
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
}
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
if len(paths) == 0 {
return errors.New("module check <manifest.json>... — one file per module; pass every " +
"manifest of a repository together so the rules between them are checked too")
@@ -84,24 +74,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
}
failed += len(problems)
// Between the manifests too: an identity against the bounds of the provisions it wants, which
// only the provider's manifest states.
identities := catalogue.IdentityProblems(shelf, longestMachine)
sort.Strings(identities)
for _, p := range identities {
fmt.Fprintln(out, p)
}
failed += len(identities)
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
data := catalogue.DataProblems(shelf)
sort.Strings(data)
for _, p := range data {
fmt.Fprintln(out, p)
}
failed += len(data)
var names []string
for name := range shelf {
names = append(names, name)
@@ -130,14 +102,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
if len(m.Reads) > 0 {
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
}
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
if items := m.DataItems(); len(items) > 0 {
kept := make([]string, 0, len(items))
for _, it := range items {
kept = append(kept, it.ID+" ("+it.Class+")")
}
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
}
fmt.Fprintln(out)
}
if failed > 0 {
@@ -145,8 +109,7 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
}
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
"against the bounds of the providers given here\n", len(paths), longestMachine)
"module not given here reads as unknown\n", len(paths))
return nil
}
cmd/mesh-controller/check_here.go
-201
View File
@@ -1,201 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// `check-here` is a pull request's merge check run on the machine at hand **exactly as the build seat
// runs it** (novox/hq issue 286): the same code (builder.Check), the same ask the controller would make of
// the seat — the gate's modules and judge by the planner's own answer over the facts snapshot, the
// repositories beside it at the refs the snapshot says the seat clones them at — in the toolchain image
// the mesh holds, as the user the build seat runs as, against a throwaway store and bus of the versions
// the mesh runs.
//
// **The build seat is the reference.** A merge-check.sh that passed on an agent's machine failed on the
// seat for three reasons that were each the agent's environment, never the change: a newer Go whose gofmt
// lays a file out differently, a sibling checkout at the agent's feature branch where the seat had the
// commit the mesh runs, and a different user. Run here, a check sees what the seat will.
//
// check-here [--tree <checkout>] [--base main] [--registry <artifact store>] [--forge <url of the owner>]
// [--number <n>] [--user uid:gid] [--facts store|<file>] [--keep]
//
// The checkout's HEAD is what is checked, and it must be committed: the seat checks a commit, never a
// working tree.
func checkHereCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("check-here", flag.ContinueOnError)
tree := set.String("tree", ".", "the change's checkout; its HEAD is checked")
base := set.String("base", "main", "the branch the change would merge into")
registry := set.String("registry", os.Getenv("MESH_REGISTRY"), "the artifact store holding the facts and the toolchains")
forge := set.String("forge", "", "where the repositories beside it are cloned from, as <forge>/<repository>.git; "+
"the checkout's origin without its own name when not given")
number := set.Int("number", 0, "the pull request's number, when there is one")
// The build seat's service runs as root, and its check containers run as the builder does.
user := set.String("user", "0:0", "the user the check's containers run as: the build seat's")
keep := set.Bool("keep", false, "keep the workspace afterwards")
factsFrom := set.String("facts", "store", "the facts snapshot: `store`, the one the artifact store holds — "+
"what the seat reads — or a file")
if _, err := parseAround(set, args); err != nil {
return err
}
if *registry == "" {
return errors.New("check-here reads the facts and the toolchains from the artifact store: --registry <host:port> or MESH_REGISTRY")
}
dir, err := filepath.Abs(*tree)
if err != nil {
return err
}
git := func(args ...string) (string, error) {
cmd := exec.CommandContext(ctx, "git", args...)
cmd.Dir = dir
out, err := cmd.Output()
return strings.TrimSpace(string(out)), err
}
if dirty, err := git("status", "--porcelain", "--untracked-files=no"); err != nil {
return fmt.Errorf("%s is not a checkout: %w", dir, err)
} else if dirty != "" {
return errors.New("the checkout has changes not committed: the build seat checks a commit, so commit first")
}
head, err := git("rev-parse", "HEAD")
if err != nil {
return err
}
origin, err := git("remote", "get-url", "origin")
if err != nil {
return fmt.Errorf("the checkout has no origin to say which repository it is: %w", err)
}
owner, repo, prefix := ownerRepoOf(origin)
if *forge == "" {
*forge = prefix
}
if _, err := git("fetch", "--quiet", "origin", *base); err != nil {
return fmt.Errorf("cannot fetch %s to say what the change touches: %w", *base, err)
}
changedText, err := git("diff", "--name-only", "origin/"+*base+"...HEAD")
if err != nil {
return err
}
var paths, removed []string
for _, p := range strings.Split(changedText, "\n") {
if p = strings.TrimSpace(p); p == "" {
continue
}
paths = append(paths, p)
if _, err := os.Stat(filepath.Join(dir, p)); os.IsNotExist(err) {
removed = append(removed, p)
}
}
_ = os.Setenv("MESH_REGISTRY", *registry)
f, err := readFacts(ctx, *factsFrom)
if err != nil {
return fmt.Errorf("the facts snapshot cannot be read: %w", err)
}
entries, read, edges, err := graphOfFacts(f)
if err != nil {
return err
}
p := link.PullUpdated{Owner: owner, Repo: repo, Number: *number, Base: *base, Commit: head, Paths: paths,
Removed: removed, ModuleDirs: moduleDirsIn(dir, paths), ModuleDirsSaid: true}
scope := pullScope(p, entries, read, edges)
_, scriptErr := os.Stat(filepath.Join(dir, builder.CheckScript))
if !scope.gated() && !scope.Mesh {
fmt.Printf("%s: %s, and the repository is not the mesh's: the build seat runs nothing for it\n",
owner+"/"+repo, noModuleTouched)
return nil
}
if !scope.gated() && scriptErr != nil {
fmt.Printf("%s: %s; %s\n", owner+"/"+repo, noModuleTouched, noMergeCheck)
return nil
}
toolchains := map[string]string{}
for language, reference := range f.Versions.Toolchains {
toolchains[language] = catalogue.Rerouted(reference, *registry)
}
if len(toolchains) == 0 {
return errors.New("the facts snapshot names no toolchain: it was taken by a controller from before " +
"issue 286, and the seat's toolchain cannot be known here")
}
beside := map[string]builder.Beside{}
for d, ref := range f.Beside {
from := d
if d == "mesh-controller-main" {
from = "mesh-controller"
}
beside[d] = builder.Beside{Repository: strings.TrimSuffix(*forge, "/") + "/" + from + ".git", Ref: ref}
}
id := fmt.Sprintf("check-here-%d", time.Now().UnixNano())
spec := builder.CheckSpec{ID: id, Repository: dir, Ref: head, Owner: owner, Repo: repo, Number: *number,
Paths: paths, Beside: beside, Modules: scope.Modules, New: scope.New, Manifests: scope.Manifests,
Base: *base, Judge: scope.Judge, Toolchain: toolchains["go"], Toolchains: toolchains, User: *user}
workspace, err := os.MkdirTemp("", "mesh-check-here-")
if err != nil {
return err
}
if !*keep {
defer removeWorkspace(spec.Toolchain, workspace, *user)
}
fmt.Fprintf(os.Stderr, "checking %s/%s at %.8s as the build seat would, against the facts of %s, in %s\n",
owner, repo, head, f.Taken.Format(time.RFC3339), workspace)
v, err := builder.Check(ctx, builder.Command, spec, workspace, *registry, builder.GitCredential{},
func(step, message string) {
if step != "output" {
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
}
})
if err != nil {
return fmt.Errorf("the check could not run — on the seat an error, never a pass: %w", err)
}
fmt.Println(v.Report)
fmt.Println()
fmt.Printf("mesh/merge-gate: %s — %s\n", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary)
if v.Repo != nil {
fmt.Printf("mesh/repo-check: %s — %s\n", strings.ToUpper(v.Repo.Verdict), v.Repo.Summary)
}
for _, l := range []*builder.Layer{v.Gate, v.Repo} {
if l != nil && l.Verdict != "pass" && l.Verdict != "warning" {
return errors.New("the build seat would not pass this change")
}
}
return nil
}
// ownerRepoOf reads owner, repository and the owner's URL from a remote: ssh://git@host:222/novox/mesh-host.git
// → novox, mesh-host, ssh://git@host:222/novox.
func ownerRepoOf(remote string) (string, string, string) {
trimmed := strings.TrimSuffix(strings.TrimSuffix(remote, "/"), ".git")
cut := strings.LastIndexAny(trimmed, "/:")
if cut < 0 {
return "", trimmed, ""
}
repo, prefix := trimmed[cut+1:], trimmed[:cut]
owner := prefix
if at := strings.LastIndexAny(prefix, "/:"); at >= 0 {
owner = prefix[at+1:]
}
return owner, repo, prefix
}
// removeWorkspace removes what the check left, written as the seat's user: by a container of that user
// when it is not this one.
func removeWorkspace(image, workspace, user string) {
if image != "" && user != fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) {
// Everything in it — the check's HOME is the workspace, so the toolchain's own files are there too.
_ = exec.Command("docker", "run", "--rm", "--user", user, "--volume", workspace+":/workspace", image,
"sh", "-c", "rm -rf /workspace/* /workspace/.[!.]*").Run()
}
_ = os.RemoveAll(workspace)
}
cmd/mesh-controller/check_test.go
-11
View File
@@ -92,14 +92,3 @@ func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
t.Fatalf("a name declared on purpose passes; got %v", err)
}
}
// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged
// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it —
// refused it. Held here, so a verb added to the table without the manifest fails this repository's
// own suite rather than its next pull request's gate.
func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
var out strings.Builder
if err := moduleCheck([]string{"../../module.json"}, &out); err != nil {
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
}
}
cmd/mesh-controller/checks.go
-400
View File
@@ -1,400 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"path"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A pull request's merge check (novox/hq to-be 45 §9, ADR 0237 as amended 2026-10-06): the forge
// announces every pull request's new head, the controller decides what is checked, asks the build seat to
// check it, and says the verdict as `checked`, which the forge's holder sets as the pull request's
// statuses. **Before merge, never after**: every check the mesh had ran after a merge, on a machine.
//
// **The mesh's module graph decides, not the repository.** The controller holds the graph — every module,
// the repository and directory it is built from — and maps the pull request's changed paths onto it by
// the planner's own answer (reachOfMerge, touchedBy — the one place a changed file is mapped onto modules,
// for the merge handler, the release planner, the merge gate and this check; issue 280): **a changed file
// touches exactly the modules whose build reads it** — a module's own directory (the whole repository for
// one built from its root), or a repository its recipe packages. A file no build reads — a script at the
// root, a README — touches no module. A directory the change adds a module.json in, which the graph does
// not hold yet (said by the head, issue 278), is a new module and is checked too.
//
// - touches a module: the build seat runs **the gate** — `mesh/merge-gate`, the touched manifests, every
// machine composed with the change, the replays — and the repository's own merge-check.sh beside it;
// - touches none, in a repository that is the mesh's (it sources a module on some branch, or shares the
// core's owner): the gate is a pass that says so — a fact, not a missing check — and the repository's
// own merge-check.sh runs as `mesh/repo-check`, a warning when it has none;
// - touches none, anywhere else: the gate is a pass that says so, and nothing more is said.
//
// What is checked is decided here and run there (internal/builder/check.go).
// checkTimeout is how long one check may run on the build seat. Said here so the ask's watchdog (S6)
// and the builder agree on what late means.
const checkTimeout = 45 * time.Minute
// noModuleTouched is the gate's word for a change that touches nothing of the graph.
const noModuleTouched = "the change touches no module of the mesh's graph"
// noMergeCheck is the repository layer's word for a repository of the mesh with no merge-check.sh.
const noMergeCheck = "the repository declares no merge-check.sh: none of its own tests run before it merges"
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
// each beside it — and whose owner is the mesh's own.
var coreModules = map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host",
"node-tools": "mesh-tools", "nats": "mesh-catalog"}
// checkScope is what a pull request reaches of the mesh's graph, as the planner reckons it.
type checkScope struct {
// Modules are the modules a merge of the change would move itself — built from the repository into
// the pull request's base and reading a changed file, or packaging the repository's source — and
// Dependents those the plan would build after them; New the directories it adds a module in.
Modules []string
Dependents []string
New []string
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
Manifests []string
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
// module's build reads.
Width, Tiers int
Unread []string
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
// core's, or the change adds a module to it.
Mesh bool
// Judge is who judges the gate (link.JudgeSelf, link.JudgeValidator, or the running controller).
Judge string
// From is a module built from the repository, for how the mesh clones it; nil when none is.
From *inventory.Entry
// Reach is the planner's whole answer, which the change plan is made from.
Reach mergeReach
}
func (s checkScope) gated() bool { return len(s.Modules)+len(s.New) > 0 }
// pullScope is what a pull request reaches: **the planner's own answer** (reachOfMerge), asked as if the
// head were merged into the base — never a mapping of its own, so a change to what a merge touches
// changes what is checked with it (novox/hq ADR 0238).
func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
edges []inventory.Edge) checkScope {
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL, Commit: p.Commit,
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
ModuleDirsSaid: p.ModuleDirsSaid}
r := reachOfMerge(m, entries, read, edges)
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
switch e.Manifest.Module {
case "mesh-controller":
s.Judge = link.JudgeSelf
case "mesh-host":
if s.Judge == "" {
s.Judge = link.JudgeValidator
}
}
}
for _, d := range r.Added {
s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile))
}
sort.Strings(s.Manifests)
s.Manifests = slices.Compact(s.Manifests)
// Whose repository it is, for how it is cloned and whether its own check is the mesh's to run.
owners := map[string]bool{}
for i, e := range entries {
if e.Provided {
continue
}
if _, core := coreModules[e.Manifest.Module]; core {
if owner := sourceOwner(e.Source.Repository); owner != "" {
owners[owner] = true
}
}
if sameRepository(e.Source.Repository, m) && s.From == nil {
s.From = &entries[i]
}
}
s.Mesh = s.From != nil || owners[strings.ToLower(p.Owner)] || s.gated()
return s
}
// sourceOwner is the owner of a recorded repository, a path on the git seat or a URL: novox/mesh-host → novox.
func sourceOwner(repository string) string {
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
if len(parts) < 2 {
return ""
}
return strings.ToLower(parts[len(parts)-2])
}
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
inv := f.open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return err
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return err
}
scope := pullScope(p, entries, read, edges)
// The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted
// with the verdict whatever the verdict is.
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
u, err := inv.UpgradeOf(ctx, module)
return u, err == nil
})
fmt.Print(planText(plan))
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
switch {
case !scope.gated() && !scope.Mesh:
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
sayChecked(ctx, direct)
return nil
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
sayChecked(ctx, direct)
return nil
}
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
if err != nil {
return err
}
request.Check.Plan = &plan
seat := buildSeatHeld(ctx)
ask, err := askOverOn(seat)
if err != nil {
return err
}
defer ask.Close()
if err := ask.Ask(ctx, request); err != nil {
return err
}
what := "its own merge-check.sh alone: " + noModuleTouched
if scope.gated() {
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
scope.Width, scope.Tiers)
}
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
p.Commit, seat, what, request.ID)
return nil
}
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
// and what is read beside it.
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
entries []inventory.Entry) (link.BuildRequest, error) {
shelf := map[string]catalogue.Manifest{}
for _, e := range entries {
shelf[e.Manifest.Module] = e.Manifest
}
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
if err != nil {
return link.BuildRequest{}, err
}
clone := func(s inventory.Source) (string, error) {
if s.Seat == "" {
return s.Repository, nil
}
return clonedFromSeat(world, s.Seat, s.Repository)
}
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
source := inventory.Source{Seat: gitSeat, Repository: p.Owner + "/" + p.Repo}
if scope.From != nil {
source = scope.From.Source
}
repository, err := clone(source)
if err != nil {
return link.BuildRequest{}, err
}
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return link.BuildRequest{}, err
}
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
beside := map[string]link.CheckedOut{}
for _, e := range entries {
dir, core := coreModules[e.Manifest.Module]
if !core || e.Provided || e.Source.Repository == "" {
continue
}
url, err := clone(e.Source)
if err != nil {
return link.BuildRequest{}, err
}
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
if dir == "mesh-controller" {
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
if e.Source.Seat != "" {
if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
"mesh-lab")}); err == nil {
beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]}
}
}
}
}
return link.BuildRequest{
ID: link.NewBuildID(time.Now()),
Repository: repository,
Ref: p.Commit,
Held: heldBy(ctx),
Seats: seatBases(ctx),
Source: sourceOnSeat(source),
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
Manifests: scope.Manifests, Judge: scope.Judge},
}, nil
}
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
// builds from; and beside the controller its main, for a judge the running controller predates, and the
// lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by
// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones.
func besideRefs(dir, running string) map[string]string {
switch dir {
case "mesh-catalog":
return map[string]string{dir: "main"}
case "mesh-controller":
return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"}
}
return map[string]string{dir: running}
}
// siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab.
func siblingOf(repository, name string) string {
if cut := strings.LastIndex(repository, "/"); cut >= 0 {
return repository[:cut+1] + name
}
return name
}
// sourceOnSeat is a source's seat form, nil for one on no seat.
func sourceOnSeat(s inventory.Source) *link.SourceOnSeat {
if s.Seat == "" {
return nil
}
return &link.SourceOnSeat{Seat: s.Seat, Repository: s.Repository}
}
// checkEvents is where the serving controller says a check's verdict; nil in a command.
var checkEvents link.Bus
// maxCheckReport is how much of a check's report travels in its verdict: enough for the failures and
// the machines, never a log.
const maxCheckReport = 60 << 10
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
func checked(ctx context.Context, result link.BuildResult) {
sayChecked(ctx, checkedOf(result))
}
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
// could not run.
func checkedOf(result link.BuildResult) link.Checked {
c := link.Checked{ID: result.ID, On: result.On, Commit: result.Ref}
if result.Checked != nil {
c.Owner, c.Repo, c.Number = result.Checked.Owner, result.Checked.Repo, result.Checked.Number
}
switch {
case result.Check != nil:
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
if c.Gate == nil {
// A build seat from before the layers: its verdict is the gate's.
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
}
case result.Failed != "":
// The check could not run: an error, never read as a pass — on both layers it was asked for.
c.Verdict, c.Summary = "error", "the check could not run: "+firstLine(result.Failed)
default:
c.Verdict, c.Summary = "error", "the build seat answered the check with no verdict"
}
if c.Verdict == "" {
c.Verdict = "error"
}
if result.Check == nil {
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
}
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
}
if result.Checked != nil && c.Gate != nil && len(c.Gate.Dependents) == 0 {
c.Gate.Dependents = result.Checked.Dependents
}
if result.Checked != nil {
c.Plan = result.Checked.Plan
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
if g := result.Checked; g.Group != "" {
c.Group = g.Group
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
Commit: result.Ref})
for _, m := range g.Members {
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
Commit: m.Ref})
}
}
}
for _, l := range []*link.CheckLayer{c.Gate, c.RepoCheck} {
if l != nil && l.Verdict == "" {
l.Verdict = "error"
}
}
if len(c.Report) > maxCheckReport {
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
}
return c
}
func prefixedAll(prefix string, items []string) []string {
out := make([]string, 0, len(items))
for _, i := range items {
out = append(out, prefix+i)
}
return out
}
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
func sayChecked(ctx context.Context, c link.Checked) {
repo := "none"
if c.RepoCheck != nil {
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
}
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
if checkEvents == nil {
return
}
body, err := json.Marshal(c)
if err != nil {
return
}
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer stop()
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
}
}
cmd/mesh-controller/checks_test.go
-212
View File
@@ -1,212 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **The mesh's module graph decides what a pull request's check runs**, not the repository (novox/hq
// ADR 0237 as amended): a change is mapped onto the graph by the rule a merge is (issue 278), the gate
// runs when it touches a module — a new one included — and a repository that is the mesh's and touches
// none still has its own merge-check.sh run.
func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) {
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
const controller = "http://forge.internal:20000/novox/mesh-controller.git"
const host = "http://forge.internal:20000/novox/mesh-host.git"
photos := fromRepo("photos", "http://forge.internal:20000/novox/photos.git", "")
photos.Source.Ref = "nox-mesh"
snake := fromRepo("snake", "jschoubben/snake", "")
snake.Source.Seat = "git"
entries := []inventory.Entry{
fromRepo("gitea", catalogue, "modules/gitea"),
fromRepo("keycloak", catalogue, "modules/keycloak"),
fromRepo("nats", catalogue, "modules/nats"),
fromRepo("mesh-controller", controller, ""),
fromRepo("mesh-host", host, ""),
photos, snake,
}
pull := func(owner, repo, base string, paths []string, dirs ...string) link.PullUpdated {
return link.PullUpdated{Owner: owner, Repo: repo, Base: base, Commit: "abc", Paths: paths,
ModuleDirs: dirs, ModuleDirsSaid: true}
}
for _, c := range []struct {
what string
p link.PullUpdated
modules, new, manifest string
mesh bool
judge string
}{
{"one module's own files", pull("novox", "mesh-catalog", "main", []string{"modules/gitea/index.ts"}, "modules/gitea"),
"gitea", "", "modules/gitea/module.json", true, ""},
{"a file no module's build reads touches no module (issue 280)",
pull("novox", "mesh-catalog", "main", []string{"merge-check.sh", "README.md"}), "", "", "", true, ""},
{"files the announcer could not list: everything built from it",
link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", PathsTruncated: true,
Paths: []string{"README.md"}}, "gitea,keycloak,nats", "",
"modules/gitea/module.json,modules/keycloak/module.json,modules/nats/module.json", true, ""},
{"a new module, said by the head", pull("novox", "mesh-catalog", "main",
[]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, "modules/newmod"),
"", "modules/newmod", "modules/newmod/module.json", true, ""},
{"the controller judges itself", pull("novox", "mesh-controller", "main", []string{"cmd/x.go"}),
"mesh-controller", "", "module.json", true, "self"},
{"the node-engine is judged with its validator", pull("novox", "mesh-host", "main", []string{"validate/v.go"}),
"mesh-host", "", "module.json", true, "validator"},
{"the core's owner, no module: the mesh's, its own check alone", pull("novox", "hq", "main", []string{"README.md"}),
"", "", "", true, ""},
{"a branch nothing is built from: the mesh's repository, no module", pull("novox", "photos", "master",
[]string{"server/x.js"}), "", "", "", true, ""},
{"the branch a module is built from", pull("novox", "photos", "nox-mesh", []string{"server/x.js"}),
"photos", "", "module.json", true, ""},
{"a repository on the forge's seat", pull("jschoubben", "snake", "main", []string{"index.html"}),
"snake", "", "module.json", true, ""},
{"a repository of nobody's, touching nothing", pull("someone", "dotfiles", "main", []string{"x"}),
"", "", "", false, ""},
{"a repository adding a module at its root", pull("someone", "newapp", "main", []string{"module.json", "x.js"}),
"", ".", "module.json", true, ""},
} {
s := pullScope(c.p, entries, nil, nil)
got := []string{strings.Join(s.Modules, ","), strings.Join(s.New, ","), strings.Join(s.Manifests, ",")}
want := []string{c.modules, c.new, c.manifest}
for i, what := range []string{"modules", "new", "manifests"} {
if got[i] != want[i] {
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
}
}
if s.Mesh != c.mesh || s.Judge != c.judge {
t.Errorf("%s: the mesh's %v judged by %q, wanted %v by %q", c.what, s.Mesh, s.Judge, c.mesh, c.judge)
}
if s.gated() != (c.modules != "" || c.new != "") {
t.Errorf("%s: gated %v", c.what, s.gated())
}
}
}
// A module whose build packages another repository's source is touched by a change to it.
func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
entries := []inventory.Entry{fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")}
read := map[string][]inventory.ReadRepository{"node-tools": {{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}}
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "abc", Paths: []string{"go/x.go"}}, entries, read, nil)
if strings.Join(s.Modules, ",") != "node-tools" || len(s.Manifests) != 0 {
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
}
}
// Each layer is said; a check that could not run is an error on both, never a pass; a build seat from
// before the layers is read as the gate.
func TestAChecksLayersAreEachSaidAndAnErrorIsNeverAPass(t *testing.T) {
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3, Modules: []string{"gitea"}}
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
if c.Verdict != "error" || c.Gate == nil || c.Gate.Verdict != "error" || c.RepoCheck == nil || c.RepoCheck.Verdict != "error" {
t.Fatalf("a check that could not run said %+v", c)
}
if strings.Join(c.Gate.Modules, ",") != "gitea" {
t.Errorf("the gate names %v", c.Gate.Modules)
}
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "warning", Summary: "wide"}})
if c.Gate == nil || c.Gate.Verdict != "warning" || c.RepoCheck != nil {
t.Fatalf("an outcome without layers said %+v", c)
}
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "pass",
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "its merge-check.sh failed"}}})
if c.RepoCheck.Verdict != "fail" || c.Gate.Verdict != "pass" {
t.Fatalf("the layers said %+v / %+v", c.Gate, c.RepoCheck)
}
}
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
// module in both, and what stands on a touched module is named as its dependents in both.
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
nats := fromRepo("nats", catalogue, "modules/nats")
nats.Source.BuiltFrom = "old"
gitea := fromRepo("gitea", catalogue, "modules/gitea")
gitea.Source.BuiltFrom = "old"
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
entries := []inventory.Entry{nats, gitea, tools}
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
read := map[string][]inventory.ReadRepository{}
for _, c := range []struct {
what string
paths []string
moved, dependents string
width int
unread string
}{
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
} {
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
s := pullScope(p, entries, read, edges)
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
r := reachOfMerge(m, entries, read, edges)
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
}
if got := strings.Join(s.Dependents, ","); got != c.dependents {
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
}
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
}
if got := strings.Join(s.Unread, ","); got != c.unread {
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
}
}
// A repository whose build another module's recipe packages: that module is reached through the
// planner's `read`, and what stands on it after it.
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
Paths: []string{"src/index.ts"}}, entries, read, edges)
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
}
}
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a build of a commit off its repository's
// default branch — a pull request's head, a branch built by hand, a rebuild or replay of one — is recorded
// and never registered, so nothing can send it; a check or a dry run never is either.
func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
on := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main", OnTrunk: true}
if err := publishable(on, ""); err != nil {
t.Errorf("a build on the trunk was refused: %v", err)
}
off := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main"}
if err := publishable(off, ""); !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "main") {
t.Errorf("a build off the trunk was let through: %v", err)
}
for _, r := range []link.BuildResult{
{ID: "c", Trunk: "main", OnTrunk: true, Check: &link.CheckOutcome{Verdict: "pass"}},
{ID: "d", Trunk: "main", OnTrunk: true, Checked: &link.CheckRequest{}},
{ID: "e", Trunk: "main", OnTrunk: true, DryRun: true},
} {
if publishable(r, "") == nil {
t.Errorf("%s, a check or a dry run, was publishable", r.ID)
}
}
// A module that follows a branch other than the default: that branch is its trunk, and the default
// is not.
follows := link.BuildResult{ID: "g", Commit: "abc", Trunk: "master", Branches: []string{"nox-mesh"}}
if err := publishable(follows, "nox-mesh"); err != nil {
t.Errorf("a commit on the branch a module follows was refused: %v", err)
}
if err := publishable(link.BuildResult{ID: "h", Commit: "abc", Trunk: "master", OnTrunk: true,
Branches: []string{"master"}}, "nox-mesh"); err == nil {
t.Error("a commit on the default but not on the branch the module follows was published")
}
if err := publishable(link.BuildResult{ID: "i", Commit: "abc", Trunk: "main", Branches: []string{"feat/x"}}, ""); err == nil {
t.Error("a feature branch's commit was published")
}
// A build seat older than the rule says nothing: let through and said, so the rule can reach the mesh.
if err := publishable(link.BuildResult{ID: "f", Commit: "abc"}, ""); err != nil {
t.Errorf("a build seat that said nothing was refused: %v", err)
}
}
cmd/mesh-controller/conditions.go
-441
View File
@@ -1,441 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"os"
"slices"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
//
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
// while, with a reason, and that is recorded as a hand act.
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
var conditionsFrom *conditions.Keeper
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
store, history, err := conditions.OnTheBus(ctx, conn)
if err != nil {
return nil, err
}
js, err := conn.JetStream()
if err != nil {
return nil, err
}
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
Teller: link.OverNATS{Conn: conn, JS: js},
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
// What status leads with changed: composed again soon (a nudge outside the serving controller
// does nothing).
Changed: statusFrom.nudge,
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
}
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
// it was given before it returns.
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
if conditionsFrom != nil {
return f(conditionsFrom)
}
return onTheBus(func(conn *nats.Conn) error {
k, err := keeperOn(ctx, conn)
if err != nil {
return err
}
defer func() {
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
k.Close(flushing)
}()
return f(k)
})
}
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
if conditionsFrom != nil {
return conditionsFrom.Open(ctx)
}
if _, err := broker.BusAddress(); err != nil {
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
}
var out []conditions.Condition
err := onTheBus(func(conn *nats.Conn) error {
store, _, err := conditions.OnTheBus(ctx, conn)
if err != nil {
return err
}
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
out, err = conditions.Read(reading, store)
return err
})
return out, err
}
// conditionsUsage is how the verb is typed.
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
"conditions history [--days N] [--key K] [--json]"
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
func conditionsCommand(ctx context.Context, args []string) error {
sub := "list"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
switch sub {
case "list":
return listConditions(ctx, args)
case "show":
return showCondition(ctx, args)
case "silence":
return silenceCondition(ctx, args)
case "history":
return conditionHistory(ctx, args)
}
return errors.New(conditionsUsage)
}
func listConditions(ctx context.Context, args []string) error {
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
severity := set.String("severity", "", "only urgent, or only warning")
machine := set.String("machine", "", "only those about this machine")
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(conditionsUsage)
}
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
}
open, err := openConditions(ctx)
if err != nil {
return err
}
var out []conditions.Condition
for _, c := range open {
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
(*machine == "" || concerns(c, *machine)) {
out = append(out, c)
}
}
if *asJSON {
if out == nil {
out = []conditions.Condition{}
}
return printJSON(map[string]any{"conditions": out, "open": len(open),
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
}
if len(out) == 0 {
if len(open) == 0 {
fmt.Println("no open conditions")
} else {
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
}
return nil
}
for _, line := range conditionLines(out, time.Now()) {
fmt.Println(line)
}
return nil
}
// concerns says whether a condition is about a machine: it names it, or its key does.
func concerns(c conditions.Condition, machine string) bool {
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
return true
}
for _, part := range strings.Split(c.Key, ".") {
if part == machine {
return true
}
}
return false
}
// conditionLines is how a list of conditions reads: one line each, its silence under it.
func conditionLines(list []conditions.Condition, now time.Time) []string {
var out []string
for _, c := range list {
times := ""
if c.Count > 1 {
times = fmt.Sprintf(", raised %d times", c.Count)
}
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
if c.SilencedAt(now) {
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
}
}
return out
}
func showCondition(ctx context.Context, args []string) error {
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
rest, err := parseAround(set, args)
if err != nil {
return err
}
if len(rest) != 1 {
return errors.New("conditions show <key>")
}
key := rest[0]
var c conditions.Condition
var found bool
if conditionsFrom != nil {
c, found, err = conditionsFrom.Get(ctx, key)
} else {
err = onTheBus(func(conn *nats.Conn) error {
store, _, err := conditions.OnTheBus(ctx, conn)
if err != nil {
return err
}
c, found, err = conditions.ReadOne(ctx, store, key)
return err
})
}
if err != nil {
return err
}
if !found {
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
"history --key %s` what became of it", key, key)
}
if *asJSON {
return printJSON(c)
}
now := time.Now()
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
if c.Subject.Machine != "" {
fmt.Printf(", on %s", c.Subject.Machine)
}
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
now.Sub(c.LastObserved).Round(time.Second))
if c.Count > 1 {
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
}
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
if c.Silenced != nil {
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
c.Silenced.By, c.Silenced.Why)
}
if len(c.Tried) > 0 {
fmt.Println("\n tried:")
for _, t := range c.Tried {
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
}
}
fmt.Println("\n evidence, newest first:")
for _, e := range c.Evidence {
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
}
return nil
}
func resolverWords(r string) string {
switch r {
case conditions.ResolverSelf:
return "itself: it clears when observation says it is resolved"
case conditions.ResolverOperator:
return "the operator: nothing in the mesh will repair it"
case conditions.ResolverAgent:
return "an agent"
}
return r
}
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
// who and why before it is done, its cause the condition's kind unless one is given.
func silenceCondition(ctx context.Context, args []string) error {
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
acts := addHandActFlags(set)
rest, err := parseAround(set, args)
if err != nil {
return err
}
if len(rest) != 1 {
return errors.New("conditions silence <key> --for <duration> --why <text>")
}
key := rest[0]
if err := acts.require("conditions silence"); err != nil {
return err
}
d, err := parseFor(*forFlag)
if err != nil {
return err
}
if d > conditions.MaxSilence {
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
}
return withKeeper(ctx, func(k *conditions.Keeper) error {
c, found, err := k.Get(ctx, key)
if err != nil {
return err
}
if !found {
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
}
if strings.TrimSpace(*acts.cause) == "" {
*acts.cause = c.Kind
}
*acts.condition = key
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
if err != nil {
return err
}
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
"`status` still says it; it clears when observation says it is resolved\n",
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
return nil
})
}
// parseFor reads a duration, days included.
func parseFor(s string) (time.Duration, error) {
s = strings.TrimSpace(s)
if s == "" {
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
}
if days, ok := strings.CutSuffix(s, "d"); ok {
n, err := strconv.Atoi(days)
if err != nil || n <= 0 {
return 0, fmt.Errorf("%q is not a number of days", s)
}
return time.Duration(n) * 24 * time.Hour, nil
}
d, err := time.ParseDuration(s)
if err != nil || d <= 0 {
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
}
return d, nil
}
func conditionHistory(ctx context.Context, args []string) error {
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
days := set.Int("days", 7, "how many days back, at most 90")
key := set.String("key", "", "only this condition")
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("conditions history [--days N] [--key K] [--json]")
}
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
var events []conditions.Event
read := func(h conditions.History) error {
var err error
events, err = h.Since(ctx, since)
return err
}
var err error
if conditionsFrom != nil {
events, err = conditionsFrom.HistorySince(ctx, since)
} else {
err = onTheBus(func(conn *nats.Conn) error {
_, history, err := conditions.OnTheBus(ctx, conn)
if err != nil {
return err
}
return read(history)
})
}
if err != nil {
return err
}
var out []conditions.Event
for _, e := range events {
if *key == "" || e.Key == *key {
out = append(out, e)
}
}
if *asJSON {
if out == nil {
out = []conditions.Event{}
}
return printJSON(map[string]any{"history": out, "days": *days})
}
if len(out) == 0 {
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
return nil
}
for _, e := range out {
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
switch e.Change {
case conditions.ChangeRaised, conditions.ChangeReopened:
line += " — " + e.Summary
case conditions.ChangeSeverity:
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
case conditions.ChangeResolver:
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
default:
if e.Why != "" {
line += " — " + e.Why
}
}
fmt.Println(line)
}
return nil
}
// printConditions is the status section that leads it: every open condition, urgent first, oldest
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
// is not "none open".
func printConditions(list []conditions.Condition, unread string, now time.Time) {
if unread != "" {
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
return
}
if len(list) == 0 {
return
}
urgent := 0
for _, c := range list {
if c.Severity == conditions.Urgent {
urgent++
}
}
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
for _, line := range conditionLines(list, now) {
fmt.Println(line)
}
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
}
// orHealer is who tried, as an attempt names it.
func orHealer(by string) string {
if by == "" {
return "a healer"
}
return by
}
cmd/mesh-controller/conditions_test.go
-107
View File
@@ -1,107 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/conditions"
)
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
// by what is open.
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
for _, c := range []struct {
args map[string]any
want string
}{
{map[string]any{}, "conditions --json"},
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
"conditions history --json --days 3 --key machine.ace.silent"},
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
"conditions silence machine.ace.silent --for 2h --why on the train"},
{map[string]any{"run": "true"}, "doctor run --json"},
{map[string]any{}, "doctor --json"},
} {
verb := "conditions"
if strings.HasPrefix(c.want, "doctor") {
verb = "doctor"
}
argv, err := argvFor(verb, c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
}
}
for _, c := range []struct {
verb string
args map[string]any
}{
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
} {
if argv, err := argvFor(c.verb, c.args); err == nil {
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
}
}
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
t.Error("a silence is not a hand act")
}
}
// **A silence through the verb is recorded and bounded**; the condition stays open.
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
k, _ := withConditionsInMemory(t)
ctx := t.Context()
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
t.Fatal(err)
}
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
t.Fatal("silenced without saying why")
}
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
t.Fatal("silenced for more than a week")
}
said := printed(t, func() error {
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
})
if !strings.Contains(said, "is silenced until") {
t.Fatalf("%s", said)
}
c, found, _ := k.Get(ctx, "machine.ace.silent")
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
t.Fatalf("%+v", c)
}
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
t.Fatalf("%s", listed)
}
}
// **Conditions that cannot be read are not none open**: status says so, and is not well.
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
open := aMesh(t)
_, store := withConditionsInMemory(t)
store.Fail = errors.New("the bus is away")
asked, err := theThreeQuestions(t.Context(), open)
if err != nil {
t.Fatal(err)
}
if asked.well() || asked.conditionsUnread == "" {
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
}
said := printed(t, func() error { return printStatus(asked) })
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
t.Fatalf("%s", said)
}
store.Fail = nil
asked, _ = theThreeQuestions(t.Context(), open)
said = printed(t, func() error { return printStatus(asked) })
if asked.well() && !strings.Contains(said, "no open conditions;") {
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
}
}
cmd/mesh-controller/confirm.go
-67
View File
@@ -1,67 +0,0 @@
package main
import (
"context"
"sync"
"github.com/novox/mesh-controller/internal/conditions"
)
// **A finding one look can be wrong about is raised on the second look in a row** (novox/hq issue 277).
//
// D2 raised its resolver urgent on one unanswered question, asked once, while the resolver's machine
// was loaded by a push and a build starting; thirty questions right after were answered at once. A
// single sample of something that is answered over the network, or timed on a machine under load,
// is not the invariant failing. So a source marks such a finding `Confirm`, and this holds it back:
//
// - raised when the source's previous look saw it too — two runs of the self-check in a row (five
// minutes apart), or two ticks of the watchdogs (half a minute) — at the severity the source says;
// - kept while it is already open, however it is seen, so a condition the next look still sees is
// never cleared and raised again (flapping is not news; a reopening within ReopenWithin still is);
// - cleared, as everything is, by the look that no longer sees it.
//
// A finding held back is not a pass: the self-check's verdict names it as unconfirmed. A finding that is
// a definite answer — a resolver that answered wrongly, a stream that is not there — is not marked, and
// is raised at once.
type confirming struct {
mu sync.Mutex
// last is, by source, the keys of the Confirm findings its previous look saw.
last map[string]map[string]bool
}
// pass splits one source's findings into what is raised now and what is held for the next look, and
// remembers what it saw. An open condition that cannot be read is kept: unknown is not a reason to
// hold a finding back.
func (c *confirming) pass(ctx context.Context, keeper *conditions.Keeper, source string,
found []conditions.Observation) (raise, held []conditions.Observation) {
c.mu.Lock()
defer c.mu.Unlock()
if c.last == nil {
c.last = map[string]map[string]bool{}
}
before, seen := c.last[source], map[string]bool{}
for _, o := range found {
if !o.Confirm {
raise = append(raise, o)
continue
}
key := o.Key()
seen[key] = true
if before[key] || isOpen(ctx, keeper, key) {
raise = append(raise, o)
continue
}
held = append(held, o)
}
c.last[source] = seen
return raise, held
}
// isOpen says whether a condition is open; one that cannot be read is taken as open.
func isOpen(ctx context.Context, keeper *conditions.Keeper, key string) bool {
if keeper == nil {
return false
}
_, open, err := keeper.Get(ctx, key)
return open || err != nil
}
cmd/mesh-controller/data.go
-931
View File
@@ -1,931 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"log"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A module declares the data it holds, and the mesh protects and watches it from that declaration
// (novox/hq ADR 0233).
//
// The self-check's D13 composes what every machine declares, asks each machine's backup holder what
// it measured of every item — size, newest write, newest good backup, the redundant storage it is on —
// and keeps both. From that, and from what every provider says it holds for its consumers, it raises,
// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking):
//
// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least
// shrinkFloor less; `data-missing`: its path is gone;
// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a
// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on
// empty databases while their real ones sat on another machine (issue 273);
// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot
// be compared;
// - `data-quiet`: an item said to be written all the time has not been, within its bound;
// - `backup-stale`: an item's newest good backup is older than its bound, or there is none;
// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read;
// `protection-missing`: an item said to be protected by redundancy is on storage that is not;
// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person.
//
// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no
// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by
// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with
// anything in it; this is the record of what it kept. An operator's path is never retired or deleted.
// The condition kinds of D13.
const (
kindDataShrank = "data-shrank"
kindEmptyReplacement = "empty-replacement"
kindDataHeldTwice = "data-held-twice"
kindDataQuiet = "data-quiet"
kindBackupStale = "backup-stale"
kindDataUnmeasured = "data-unmeasured"
// kindDataMissing is a watched item whose path is gone.
kindDataMissing = "data-missing"
// kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read.
kindArrayDegraded = "array-degraded"
// kindProtectionMissing is an item said to be protected by redundancy, on storage that is not.
kindProtectionMissing = "protection-missing"
)
// probeDataID is the self-check's id for this probe.
const probeDataID = "D13"
// The bounds the findings are read against.
var (
// shrinkWindow is how far back the largest size is looked for.
shrinkWindow = 7 * 24 * time.Hour
// shrinkFloor is the least loss that is worth saying: two empty databases differ by a few
// megabytes, and half of almost nothing is noise.
shrinkFloor int64 = 16 << 20
// dataAsk is how long one machine's holder, or one provider, is given to answer.
dataAsk = 8 * time.Second
)
// keyOfItem is one item's condition id: its machine, module and item.
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data).
type holderAnswer struct {
Module string `json:"module"`
Data []holderItem `json:"data"`
}
// holderItem is one item as the holder measured it.
type holderItem struct {
Item string `json:"item"`
Class string `json:"class"`
Path string `json:"path"`
SizeBytes *int64 `json:"size_bytes"`
LastWrite *time.Time `json:"last_write"`
MeasuredAt *time.Time `json:"measured_at"`
LastBackup *time.Time `json:"last_backup"`
Error string `json:"error,omitempty"`
// Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233).
Precision string `json:"precision,omitempty"`
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
}
// readHolder reads a holder's answer into measurements by module and item.
func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) {
var modules []holderAnswer
if err := json.Unmarshal(raw, &modules); err != nil {
return nil, fmt.Errorf("its answer is not readable: %w", err)
}
out := map[string]map[string]inventory.Measurement{}
for _, m := range modules {
for _, it := range m.Data {
if out[m.Module] == nil {
out[m.Module] = map[string]inventory.Measurement{}
}
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy,
Precision: it.Precision}
}
}
return out, nil
}
// declaredOn is every data item a machine's composition declares, and the module there that holds
// node-backup to measure them — empty for none.
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) {
var out []inventory.DeclaredData
held := ""
for _, m := range plan.Modules {
for _, c := range m.Claims {
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
held = m.Module
}
}
for _, it := range m.DataItems() {
out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class,
Owned: it.OwnedByModule(), Protection: it.Protection()})
}
}
return out, held
}
// consumerCopy is one provider's account of one consumer: where, how big, and whether still active.
type consumerCopy struct {
Node, Module, Consumer string
Size *int64
Retired bool
// Class is how precious the consumer's data is: the stricter of what the provider keeps for its
// consumers and what the consumer says it keeps there (`kept-by`).
Class string
}
// probeData is D13.
func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
if d.js == nil {
return nil, errors.New("no bus to ask the machines over")
}
open := d.open
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return nil, err
}
nodes, err := open.inventory.Nodes(ctx)
if err != nil {
return nil, err
}
heard := heardMachines(d)
now := time.Now()
delivered, err := readDeliveries(ctx, open.inventory)
if err != nil {
return nil, err
}
type machine struct {
name string
declared []inventory.DeclaredData
held bool
measured map[string]map[string]inventory.Measurement
askErr error
}
var machines []*machine
for _, n := range nodes {
plan, _, err := planFor(ctx, open, n.Name)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
// A machine that cannot be worked out declares nothing this run — which is not the same as
// declaring nothing: retiring its data on that would be acting on an unreadable result.
continue
}
declared, holder := declaredOn(plan)
// **A holder is asked only once its machine has been sent it and had time to report** (novox/hq
// issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it
// measured" about it was a warning for a push nobody had made yet.
held := holder != "" && delivered[n.Name].settled(holder, now)
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
}
// Every holder asked at once, as D8 asks every ban list.
var wg sync.WaitGroup
for _, m := range machines {
if !m.held || !heard[m.name] {
continue
}
wg.Add(1)
go func(m *machine) {
defer wg.Done()
asking, cancel := context.WithTimeout(ctx, dataAsk)
defer cancel()
raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name)
if err == nil {
m.measured, err = readHolder(raw)
}
m.askErr = err
}(m)
}
wg.Wait()
var out []conditions.Observation
for _, m := range machines {
if m.askErr != nil {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning, Confirm: true,
Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
"nothing about that data is known", m.name),
Said: firstLine(m.askErr.Error())})
}
why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now)
if err != nil {
return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err)
}
for _, r := range change.Retired {
log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+
"delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why)
}
for _, r := range change.Reenabled {
log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine)
}
}
records, err := open.inventory.Data(ctx)
if err != nil {
return nil, err
}
peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow))
if err != nil {
return nil, err
}
bindings, err := open.inventory.Bindings(ctx)
if err != nil {
return nil, err
}
upgraded, keptBy := keptByClasses(bindings, shelf)
copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy)
if err != nil {
return nil, err
}
out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...)
return out, nil
}
func orUnknownPath(p string) string {
if p == "" {
return "a path its backup holder never named"
}
return p
}
// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at
// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding.
func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) {
instances, err := providerInstances(ctx, inv)
if err != nil {
return nil, err
}
var asked []providerInstance
for _, p := range instances {
m := shelf[p.Module]
keeps := false
for provision := range m.Grants {
keeps = keeps || m.KeepsConsumerData(provision)
}
if keeps {
asked = append(asked, p)
}
}
states := make([]*link.RetirementState, len(asked))
var wg sync.WaitGroup
for i, p := range asked {
wg.Add(1)
go func(i int, p providerInstance) {
defer wg.Done()
asking, cancel := context.WithTimeout(ctx, dataAsk)
defer cancel()
if s, err := askRetirement(asking, conn, p); err == nil {
states[i] = &s
}
}(i, p)
}
wg.Wait()
var out []consumerCopy
for i, p := range asked {
s := states[i]
if s == nil {
continue
}
class := consumersClass(shelf[p.Module])
for _, c := range s.Held {
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c,
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])}
if size, ok := s.HeldSizes[c]; ok && size >= 0 {
size := size
cp.Size = &size
}
out = append(out, cp)
}
for _, r := range s.Retired {
if r.Kind != "" && r.Kind != "consumer" {
continue
}
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true,
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])}
if r.SizeBytes != nil && *r.SizeBytes >= 0 {
cp.Size = r.SizeBytes
}
out = append(out, cp)
}
}
return out, nil
}
// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning
// for anything else watched (the operator's ranking, ADR 0233).
func severityOf(class string) conditions.Severity {
if class == catalogue.ClassIrreplaceable {
return conditions.Urgent
}
return conditions.Warning
}
// consumersClass is the most precious class a provider keeps any of its consumers' data as.
func consumersClass(m catalogue.Manifest) string {
class := catalogue.ClassNone
for provision := range m.Grants {
if c, ok := m.ConsumerDataOf(provision); ok {
class = catalogue.StricterClass(class, c.Class)
} else if m.KeepsConsumerData(provision) {
class = catalogue.StricterClass(class, catalogue.ClassValuable)
}
}
return class
}
// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read
// through where each is bound: by provider module and consumer identity, the class of that consumer's
// data there; and by provider item key (machine/module/item), the class the item holding it is held to.
func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) {
upgraded, keptBy := map[string]string{}, map[string]string{}
for _, b := range bindings {
m, ok := shelf[b.Consumer]
if !ok {
continue
}
k, said := m.KeptByOf(b.Provision)
if !said {
continue
}
identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module))
key := b.Provider.Module + "/" + identity
keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class)
if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" {
if _, own := shelf[b.Provider.Module].DataItem(pc.In); own {
item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In
upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class)
}
}
}
return upgraded, keptBy
}
// dataFindings is every condition the data on record raises now. A function of what is known, so the
// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer
// of its module keeps data in it more precious than its own class says (`kept-by`), by its key.
func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest,
copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation {
var out []conditions.Observation
byItem := map[string][]inventory.DataRecord{}
arrays := map[string][]inventory.DataRecord{}
type shrunk struct {
machine, dataset, class string
size, peak int64
items []string
}
shrunkDatasets := map[string]shrunk{}
for _, r := range records {
if r.DeletedAt != nil {
continue
}
class := catalogue.StricterClass(r.Class, upgraded[r.Key()])
r.Class = class
byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r)
item, declared := shelf[r.Module].DataItem(r.Item)
id := keyOfItem(r.Machine, r.Module, r.Item)
if r.Retired() {
if now.Sub(*r.RetiredAt) > cleanupAfter {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module),
Said: "kept at " + orUnknownPath(r.Path)})
}
continue
}
if !catalogue.Watched(class) {
continue
}
severity := severityOf(class)
if r.Redundancy != nil {
where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
arrays[where] = append(arrays[where], r)
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
"and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
r.Item, r.Module, r.Machine, class),
Said: orUnknownPath(r.Path) + " is on no redundant storage the backup holder can read"})
}
if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.Item,
r.Module, r.Machine, class),
Said: orUnknownPath(r.Path) + " does not exist: " + firstLine(r.MeasureError)})
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
// Several items on one dataset share its size: one condition for the dataset, as loud as the
// most precious item on it.
k := r.Machine + "/" + inventory.Dataset(r.Precision)
ds := shrunkDatasets[k]
ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak
ds.class = catalogue.StricterClass(ds.class, class)
ds.items = append(ds.items, r.Module+"/"+r.Item)
shrunkDatasets[k] = ds
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
"good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak),
int(shrinkWindow.Hours()/24))})
}
if !declared {
continue
}
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet,
Kind: kindDataQuiet, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
"whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339),
within)})
}
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
// plan, said only where the machine was measured — where a holder is there to take it.
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
within := item.BackupWithin()
switch {
case r.LastBackup == nil && now.Sub(r.FirstSeen) > within:
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))})
case r.LastBackup != nil && now.Sub(*r.LastBackup) > within:
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
"of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)})
}
}
}
for _, k := range keysSorted(shrunkDatasets) {
ds := shrunkDatasets[k]
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than half of "+
"what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak),
int(shrinkWindow.Hours()/24)),
Said: "the dataset " + ds.dataset})
}
// The redundant storage watched data is on: one condition per array, as loud as the most precious
// item on it — the array, not each item, is what degrades.
for _, where := range keysSorted(arrays) {
rs := arrays[where]
red := rs[0].Redundancy
if red.Healthy != nil && *red.Healthy {
continue
}
class, machine := catalogue.ClassValuable, rs[0].Machine
var names []string
for _, r := range rs {
class = catalogue.StricterClass(class, r.Class)
names = append(names, r.Module+"/"+r.Item)
}
state := "could not be read"
if red.Healthy != nil {
state = "is NOT healthy"
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, strings.Join(names, ", "),
state),
Said: fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said))})
}
// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
// an empty replacement. Only against a retired copy — a module running on two machines on purpose
// keeps two different sets of data.
for _, key := range keysSorted(byItem) {
rs := byItem[key]
for _, a := range rs {
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) {
continue
}
for _, o := range rs {
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) ||
!replacedByLess(*a.Size, *o.Size) {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
"an empty replacement of its data. Move the data, or assign it back where its data is",
a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)})
break
}
}
}
out = append(out, consumerFindings(copies)...)
return out
}
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than
// half of it, and at least shrinkFloor less.
func replacedByLess(inUse, kept int64) bool {
return inUse*2 < kept && kept-inUse >= shrinkFloor
}
// consumerFindings is the same question of consumers' data at providers: one consumer, the same
// provider module on two machines.
func consumerFindings(copies []consumerCopy) []conditions.Observation {
by := map[string][]consumerCopy{}
for _, c := range copies {
k := c.Module + "/" + c.Consumer
by[k] = append(by[k], c)
}
var out []conditions.Observation
for _, k := range keysSorted(by) {
cs := by[k]
if len(cs) < 2 {
continue
}
found := false
for _, a := range cs {
if a.Retired || a.Size == nil {
continue
}
for _, o := range cs {
if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
continue
}
state := "active"
if o.Retired {
state = "retired"
}
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)),
Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+
"consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+
"the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node,
sizeWords(o.Size), o.Node)})
found = true
break
}
if found {
break
}
}
if found {
continue
}
var active []consumerCopy
for _, c := range cs {
if !c.Retired {
active = append(active, c)
}
}
if len(active) >= 2 {
var where []string
var also []string
for _, c := range active {
where = append(where, c.Node+" ("+sizeWords(c.Size)+")")
also = append(also, c.Node)
}
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice,
Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses",
active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))})
}
}
return out
}
func keysSorted[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// ---- the `data` verb ---------------------------------------------------------------------------
const dataUsage = "data [--json] [--machine <name>] [--retired]"
// dataRow is one item as `data` lists it.
type dataRow struct {
Machine string `json:"machine"`
Module string `json:"module"`
Item string `json:"item"`
Class string `json:"class"`
Path string `json:"path,omitempty"`
Protection string `json:"protection,omitempty"`
// Array is the redundant storage it is on and its state, where its holder could tell.
Array string `json:"array,omitempty"`
Unmeasured string `json:"unmeasured,omitempty"`
// Precision says what the size is: exact, a dataset's whole size, partial, or none.
Precision string `json:"precision,omitempty"`
SizeBytes *int64 `json:"size-bytes,omitempty"`
LastWrite string `json:"last-write,omitempty"`
MeasuredAt string `json:"measured-at,omitempty"`
LastBackup string `json:"last-backup,omitempty"`
BackupDue string `json:"backup-within,omitempty"`
Retired string `json:"retired,omitempty"`
RetiredWhy string `json:"retired-why,omitempty"`
Deleted string `json:"deleted,omitempty"`
}
// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last
// found it.
func dataCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("data", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
only := set.String("machine", "", "one machine")
retiredOnly := set.Bool("retired", false, "only what is retired")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(dataUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
records, err := open.inventory.Data(ctx)
if err != nil {
return err
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return err
}
rows := dataRows(records, shelf, *only, *retiredOnly)
if *asJSON {
return printJSON(map[string]any{"data": rows})
}
if len(rows) == 0 {
fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run")
return nil
}
for _, r := range rows {
state := ""
switch {
case r.Deleted != "":
state = " DELETED " + r.Deleted
case r.Retired != "":
state = " RETIRED " + r.Retired + " — " + r.RetiredWhy
}
fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class,
sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state)
if r.Array != "" {
fmt.Printf(" on %s\n", r.Array)
}
if r.Precision != "" && r.Precision != "exact" {
fmt.Printf(" size: %s\n", r.Precision)
}
if r.Unmeasured != "" {
fmt.Printf(" not measured: %s\n", r.Unmeasured)
}
if r.Class == catalogue.ClassCache {
continue
}
fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt),
orNever(r.LastBackup), within(r.BackupDue))
}
return nil
}
func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow {
rows := []dataRow{}
stamp := func(t *time.Time) string {
if t == nil {
return ""
}
return t.UTC().Format(time.RFC3339)
}
for _, r := range records {
if only != "" && r.Machine != only {
continue
}
if retiredOnly && !r.Retired() {
continue
}
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
Deleted: stamp(r.DeletedAt)}
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
row.BackupDue = it.BackupWithin().String()
}
if red := r.Redundancy; red != nil {
state := "state unread"
if red.Healthy != nil && *red.Healthy {
state = "healthy"
} else if red.Healthy != nil {
state = "NOT HEALTHY"
}
row.Array = red.Kind + " " + red.Where + ", " + state
}
rows = append(rows, row)
}
return rows
}
func orNothingWord(s string) string {
if s == "" || s == "none" {
return "nothing"
}
return s
}
func orNever(s string) string {
if s == "" {
return "never"
}
return s
}
func within(s string) string {
if s == "" {
return " (not backed up)"
}
return " (bound " + s + ")"
}
// ---- cleanup of retired own data ---------------------------------------------------------------
// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first
// takes a last restore point of it, tagged as retired, and only then removes it — in the background,
// because a large item outlasts any call — and the second says how that went. Module tools, not seat
// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete.
const (
ToolDeleteRetired = "backup_delete_retired"
ToolDeletedOutcome = "backup_deleted"
)
// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person.
var deletionWait = 8 * time.Minute
// deletionPoll is how often it asks.
var deletionPoll = 5 * time.Second
// deletion is a holder's account of one deletion.
type deletion struct {
Started bool `json:"started"`
Running bool `json:"running"`
Done bool `json:"done"`
OK bool `json:"ok"`
Snapshot string `json:"snapshot"`
Error string `json:"error"`
}
// retiredData is every retired item on record, as `cleanup list` shows them.
func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow {
var out []retiredRow
for _, r := range records {
if !r.Retired() {
continue
}
out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind,
RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24),
SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class})
}
return out
}
// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer.
const retiredDataKind = "own-data"
// holderOn is the module holding node-backup on a machine.
func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) {
held, err := inv.Holdings(ctx)
if err != nil {
return "", err
}
for _, h := range held {
if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine {
return h.Module, nil
}
}
return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+
"(after a last restore point)", catalogue.BackupSeat, machine)
}
// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and
// never one not retired. The holder takes a last restore point of it first, so the deletion can be
// undone until a person forgets that restore point; the record says deleted only once the holder says
// it is.
func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error {
inv := open.inventory
if !r.Retired() {
return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done",
r.Item, r.Module, r.Machine)
}
if r.Path == "" {
return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted",
r.Item, r.Module, r.Machine)
}
if plan, _, err := planFor(ctx, open, r.Machine); err == nil {
for _, m := range plan.Modules {
if _, still := m.DataItem(r.Item); still && m.Module == r.Module {
return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done",
r.Module, r.Machine, r.Item)
}
}
}
holder, err := holderOn(ctx, inv, r.Machine)
if err != nil {
return err
}
f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item})
args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item,
"why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController}
ask := func(tool string) (deletion, error) {
var d deletion
answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second)
if err != nil {
return d, err
}
if answer.Error != "" {
return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error)
}
return d, unmarshalAnswer(answer, &d)
}
d, err := ask(ToolDeleteRetired)
if err != nil {
return err
}
for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(deletionPoll):
}
if d, err = ask(ToolDeletedOutcome); err != nil {
return err
}
}
switch {
case !d.Done:
fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+
"showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n",
holder, r.Machine, r.Path)
return nil
case !d.OK:
return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error)
}
if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil {
return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err)
}
fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n",
holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot))
return nil
}
// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its
// own directories on the machine:
// kept, and retired at the self-check's next run.
func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string {
records, err := inv.Data(ctx)
if err != nil {
return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()}
}
var out []string
for _, r := range records {
if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned {
continue
}
for _, m := range modules {
if r.Module == m {
out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+
"never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)",
r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class))
}
}
}
return out
}
cmd/mesh-controller/data_test.go
-465
View File
@@ -1,465 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/testbus"
)
func bytesOf(n int64) *int64 { return &n }
func when(t time.Time) *time.Time { return &t }
func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
t.Helper()
out := map[string]catalogue.Manifest{}
for _, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatal(err)
}
out[m.Module] = m
}
return out
}
const houseManifest = `{"module":"house","version":"1",
"data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]},
"resources":[{"id":"config","type":"directory","mode":"0700"}]}`
func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation {
out := map[string]conditions.Observation{}
for _, o := range linted(obs) {
out[o.Kind] = o
}
return out
}
// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound,
// by one changed rule, to the store on the control node, which made each an empty database — while
// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired
// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines
// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says
// its data there is irreplaceable (`kept-by`).
func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) {
var copies []consumerCopy
for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} {
copies = append(copies,
consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"},
consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"})
}
copies[1].Class = "irreplaceable" // the photo site's own database says so
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
if len(got) != 5 {
t.Fatalf("%d findings for five empty replacements: %+v", len(got), got)
}
for i, o := range got {
want := conditions.Warning
if strings.Contains(o.ID, "mesh_home_board") {
want = conditions.Urgent
}
_ = i
if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" ||
len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") {
t.Errorf("%+v", o)
}
}
// While the old copy is still active (the first ten minutes), it is the same finding.
copies[0].Retired = false
copies[1].Class = "valuable"
if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement {
t.Fatalf("with the old copy still active: %+v", got)
}
}
// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and
// a retired one at the old: nothing to say here; `cleanup` covers the old one.
func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) {
copies := []consumerCopy{
{Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true},
{Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)},
}
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
t.Fatalf("a deliberate move raised %+v", got)
}
// Two small databases differing by less than the floor are not a finding either.
copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20)
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
t.Fatalf("noise between two empty databases raised %+v", got)
}
}
// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning,
// since which one is empty cannot be told.
func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) {
copies := []consumerCopy{
{Node: "home", Module: "minio", Consumer: "mesh_home_photos"},
{Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"},
}
got := linted(dataFindings(nil, nil, nil, copies, nil, time.Now()))
if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning {
t.Fatalf("%+v", got)
}
}
// The same incident for a module's own data: a module unassigned from one machine and assigned on
// another starts over in an empty directory while its full one is kept, retired, where it was.
func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) {
now := time.Now()
retired := now.Add(-time.Hour)
records := []inventory.DataRecord{
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)},
{Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)},
}
got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
o, ok := got[kindEmptyReplacement]
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" {
t.Fatalf("%+v", got)
}
// The same of a valuable item is a warning.
records[0].Class, records[1].Class = "valuable", "valuable"
if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning {
t.Fatalf("a valuable empty replacement: %+v", o)
}
records[0].Class, records[1].Class = "irreplaceable", "irreplaceable"
// Two machines running a module on purpose, both active, keep two sets of data: nothing to say.
records[0].RetiredAt = nil
if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" {
t.Fatalf("two active copies were read as a replacement: %+v", got)
}
}
// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss,
// or a loss under the floor, is not a finding.
func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
now := time.Now()
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)}
shelf := shelfFor(t, houseManifest)
o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank]
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") {
t.Fatalf("%+v", o)
}
for _, peak := range []int64{500 << 20, 20 << 20} {
if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
t.Errorf("a peak of %d raised a shrink", peak)
}
}
small := r
small.Size = bytesOf(1 << 20)
if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
t.Error("a loss under the floor raised a shrink")
}
}
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
now := time.Now()
shelf := shelfFor(t, houseManifest)
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)),
LastBackup: when(now.Add(-72 * time.Hour))}
got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))
if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent {
t.Fatalf("irreplaceable: %+v", got)
}
valuable := r
valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup
if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning ||
got[kindBackupStale].Severity != conditions.Warning {
t.Fatalf("valuable: %+v", got)
}
never := r
never.LastBackup = nil
if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") {
t.Fatalf("never backed up: %+v", o)
}
fresh := never
fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now)
if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got)
}
}
// An item retired more than thirty days waits for a person; less, it is only listed.
func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) {
now := time.Now()
old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour)
records := []inventory.DataRecord{
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old},
{Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent},
}
got := linted(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") {
t.Fatalf("%+v", got)
}
if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind {
t.Fatalf("cleanup list: %+v", rows)
}
}
// The holder's answer reads into measurements, by module and item.
func TestTheHoldersAnswerIsRead(t *testing.T) {
raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3,
"data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps",
"size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`)
got, err := readHolder(raw)
if err != nil {
t.Fatal(err)
}
m := got["postgres"]["store"]
if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil {
t.Fatalf("%+v", m)
}
// An older holder, which says no data, reads as nothing measured rather than a failure.
if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 {
t.Fatalf("%v, %v", got, err)
}
}
// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it
// measured.
type fakeHolder struct {
mu sync.Mutex
modules []map[string]any
}
func (f *fakeHolder) set(modules ...map[string]any) {
f.mu.Lock()
defer f.mu.Unlock()
f.modules = modules
}
func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) {
t.Helper()
sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
f.mu.Lock()
defer f.mu.Unlock()
body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node})
_ = m.Respond(body)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
}
func measuredHouse(path string, size int64, at time.Time) map[string]any {
return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{
"item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size,
"last_write": at, "measured_at": at, "last_backup": at}}}
}
// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
// up on another machine with an empty directory while the full one waits retired, that is urgent.
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
house, err := catalogue.ParseManifest([]byte(houseManifest))
if err != nil {
t.Fatal(err)
}
register(t, open, house)
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
}
for _, node := range []string{"laptop", "anchor"} {
if _, err := assign(ctx, open, node, "keeper"); err != nil {
t.Fatal(err)
}
}
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
t.Fatal(err)
}
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
for _, node := range []string{"laptop", "anchor"} {
pushedAndApplied(t, open, node)
}
conn := onATestBus(t)
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
laptop, anchor := &fakeHolder{}, &fakeHolder{}
laptop.serve(t, conn, "laptop")
anchor.serve(t, conn, "anchor")
now := time.Now()
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
d := &doctor{open: open, js: js, watchdogs: heard}
var d13 probe
for _, p := range probeRegistry {
if p.ID == probeDataID {
d13 = p
}
}
run := func() []conditions.Observation {
t.Helper()
probing := context.WithValue(ctx, probeAsksKey{}, d13)
obs, err := probeData(probing, d)
if err != nil {
t.Fatal(err)
}
return obs
}
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
if obs := run(); len(obs) != 0 {
t.Fatalf("a measured, backed-up item raised %+v", obs)
}
r, err := inv.DataOf(ctx, "laptop", "house", "config")
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
}
said, err := unassign(ctx, open, "laptop", "house")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
}
laptop.set()
run()
r, err = inv.DataOf(ctx, "laptop", "house", "config")
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
}
records, _ := inv.Data(ctx)
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
t.Fatalf("cleanup list: %+v", rows)
}
// Assigned on the anchor, onto an empty directory.
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
t.Fatal(err)
}
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
obs := findingsByKind(run())
o, ok := obs[kindEmptyReplacement]
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
}
}
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
// whose path is gone is said.
func TestTheArrayUnderDataIsWatched(t *testing.T) {
now := time.Now()
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"},
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}`
shelf := shelfFor(t, media)
sick := false
on := func(item string, healthy *bool) inventory.DataRecord {
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false,
Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}}
}
got := linted(dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now))
if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
t.Fatalf("%+v", got)
}
well := true
if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("a healthy array raised %+v", got)
}
plain := on("films", nil)
plain.Redundancy = nil
if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent {
t.Fatalf("redundancy said and not found: %+v", o)
}
gone := on("films", &well)
gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0)
if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent {
t.Fatalf("a vanished library: %+v", o)
}
}
// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class:
// the photo site's objects make the object store's data an urgent matter.
func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"},
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`
photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`
shelf := shelfFor(t, objects, photos)
bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket",
Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}}
upgraded, keptBy := keptByClasses(bindings, shelf)
if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" {
t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy)
}
now := time.Now()
r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true,
Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))}
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent {
t.Fatalf("the photos' store without a backup: %+v", o)
}
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning {
t.Fatalf("a valuable store without a backup: %+v", o)
}
}
// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition
// naming every item on it, not one per item; and a partial walk's lower bound is never compared.
func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) {
now := time.Now()
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"},
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}`
shelf := shelfFor(t, media, houseManifest)
well := true
on := func(item string, size int64) inventory.DataRecord {
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable",
Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
Precision: "dataset tank/media: its whole size",
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}}
}
films, shows := on("films", 30<<40), on("shows", 30<<40)
peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40}
got := linted(dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now))
if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
t.Fatalf("%+v", got)
}
partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now),
LastBackup: when(now), Precision: "partial: measured partially"}
if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("a partial size was compared: %+v", got)
}
}
cmd/mesh-controller/delivery.go
-643
View File
@@ -1,643 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The controller's part in a delivery (novox/hq ADR 0239, to-be 47).
//
// **A delivery is mesh-delivery's; the walk is the controller's.** A delivery is one commit in one
// repository from its pull request's head to every machine, and a delivery group a few of them sharing a
// branch name; their states, order, holds and record belong to the module holding the `mesh-delivery`
// seat. The controller keeps what it already owned — the planner, the gate, registration, sending, the
// rollback — and the **walk**: one trunk commit's plan, tier by tier across the machines, one machine
// first and judged at the gate (ADR 0236). What changes here is when a walk starts.
//
// - A walk that moves the controller, the node-engine, the node tools, the bus or mesh-delivery itself is
// **on the controller's own path**: started by the merge, as every plan was. mesh-delivery cannot gate
// or deliver itself, and nothing the core needs to be repaired may wait for it.
// - Any other walk, **while the seat has a holder on record**, is opened by the merge and waits — nothing
// asked, nothing registered, nothing sent — until mesh-delivery says `deliver`, or a person says
// `plans go`. Nothing of it is registered before then, so no other send can carry it to a machine
// before its turn (ADR 0236 §4a carries everything registered).
// - With no holder on record, every walk starts at the merge, exactly as before this existed.
//
// The verbs mesh-delivery asks with are here: `delivery plan` (the planner's one answer for a diffset),
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
// go`, `delivery stop` and `delivery walks`.
// onTheControllersPath are the modules whose walk never waits for the delivery's owner, and what each is.
var onTheControllersPath = map[string]string{
"mesh-controller": "the controller",
"mesh-host": "the node-engine",
"node-tools": "the node tools",
"nats": "the bus",
catalogue.DeliverySeat: "the delivery's owner",
}
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
// holder on record. Read from the catalogue the merge handler already holds; never from whether the
// holder answers, so a walk does not start by itself because mesh-delivery is down — that wait is said.
func deliverySeatHeld(entries []inventory.Entry) bool {
for _, e := range entries {
if len(e.On) > 0 && e.Manifest.ClaimsSeat(catalogue.DeliverySeat) {
return true
}
}
return false
}
// awaitsFor is what a new walk waits for: nil when it starts at once — no holder on record, or a module
// on the controller's own path among those it moves.
func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDelivery {
if !deliverySeatHeld(entries) {
return nil
}
for _, m := range modules {
if _, own := onTheControllersPath[m]; own {
return nil
}
}
return &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}
}
// waitingNote is what a walk waiting for its word says, wherever it is read.
func waitingNote(p inventory.Plan) string {
return fmt.Sprintf("published; its walk waits for %s's word — `plans go %s --why …` starts it by hand",
p.Delivery.Awaits, p.ID)
}
// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its
// first tier. Refused for a walk that does not wait.
func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return inventory.Plan{}, err
}
defer release()
p, err := inv.PlanByID(ctx, id)
if err != nil {
return inventory.Plan{}, err
}
switch {
case !p.Open():
return p, fmt.Errorf("%s is %s: there is no walk to start", p.ID, p.State)
case p.Delivery == nil || p.Delivery.Awaits == "":
return p, fmt.Errorf("%s waits for nobody: it started at its merge", p.ID)
case p.Delivery.Go != nil:
return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By,
p.Delivery.Go.Local().Format("15:04:05"))
}
now := time.Now().UTC()
p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why
p.Note = "let go by " + by + "; its first tier is asked next"
if err := inv.SavePlan(ctx, &p); err != nil {
return p, err
}
return p, nil
}
// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still
// builds and registers; nothing further is asked or sent.
func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) {
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return inventory.Plan{}, err
}
defer release()
p, err := inv.PlanByID(ctx, id)
if err != nil {
return inventory.Plan{}, err
}
if !p.Open() {
return p, fmt.Errorf("%s is already %s", p.ID, p.State)
}
if p.Delivery == nil {
p.Delivery = &inventory.PlanDelivery{}
}
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
if err := inv.SavePlan(ctx, &p); err != nil {
return p, err
}
return p, nil
}
// orderMember is one member of a group, as mesh-delivery says it.
type orderMember struct {
ID string `json:"id"`
Repository string `json:"repository"`
Base string `json:"base,omitempty"`
Head string `json:"head,omitempty"`
Number int `json:"number,omitempty"`
Paths []string `json:"paths,omitempty"`
PathsTruncated bool `json:"paths_truncated,omitempty"`
Removed []string `json:"removed,omitempty"`
ModuleDirs []string `json:"module_dirs,omitempty"`
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
CloneURL string `json:"clone_url,omitempty"`
// After are the repositories its pull request says it goes after (`after: <repository>`).
After []string `json:"after,omitempty"`
}
// pull is the member as the forge announced it.
func (m orderMember) pull() link.PullUpdated {
owner, repo, _ := strings.Cut(m.Repository, "/")
base := m.Base
if base == "" {
base = "main"
}
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head,
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
}
// orderPair is one "before" among a group's members, and why.
type orderPair struct {
Before string `json:"before"`
After string `json:"after"`
Why string `json:"why"`
}
// orderReach is what a member moves, as the planner says.
type orderReach struct {
Moved []string `json:"moved,omitempty"`
Dependents []string `json:"dependents,omitempty"`
New []string `json:"new,omitempty"`
Manifests []string `json:"manifests,omitempty"`
}
// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one.
type groupOrder struct {
Order []string `json:"order"`
Pairs []orderPair `json:"pairs,omitempty"`
Cycle []string `json:"cycle,omitempty"`
Reach map[string]orderReach `json:"reach,omitempty"`
}
// The reasons a pair is ordered, in the words the delivery plan shows.
const (
orderDeclared = "declared"
orderBuiltBy = "built by"
orderVersionSkew = "version skew"
orderEngineFirst = "engine before controller"
)
// orderOf is a group's order (novox/hq ADR 0239 decision 4), pure. A member goes before another when:
//
// - its pull request is named in the other's `after:` lines (declared);
// - it moves a module the other's moved modules are built by, stand on, package or declare (built by);
// - it moves the controller and the other changes any module's manifest (version skew: the newer
// controller parses what the newer manifest says) — the node-engine's excepted, which goes first;
// - it moves the node-engine and the other moves the controller (the witness reads nothing the controller
// does not yet grant, and a controller sends nothing an older engine would refuse — ADR 0236's rollout
// order).
//
// Otherwise, by repository then id, so every reading gives one order. A pair both ways is a cycle: the
// members in it are named, and none of them is ordered.
func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge) groupOrder {
out := groupOrder{Reach: reach}
byID := map[string]orderMember{}
for _, m := range members {
byID[m.ID] = m
}
add := func(before, after, why string) {
if before == after {
return
}
for _, p := range out.Pairs {
if p.Before == before && p.After == after {
return
}
}
out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why})
}
named := func(said, repository string) bool {
said = strings.TrimSuffix(strings.TrimSpace(said), ".git")
if strings.EqualFold(said, repository) {
return true
}
_, repo, _ := strings.Cut(repository, "/")
return strings.EqualFold(said, repo)
}
for _, a := range members {
for _, b := range members {
if a.ID == b.ID {
continue
}
ra, rb := reach[a.ID], reach[b.ID]
for _, said := range b.After {
if named(said, a.Repository) {
add(a.ID, b.ID, orderDeclared)
}
}
for _, e := range edges {
if slices.Contains(ra.Moved, e.To) && slices.Contains(rb.Moved, e.From) && e.From != e.To {
add(a.ID, b.ID, orderBuiltBy)
break
}
}
if slices.Contains(ra.Moved, "mesh-controller") && len(rb.Manifests) > 0 &&
!slices.Contains(rb.Moved, "mesh-controller") && !slices.Contains(rb.Moved, "mesh-host") {
add(a.ID, b.ID, orderVersionSkew)
}
if slices.Contains(ra.Moved, "mesh-host") && slices.Contains(rb.Moved, "mesh-controller") &&
!slices.Contains(ra.Moved, "mesh-controller") {
add(a.ID, b.ID, orderEngineFirst)
}
}
}
sort.Slice(out.Pairs, func(i, j int) bool {
if out.Pairs[i].Before != out.Pairs[j].Before {
return out.Pairs[i].Before < out.Pairs[j].Before
}
return out.Pairs[i].After < out.Pairs[j].After
})
// Kahn's walk, the next always the first by repository and id among those with nothing before them.
waiting := map[string]int{}
for _, m := range members {
waiting[m.ID] = 0
}
for _, p := range out.Pairs {
waiting[p.After]++
}
done := map[string]bool{}
for len(done) < len(members) {
var ready []orderMember
for _, m := range members {
if !done[m.ID] && waiting[m.ID] == 0 {
ready = append(ready, m)
}
}
if len(ready) == 0 {
for _, m := range members {
if !done[m.ID] {
out.Cycle = append(out.Cycle, m.ID)
}
}
sort.Strings(out.Cycle)
return out
}
sort.Slice(ready, func(i, j int) bool {
if ready[i].Repository != ready[j].Repository {
return ready[i].Repository < ready[j].Repository
}
return ready[i].ID < ready[j].ID
})
next := ready[0]
done[next.ID] = true
out.Order = append(out.Order, next.ID)
for _, p := range out.Pairs {
if p.Before == next.ID {
waiting[p.After]--
}
}
}
return out
}
// reachOfMembers is each member's reach, as the planner says it.
func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
edges []inventory.Edge) map[string]orderReach {
out := map[string]orderReach{}
for _, m := range members {
s := pullScope(m.pull(), entries, read, edges)
out[m.ID] = orderReach{Moved: s.Modules, Dependents: s.Dependents, New: s.New, Manifests: s.Manifests}
}
return out
}
// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner:
//
// delivery plan --repository owner/repo --head <commit> [--base main] --paths a,b [--module-dirs …] [--removed …]
// delivery order --members <json>
// delivery check --group <id> --members <json>
// delivery go <plan> [--by <who>] [--why <text>]
// delivery stop <plan> --why <text> [--by <who>]
// delivery walks [-n 50] [--plan <id>]
func deliveryCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("delivery plan|order|check|go|stop|walks")
}
sub, rest := args[0], args[1:]
set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError)
repository := set.String("repository", "", "owner/repository")
base := set.String("base", "main", "the branch it merges into")
head := set.String("head", "", "the commit at hand")
paths := set.String("paths", "", "the files it changes, comma-separated")
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
removed := set.String("removed", "", "the files it deletes, comma-separated")
membersJSON := set.String("members", "", "a group's members, as JSON")
group := set.String("group", "", "a delivery group's id")
by := set.String("by", catalogue.DeliverySeat, "who says it")
why := set.String("why", "", "why")
limit := set.Int("n", 50, "how many ended walks to answer beside the open ones")
planID := set.String("plan", "", "one walk")
positionals, err := parseAround(set, rest)
if err != nil {
return err
}
var members []orderMember
if *membersJSON != "" {
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
return fmt.Errorf("the members are not readable: %w", err)
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
answer := func(v any) error {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
return enc.Encode(v)
}
switch sub {
case "plan":
if *repository == "" || *head == "" && *paths == "" {
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
}
m := orderMember{ID: *repository + "@" + *head, Repository: *repository, Base: *base, Head: *head,
Paths: splitList(*paths), Removed: splitList(*removed)}
if d := moduleDirsOf(*moduleDirs); d != nil {
m.ModuleDirs, m.ModuleDirsSaid = *d, true
}
entries, read, edges, err := theGraph(ctx, inv)
if err != nil {
return err
}
s := pullScope(m.pull(), entries, read, edges)
plan := changePlanOf(*repository, *base, *head, s.Reach, entries, func(module string) (inventory.Upgrade, bool) {
u, err := inv.UpgradeOf(ctx, module)
return u, err == nil
})
return answer(map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()})
case "order":
if len(members) == 0 {
return errors.New("delivery order --members <json>: a group has members")
}
entries, read, edges, err := theGraph(ctx, inv)
if err != nil {
return err
}
return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges))
case "check":
if *group == "" && len(members) == 1 {
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
// own path for a pull request, run because the delivery's owner asked.
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
return err
}
return answer(map[string]any{"rechecked": members[0].ID})
}
if *group == "" || len(members) < 2 {
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
}
id, err := askGroupCheck(ctx, open, *group, members)
if err != nil {
return err
}
return answer(map[string]any{"asked": id, "group": *group})
case "go":
if len(positionals) != 1 {
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
}
var p inventory.Plan
if err := sayingOnTheBus(ctx, func() (err error) {
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
return err
}); err != nil {
return err
}
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
short(p.Commit), *by)
return nil
case "stop":
if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
return errors.New("delivery stop <plan> --why <text> [--by <who>]")
}
var p inventory.Plan
if err := sayingOnTheBus(ctx, func() (err error) {
p, err = stopWalk(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
return err
}); err != nil {
return err
}
fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
return nil
case "walks":
var walks []inventory.Plan
if *planID != "" {
p, err := inv.PlanByID(ctx, *planID)
if err != nil {
return err
}
walks = []inventory.Plan{p}
} else {
if walks, err = inv.OpenPlans(ctx); err != nil {
return err
}
recent, err := inv.RecentPlans(ctx, *limit)
if err != nil {
return err
}
for _, p := range recent {
if !slices.ContainsFunc(walks, func(w inventory.Plan) bool { return w.ID == p.ID }) {
walks = append(walks, p)
}
}
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
"own-path": sortedKeysOf(ownPathWords())})
}
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
}
// ownPathWords is the controller's own path as words, for an answer.
func ownPathWords() map[string]string { return onTheControllersPath }
// theGraph is what the planner reads.
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
[]inventory.Edge, error) {
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, nil, nil, err
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return nil, nil, nil, err
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return nil, nil, nil, err
}
return entries, read, edges, nil
}
// groupPrimary is the head a group's composed check is run from: the one moving the controller, which then
// judges the group by itself; else the one moving the node-engine, whose validator judges; else the first.
func groupPrimary(members []orderMember, reach map[string]orderReach) int {
for _, want := range []string{"mesh-controller", "mesh-host"} {
for i, m := range members {
if slices.Contains(reach[m.ID].Moved, want) {
return i
}
}
}
return 0
}
// askGroupCheck asks the build seat for a group's composed check: every head laid over the mesh in turn,
// judged as one future state (novox/hq ADR 0239). The verdict comes back as `checked` with the group's id,
// for mesh-delivery; the forge's holder sets no status from it.
func askGroupCheck(ctx context.Context, open *stores, group string, members []orderMember) (string, error) {
entries, read, edges, err := theGraph(ctx, open.inventory)
if err != nil {
return "", err
}
reach := reachOfMembers(members, entries, read, edges)
primary := groupPrimary(members, reach)
p := members[primary].pull()
scope := pullScope(p, entries, read, edges)
// The gate runs over what any member moves; a judge from the primary alone.
for _, m := range members {
r := reach[m.ID]
scope.Modules = appendNew(scope.Modules, r.Moved...)
scope.Dependents = appendNew(scope.Dependents, r.Dependents...)
}
request, err := checkRequestFor(ctx, open, p, scope, entries)
if err != nil {
return "", err
}
request.Check.Group = group
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
if err != nil {
return "", err
}
for i, m := range members {
if i == primary {
continue
}
mp := m.pull()
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
source := inventory.Source{Seat: gitSeat, Repository: mp.Owner + "/" + mp.Repo}
for _, e := range entries {
if !e.Provided && sameRepository(e.Source.Repository, link.SourceMoved{Owner: mp.Owner, Repo: mp.Repo}) {
source = e.Source
break
}
}
url := source.Repository
if source.Seat != "" {
url, err = clonedFromSeat(world, source.Seat, source.Repository)
}
if err != nil {
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
}
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths})
}
seat := buildSeatHeld(ctx)
ask, err := askOverOn(seat)
if err != nil {
return "", err
}
defer ask.Close()
if err := ask.Ask(ctx, request); err != nil {
return "", err
}
fmt.Fprintf(os.Stderr, "group %s: asked %s to check %d head(s) composed together, as %s\n", group, seat,
len(members), request.ID)
return request.ID, nil
}
// appendNew appends what is not there yet.
func appendNew(to []string, items ...string) []string {
for _, i := range items {
if !slices.Contains(to, i) {
to = append(to, i)
}
}
return to
}
// shelfOf is the catalogue's manifests by name.
func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
shelf := map[string]catalogue.Manifest{}
for _, e := range entries {
shelf[e.Manifest.Module] = e.Manifest
}
return shelf
}
// sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
// comparison. Records arriving out of order are told apart by the plan's revision.
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
go publishPlanMoved(ctx, bus, p)
}
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
body, err := json.Marshal(p)
if err != nil {
return
}
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer stop()
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, body); err != nil {
fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err)
}
}
// sayingOnTheBus runs a command that keeps walks or says verdicts with the bus to say them on: the serving
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
// still runs; what it did is found by comparison.
func sayingOnTheBus(ctx context.Context, f func() error) error {
if checkEvents != nil {
return f()
}
ran := false
err := onTheBus(func(conn *nats.Conn) error {
js, err := conn.JetStream()
if err != nil {
return err
}
bus := link.OverNATS{Conn: conn, JS: js}
checkEvents = bus
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
ran = true
return f()
})
if !ran {
fmt.Fprintf(os.Stderr, "the bus cannot be reached (%v): what this does is not said, and is found by comparison\n", err)
return f()
}
return err
}
cmd/mesh-controller/delivery_conditions.go
-182
View File
@@ -1,182 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table
// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the
// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery.<id>.stalled`; healer
// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only
// the next observation says whether it worked (ADR 0231).
// probeDeliveriesID is the probe that reads the delivery's owner.
const probeDeliveriesID = "D14"
// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope.
const kindDeliveryStalled = "stalled"
// deliveryOwnerWithin is how long the owner is given to answer.
var deliveryOwnerWithin = 10 * time.Second
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
type stalledLine struct {
ID string `json:"id"`
State string `json:"state"`
For string `json:"for"`
Bound string `json:"bound"`
H2 string `json:"h2"`
Says string `json:"says"`
}
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted;
// a seam a test replaces. The holder's own refusal is an error naming it.
var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
granted := false
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
granted = granted || v.Verb == verb
}
if !granted {
return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb)
}
if conn == nil {
return nil, errors.New("this controller is not on the bus")
}
answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin)
if err != nil {
return nil, err
}
if answer.Error != "" {
return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error)
}
return unwrapToolResult(answer.Result), nil
}
// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the
// protocol's content list holding it as text.
func unwrapToolResult(raw json.RawMessage) json.RawMessage {
var wrapped struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
}
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) {
return json.RawMessage(wrapped.Content[0].Text)
}
return raw
}
// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record,
// or none answers (D3 says that one).
func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) {
if !held {
return nil, nil
}
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{})
if errors.Is(err, link.ErrNothingServes) {
return nil, nil // the holder not running is D3's holder-silent, said once there
}
if err != nil {
return nil, err
}
var lines []stalledLine
if err := json.Unmarshal(raw, &lines); err != nil {
return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err)
}
return lines, nil
}
// stalledObservations are the conditions of what is stalled.
func stalledObservations(lines []stalledLine) []conditions.Observation {
out := make([]conditions.Observation, 0, len(lines))
for _, l := range lines {
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
Severity: conditions.Warning,
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
if l.operatorsOnly() {
o.Resolver = conditions.ResolverOperator
}
out = append(out, o)
}
return out
}
// probeDeliveries is D14: every delivery held past its state's bound is said.
func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
entries, err := d.open.inventory.Catalogued(ctx)
if err != nil {
return nil, err
}
var conn *nats.Conn
if d.js != nil {
conn = d.js.Conn()
}
lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries))
if err != nil {
return nil, err
}
return stalledObservations(lines), nil
}
// --- H2, for a delivery ---------------------------------------------------------------------------------
// connOf is the bus a healer asks over.
func (h *healing) connOf() *nats.Conn {
if h.js == nil {
return nil
}
return h.js.Conn()
}
// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table
// lets H2 take; never one whose state is the operator's.
func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
lines, err := deliveriesStalled(ctx, h.connOf(), true)
if err != nil {
return "", false, "", err
}
for _, l := range lines {
if l.ID != c.Subject.ID {
continue
}
if l.operatorsOnly() {
return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil
}
return c.Key, true, "", nil
}
return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil
}
// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the
// transition its table names. A refusal is no repair, said; the next observation says whether it worked.
func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key})
if err != nil {
if errors.Is(err, link.ErrNothingServes) {
return "", "", err
}
return "closed nothing", oneLine(err.Error()), nil
}
var said string
if json.Unmarshal(raw, &said) != nil {
said = string(raw)
}
return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil
}
cmd/mesh-controller/delivery_conditions_test.go
-171
View File
@@ -1,171 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's
// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`.
// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records
// what close was asked.
func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string {
t.Helper()
var closed []string
was := askDeliveryOwner
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
switch verb {
case "stalled":
raw, _ := json.Marshal(lines)
return raw, nil
case "close":
closed = append(closed, args["id"].(string))
if closeErr != nil {
return nil, closeErr
}
return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil
}
t.Fatalf("asked the owner %s", verb)
return nil, nil
}
t.Cleanup(func() { askDeliveryOwner = was })
return &closed
}
func holdTheDeliverySeat(t *testing.T, open *stores) {
t.Helper()
m := catalogue.Manifest{Module: "mesh-delivery", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
}
var twoStalled = []stalledLine{
{ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s",
H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"},
{ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s",
H2: "none: the state is the operator's", Says: "it waits for the operator"},
}
func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
ownerAnswers(t, twoStalled, nil)
d := &doctor{open: open}
got, err := probeDeliveries(ctx, d)
if err != nil || len(got) != 0 {
t.Fatalf("with no holder on record D14 said %+v %v", got, err)
}
holdTheDeliverySeat(t, open)
got, err = probeDeliveries(ctx, d)
if err != nil || len(got) != 2 {
t.Fatalf("D14 said %+v %v", got, err)
}
if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning ||
got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") {
t.Fatalf("the first is %+v", got[0])
}
if got[1].Resolver != conditions.ResolverOperator {
t.Fatalf("a held delivery is not the operator's: %+v", got[1])
}
// The holder not running is D3's to say, not D14's.
was := askDeliveryOwner
askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) {
return nil, link.ErrNothingServes
}
defer func() { askDeliveryOwner = was }()
if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 {
t.Fatalf("an owner that is down was said by D14: %+v %v", got, err)
}
}
// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a
// refusal is no repair.
func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, _ := healingOn(t, open)
closed := ownerAnswers(t, twoStalled, nil)
for _, o := range stalledObservations(twoStalled) {
o.Source = probeDeliveriesID
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
t.Fatal(err)
}
}
h.tick(ctx)
if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) {
t.Fatalf("close was asked for %v", *closed)
}
acts := told.said()
if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" {
t.Fatalf("said %+v", acts)
}
// A refusal: closed nothing, said so.
c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled")
ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering"))
act, said, err := repairDelivery(ctx, h, c)
if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") {
t.Fatalf("a refused close reads %q %q %v", act, said, err)
}
}
// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else
// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read.
func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
for _, verb := range []string{"stalled", "close"} {
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
}
}
if _, err := askDeliveryOwner(t.Context(), nil, "stop", nil); err == nil || !strings.Contains(err.Error(), "grant") {
t.Fatalf("a verb the grant does not name was asked: %v", err)
}
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if _, err := deliveriesStalled(t.Context(), conn, true); err != nil {
t.Fatalf("an owner not running is D3's, not an error: %v", err)
}
lines, _ := json.Marshal(twoStalled)
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}})
sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) {
body, _ := json.Marshal(link.Answer{Result: wrapped})
_ = m.Respond(body)
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
got, err := deliveriesStalled(ctx, conn, true)
if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" {
t.Fatalf("over the bus: %+v %v", got, err)
}
}
cmd/mesh-controller/delivery_test.go
-318
View File
@@ -1,318 +0,0 @@
package main
import (
"encoding/json"
"reflect"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every
// reading — the build agent before what it builds, the controller before a manifest that needs it, the
// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered.
func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
members := []orderMember{
{ID: "cat", Repository: "novox/mesh-catalog"},
{ID: "ctl", Repository: "novox/mesh-controller"},
{ID: "host", Repository: "novox/mesh-host"},
{ID: "agent", Repository: "novox/build-agent"},
{ID: "app", Repository: "novox/app", After: []string{"lab"}},
{ID: "lab", Repository: "novox/lab"},
}
reach := map[string]orderReach{
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
"agent": {Moved: []string{"build-agent"}},
"app": {Moved: []string{"app"}},
"lab": {Moved: []string{"lab"}},
}
edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
got := orderOf(members, reach, edges)
if len(got.Cycle) > 0 {
t.Fatalf("a cycle where there is none: %v", got.Cycle)
}
// By repository among those with nothing before them: the build agent, the lab, then what waited on both.
want := []string{"agent", "lab", "app", "host", "ctl", "cat"}
if !reflect.DeepEqual(got.Order, want) {
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
}
why := map[string]string{}
for _, p := range got.Pairs {
why[p.Before+">"+p.After] = p.Why
}
for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew,
"ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} {
if why[pair] != reason {
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
}
}
// The same members in another order read the same.
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) {
t.Fatalf("another reading ordered %v", again.Order)
}
// A declared order against an inferred one is a cycle: named, and nothing ordered.
members[1].After = []string{"mesh-catalog"}
got = orderOf(members, reach, edges)
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) {
t.Fatalf("the cycle is %v, ordered %v", got.Cycle, got.Order)
}
}
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
// record, and never one that moves a module on the controller's own path.
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
unassigned := holder
unassigned.On = nil
for _, c := range []struct {
entries []inventory.Entry
moved []string
waits bool
}{
{nil, []string{"gitea"}, false},
{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
{[]inventory.Entry{holder}, []string{"node-tools"}, false},
{[]inventory.Entry{holder}, []string{"nats"}, false},
{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
} {
d := awaitsFor(c.entries, c.moved)
if (d != nil) != c.waits {
t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
c.moved, d != nil, c.waits)
}
if d != nil && d.Awaits != catalogue.DeliverySeat {
t.Errorf("waits for %q", d.Awaits)
}
}
}
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
// hand when the owner is down, and the owner's stop ends one as stopped.
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := asksRecorded(t)
withConditionsInMemory(t)
for _, name := range []string{"app", "mesh-delivery"} {
m := catalogue.Manifest{Module: name, Version: "1"}
if name == "mesh-delivery" {
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
}
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
}
merge := func(commit string, paths ...string) inventory.Plan {
t.Helper()
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
recent, err := inv.RecentPlans(ctx, 1)
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
}
return recent[0]
}
// finish ends a walk as done, so the next merge has nothing of it to take over.
finish := func(id string) {
t.Helper()
w, err := inv.PlanByID(ctx, id)
if err != nil {
t.Fatal(err)
}
w.State = inventory.PlanDone
if err := inv.SavePlan(ctx, &w); err != nil {
t.Fatal(err)
}
}
// No holder on record: the merge starts its walk, as before.
p := merge("c1aaaaaaaa", "modules/app/index.ts")
if p.Waiting() || len(*asked) != 1 {
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
}
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either.
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
p = merge("c2bbbbbbbb", "modules/app/index.ts")
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
}
advanceHeld(ctx, open)
if len(*asked) != 1 {
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
}
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
t.Errorf("a waiting walk reads %q", line)
}
// The delivery's word starts it.
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
t.Fatal(err)
}
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
t.Fatal("a walk was let go twice")
}
advanceHeld(ctx, open)
if len(*asked) != 2 {
t.Fatalf("the word did not start the walk: %v", *asked)
}
got, err := inv.PlanByID(ctx, p.ID)
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
}
// The delivery's owner's own merge never waits for it — and takes over what the older walk had not
// built (app, folded in: ADR 0218), which goes with it on the controller's own path.
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
if p.Waiting() || len(*asked) != 4 {
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
}
// The owner down: a person starts a waiting walk, with why.
finish(p.ID)
p = merge("c4dddddddd", "modules/app/index.ts")
if !p.Waiting() {
t.Fatal("the walk did not wait")
}
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
t.Fatalf("a walk was started by hand without why: %v", err)
}
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
t.Fatal(err)
}
advanceHeld(ctx, open)
got, _ = inv.PlanByID(ctx, p.ID)
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 5 {
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
}
// The owner's stop: failed, said as stopped by it.
finish(p.ID)
p = merge("c5eeeeeeee", "modules/app/index.ts")
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
"mesh-delivery for jochen"}); err != nil {
t.Fatal(err)
}
got, _ = inv.PlanByID(ctx, p.ID)
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
!strings.Contains(got.Note, "the operator stopped it") {
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
}
// Unassigned: the mesh is back on the controller's own path.
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
asks := len(*asked)
if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
}
}
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want []string
}{
{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
{"delivery-check", map[string]any{"group": "feat/x", "members": "[]"},
[]string{"delivery", "check", "--group", "feat/x", "--members", "[]"}},
{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"},
[]string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}},
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
} {
got, err := argvFor(c.verb, c.args)
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
}
}
if _, err := argvFor("deliver", map[string]any{}); err == nil {
t.Error("deliver without a walk was composed")
}
if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
t.Error("a stop without why was composed")
}
}
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
url := testbus.URL(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
heard := make(chan *nats.Msg, 4)
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {}},
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
t.Fatal(err)
}
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
var said inventory.Plan
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
said.Delivery.Go == nil {
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
}
case <-time.After(5 * time.Second):
t.Fatal("a walk let go by a verb was not said")
}
if checkEvents != nil || inventory.PlanSaved != nil {
t.Fatal("the command's own bus was left in place")
}
}
cmd/mesh-controller/doctor.go
-600
View File
@@ -1,600 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
//
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
// design — every machine's declaration composes and validates, every resolver answers, every seat's
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
// condition it raises when the invariant does not hold. The serving controller runs the registry every
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
// machine: a controller that stops checking is itself said, through a channel that does not pass
// through it.
//
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
// The self-check's clocks.
var (
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
// starts the first run waits, so what the controller hears at its start has arrived.
doctorEvery = 5 * time.Minute
doctorFirstAfter = time.Minute
// probeWithin is each probe's bound.
probeWithin = 30 * time.Second
)
// The verdicts a probe can have.
const (
verdictPass = "pass"
verdictFail = "fail"
verdictFailedToRun = "failed-to-run"
verdictDeferred = "deferred"
)
// probe is one live invariant.
type probe struct {
ID string
Asserts string
From string
// Kind is the condition kind raised when the invariant does not hold.
Kind string
// Raises are the other kinds its findings carry, each its own (a consumer missing, one far behind):
// what a healer may be registered against (healers.go).
Raises []string
Phase int
// Deferred says why it is not run yet; empty for one that is.
Deferred string
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
// controller's grant names every one (a test over this registry): a probe whose question the bus
// refuses checks nothing (D8, 2026-10-06).
Asks []broker.SeatVerb
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
}
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
// probe added to a design is a row added here.
var probeRegistry = []probe{
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
run: probeDeclarations},
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
Kind: "archives-unheld", Phase: 1, run: probeArchives},
{ID: "D5", Asserts: "exactly one lease holder — the key names this controller at its epoch, and the record " +
"holds no other epoch open; no message from a stale epoch refused in the last interval",
From: "issue 204", Kind: "lease-split", Phase: 2, run: probeLease},
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Raises: []string{"consumer-lost", kindConsumerBehind},
Phase: 1, run: probeConsumers},
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding " +
"its cleanup", From: "ADR 0230", Kind: kindCleanupWaiting, Phase: 2, run: probeRetired},
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
{ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " +
"written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " +
"empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " +
"valuable is retired, not forgotten", From: "issue 273, ADR 0233",
Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale,
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
Phase: 2, run: probeData,
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
// The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by
// the controller, and H2 works it through the owner's `close`.
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
// judged by on its first machine, run against every machine between upgrades too.
{ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " +
"ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8",
Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth},
{ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " +
"build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth},
{ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus",
From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth},
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " +
"it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4,
run: probeBusHealth},
// The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its
// condition until a newer build passes; each rollback a witness stands by is said.
{ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " +
"a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack,
Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates},
// The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus.
{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
}
// probeVerdict is one probe's outcome in a run.
type probeVerdict struct {
ID string `json:"id"`
Verdict string `json:"verdict"`
Found []string `json:"found,omitempty"`
// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before:
// raised if the next run sees them too (confirm.go). Not a pass, and not yet a condition.
Unconfirmed []string `json:"unconfirmed,omitempty"`
Error string `json:"error,omitempty"`
Took string `json:"took,omitempty"`
}
// doctorCounts are a run's verdicts, counted.
type doctorCounts struct {
Passed int `json:"passed"`
Failed int `json:"failed"`
FailedToRun int `json:"failed-to-run"`
Deferred int `json:"deferred"`
}
// doctorRun is one run of the registry, and the body of its heartbeat.
type doctorRun struct {
Run string `json:"run"`
At time.Time `json:"at"`
Started time.Time `json:"started"`
Took string `json:"took"`
IntervalSeconds int `json:"interval-seconds"`
Counts doctorCounts `json:"counts"`
Probes []probeVerdict `json:"probes"`
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
Controller string `json:"controller"`
Why string `json:"why"`
// Unsaid is how many condition transitions this controller could not say, since it started.
Unsaid int `json:"unsaid,omitempty"`
}
// doctor is the registry and what its probes need.
type doctor struct {
open *stores
js *broker.JetStream
keeper *conditions.Keeper
teller conditions.Teller
watchdogs *watchdogs
host string
// confirm holds back what one run alone saw of a finding a single look can be wrong about.
confirm confirming
running sync.Mutex
mu sync.Mutex
last *doctorRun
ended time.Time
}
// lastRunEnded is when the last run ended; zero before the first.
func (d *doctor) lastRunEnded() time.Time {
d.mu.Lock()
defer d.mu.Unlock()
return d.ended
}
// lastRun is the last run's verdict; nil before the first.
func (d *doctor) lastRun() *doctorRun {
d.mu.Lock()
defer d.mu.Unlock()
return d.last
}
// keep runs the registry on its schedule until ctx ends.
func (d *doctor) keep(ctx context.Context) {
select {
case <-ctx.Done():
return
case <-time.After(doctorFirstAfter):
}
tick := time.NewTicker(doctorEvery)
defer tick.Stop()
for {
// Only the controller acting checks the mesh on a schedule: one standing by would say a
// heartbeat for a self-check that is not the mesh's.
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
d.runOnce(ctx, "the schedule")
}
select {
case <-ctx.Done():
return
case <-tick.C:
}
}
}
var doctorRuns struct {
sync.Mutex
n uint64
}
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
// at a time: a person's `doctor run` during a scheduled one waits for it.
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
d.running.Lock()
defer d.running.Unlock()
doctorRuns.Lock()
doctorRuns.n++
n := doctorRuns.n
doctorRuns.Unlock()
started := time.Now()
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
type result struct {
obs []conditions.Observation
err error
took time.Duration
}
results := make([]result, len(probeRegistry))
var wg sync.WaitGroup
for i, p := range probeRegistry {
if p.run == nil {
continue
}
wg.Add(1)
go func(i int, p probe) {
defer wg.Done()
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
defer cancel()
began := time.Now()
done := make(chan result, 1)
go func() {
defer func() {
if r := recover(); r != nil {
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
}
}()
obs, err := p.run(probing, d)
done <- result{obs: obs, err: err}
}()
select {
case r := <-done:
r.took = time.Since(began)
results[i] = r
case <-probing.Done():
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
}
}(i, p)
}
wg.Wait()
var blind []conditions.Observation
for i, p := range probeRegistry {
v := probeVerdict{ID: p.ID}
r := results[i]
switch {
case p.run == nil:
v.Verdict = verdictDeferred
run.Counts.Deferred++
case r.err != nil:
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
run.Counts.FailedToRun++
// A probe that could not run once — a question timed out on a loaded machine — is said in
// the verdict at once, and raised as a condition when the next run cannot run it either.
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
Token: "failed", Severity: conditions.Warning, Confirm: true,
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
Said: firstLine(r.err.Error())})
default:
raise, held := d.confirm.pass(ctx, d.keeper, p.ID, kindedAs(r.obs, p.Kind))
if err := d.keeper.Reconcile(ctx, p.ID, raise); err != nil {
v.Error = "what it found could not be kept: " + err.Error()
}
for _, o := range held {
v.Unconfirmed = append(v.Unconfirmed, o.Summary)
}
if len(raise) == 0 {
v.Verdict = verdictPass
run.Counts.Passed++
} else {
v.Verdict = verdictFail
run.Counts.Failed++
for _, o := range raise {
v.Found = append(v.Found, o.Summary)
}
}
}
if p.run != nil {
v.Took = r.took.Round(time.Millisecond).String()
}
run.Probes = append(run.Probes, v)
}
blind, _ = d.confirm.pass(ctx, d.keeper, sourceDoctor, blind)
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
}
ended := time.Now()
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
d.mu.Lock()
d.last, d.ended = &run, ended
d.mu.Unlock()
d.sayHeartbeat(ctx, run)
return run
}
// sourceDoctor is what raises a probe's own failure to run.
const sourceDoctor = "doctor"
// kindedAs gives each observation of a probe the probe's kind where it named none.
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
out := make([]conditions.Observation, 0, len(obs))
for _, o := range obs {
if o.Kind == "" {
o.Kind = kind
}
out = append(out, o)
}
return out
}
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
// says it outward: the watcher hears nothing.
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
if d.teller == nil {
return
}
body, err := json.Marshal(run)
if err != nil {
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
return
}
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
"will say the self-check is silent: %v\n", run.Run, err)
}
}
// doctorFrom is the serving controller's self-check; nil in any other process.
var doctorFrom *doctor
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
func doctorCommand(ctx context.Context, args []string) error {
sub := ""
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("doctor [run|probes|signals] [--json]")
}
answer, err := doctorAnswer(ctx, sub)
if err != nil {
return err
}
if *asJSON {
return printJSON(answer)
}
fmt.Print(doctorText(answer))
return nil
}
// doctorAnswer is what the verb answers, as data.
func doctorAnswer(ctx context.Context, sub string) (any, error) {
switch sub {
case "":
if doctorFrom != nil {
if run := doctorFrom.lastRun(); run != nil {
return verdictAnswer(*run, time.Now()), nil
}
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
}
run, err := lastHeartbeat(ctx)
if err != nil {
return nil, err
}
return verdictAnswer(run, time.Now()), nil
case "run":
d := doctorFrom
if d == nil {
local, closeIt, err := localDoctor(ctx)
if err != nil {
return nil, err
}
defer closeIt()
d = local
}
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
case "probes":
return probesAnswer(), nil
case "signals":
if doctorFrom == nil || doctorFrom.watchdogs == nil {
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
"hears them: ask it through the mesh-controller seat's doctor verb")
}
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
}
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
}
// verdictAnswer is a run as the verb answers it, with its age.
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it, " +
"and one a single look can be wrong about — an unanswered question, a slow answer — is raised when two " +
"runs in a row see it"}
}
// probesAnswer is the registry.
func probesAnswer() map[string]any {
var out []map[string]any
for _, p := range probeRegistry {
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
if len(p.Raises) > 0 {
row["raises"] = p.Raises
}
if p.Deferred != "" {
row["deferred"] = p.Deferred
}
out = append(out, row)
}
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
}
// signalsAnswer is every row of the signals table with the age of its newest signal.
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
var rows []map[string]any
for _, r := range signalsTable {
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
switch {
case r.Deferred != "":
row["deferred"] = r.Deferred
case f == nil:
row["newest"] = "not yet looked at"
default:
if err := r.needs(f); err != nil {
row["blind"] = err.Error()
} else if newest := r.newest(f); newest.IsZero() {
row["newest"] = "none heard"
} else {
row["newest"] = newest.UTC().Format(time.RFC3339)
row["age"] = f.now.Sub(newest).Round(time.Second).String()
}
}
rows = append(rows, row)
}
out := map[string]any{"signals": rows, "every": watchEvery.String()}
if !ticked.IsZero() {
out["looked"] = ticked.UTC().Format(time.RFC3339)
}
return out
}
// doctorText is an answer as a person reads it.
func doctorText(answer any) string {
body, _ := json.Marshal(answer)
var b strings.Builder
var verdict struct {
Run doctorRun `json:"run"`
Age string `json:"age"`
}
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
r := verdict.Run
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
for _, p := range r.Probes {
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
for _, f := range p.Found {
fmt.Fprintf(&b, " %s\n", f)
}
for _, f := range p.Unconfirmed {
fmt.Fprintf(&b, " unconfirmed, raised if the next run sees it too: %s\n", f)
}
if p.Error != "" {
fmt.Fprintf(&b, " %s\n", p.Error)
}
}
return b.String()
}
pretty, _ := json.MarshalIndent(answer, "", " ")
return string(pretty) + "\n"
}
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
// the serving controller answers `doctor` from.
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
var run doctorRun
err := onTheBus(func(conn *nats.Conn) error {
js, err := conn.JetStream(nats.Context(ctx))
if err != nil {
return err
}
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
if errors.Is(err, nats.ErrMsgNotFound) {
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
}
if err != nil {
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
}
return json.Unmarshal(msg.Data, &run)
})
return run, err
}
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
// its own connection, and its own keeper, closed after.
func localDoctor(ctx context.Context) (*doctor, func(), error) {
open, err := openStores(ctx)
if err != nil {
return nil, nil, err
}
js, err := dialTheBus()
if err != nil {
open.Close()
return nil, nil, err
}
k, err := keeperOn(ctx, js.Conn())
if err != nil {
js.Close()
open.Close()
return nil, nil, err
}
jsCtx := js.Context()
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
host: controlHost(ctx, open.inventory)}
return d, func() {
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
k.Close(flushing)
js.Close()
open.Close()
}, nil
}
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
func sortedFound(obs []conditions.Observation) []conditions.Observation {
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
return obs
}
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
type probeAsksKey struct{}
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
p, ok := ctx.Value(probeAsksKey{}).(probe)
if !ok {
return "a caller outside the self-check", false
}
for _, v := range p.Asks {
if v.Seat == seat && v.Verb == verb {
return p.ID, true
}
}
return p.ID, false
}
cmd/mesh-controller/doctor_test.go
-430
View File
@@ -1,430 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"net"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/net/dns/dnsmessage"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
// withProbes runs the test with a registry of its own.
func withProbes(t *testing.T, probes ...probe) {
t.Helper()
before := probeRegistry
probeRegistry = probes
t.Cleanup(func() { probeRegistry = before })
}
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
var broken atomic.Bool
broken.Store(true)
withProbes(t,
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
if broken.Load() {
return []conditions.Observation{failing}, nil
}
return nil, nil
}},
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
if broken.Load() {
return nil, errors.New("the store is away")
}
return nil, nil
}},
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
if broken.Load() {
<-ctx.Done()
time.Sleep(50 * time.Millisecond)
}
return nil, nil
}},
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
)
before := probeWithin
probeWithin = 200 * time.Millisecond
t.Cleanup(func() { probeWithin = before })
store := conditions.NewInMemory()
told := &conditions.Told{}
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
d := &doctor{keeper: k, teller: told, host: "anchor"}
// A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot
// run it either (novox/hq issue 277): one timed-out question is not a probe gone blind.
first := d.runOnce(t.Context(), "a test")
if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
t.Fatalf("counted %+v", first.Counts)
}
if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" {
t.Fatalf("after one run, open: %+v", open)
}
run := d.runOnce(t.Context(), "a test")
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
t.Fatalf("counted %+v", run.Counts)
}
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
var keys []string
for _, c := range open {
keys = append(keys, c.Key+"="+c.Kind)
}
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
"probe.P4.failed=probe-failed"} {
if !slices.Contains(keys, want) {
t.Errorf("%s is not open: %v", want, keys)
}
}
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent {
t.Fatalf("said %v", told.Names)
}
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
t.Fatal("the run is not the last verdict")
}
body, _ := json.Marshal(run)
var shape map[string]any
_ = json.Unmarshal(body, &shape)
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
if _, ok := shape[field]; !ok {
t.Errorf("the heartbeat carries no %q: %s", field, body)
}
}
// Mended: the next run clears every one of them.
broken.Store(false)
d.runOnce(t.Context(), "a test")
if open, _ := k.Open(t.Context()); len(open) != 0 {
t.Fatalf("a passing run left open %+v", open)
}
}
// **The registry says what each probe asserts**, and a probe not built says why and when.
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
seen := map[string]bool{}
for _, p := range probeRegistry {
if seen[p.ID] {
t.Errorf("%s twice", p.ID)
}
seen[p.ID] = true
if p.Asserts == "" || p.From == "" || p.Kind == "" {
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
}
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
}
}
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
if !seen[id] {
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
}
}
}
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
// stopped is S10 (signals_test.go).
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
w := &watchdogs{started: time.Now().Add(-time.Hour)}
d := &doctor{watchdogs: w, host: "anchor"}
got, err := probeWatchdogs(t.Context(), d)
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
t.Fatalf("%+v %v", got, err)
}
w.ticked = time.Now()
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
t.Fatalf("%+v", got)
}
}
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
open := aMesh(t)
got, err := probeDeclarations(t.Context(), &doctor{open: open})
if err != nil {
t.Fatal(err)
}
if len(got) != 0 {
t.Fatalf("a healthy mesh failed D1: %+v", got)
}
}
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
one, two := rivals()
register(t, open, one)
register(t, open, two)
for _, m := range []string{"rival-one", "rival-two"} {
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
t.Fatal(err)
}
}
got, err := probeDeclarations(ctx, &doctor{open: open})
linted(got)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
t.Fatalf("%+v", got)
}
}
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
answerAs := func(rcode dnsmessage.RCode) string {
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = conn.Close() })
go func() {
buf := make([]byte, 1500)
for {
n, from, err := conn.ReadFrom(buf)
if err != nil {
return
}
var q dnsmessage.Message
if q.Unpack(buf[:n]) != nil {
continue
}
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
if q.Questions[0].Type == dnsmessage.TypeA {
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
} else {
reply.RCode = rcode
}
packed, _ := reply.Pack()
_, _ = conn.WriteTo(packed, from)
}
}()
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
return port
}
before := resolverPort
t.Cleanup(func() { resolverPort = before })
resolverPort = answerAs(dnsmessage.RCodeSuccess)
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
t.Fatalf("%v %v %v", v4, rcode, err)
}
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
}
resolverPort = answerAs(dnsmessage.RCodeNameError)
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
t.Fatalf("NXDOMAIN read as %v", rcode)
}
}
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
url := testbus.URL(t)
open := aMesh(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
_ = js.Context().DeleteStream(s)
}
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
t.Fatal(err)
}
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
d := &doctor{open: open, js: js}
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
got, err := p(t.Context(), d)
if err != nil || len(got) != 0 {
t.Fatalf("a bus just raised fails: %+v %v", got, err)
}
}
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
t.Fatal(err)
}
info, err := js.Context().StreamInfo("EVENTS")
if err != nil {
t.Fatal(err)
}
cfg := info.Config
cfg.MaxMsgsPerSubject = 3
if _, err := js.Context().UpdateStream(&cfg); err != nil {
t.Fatal(err)
}
consumers, err := probeConsumers(t.Context(), d)
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
}
streams, err := probeStreams(t.Context(), d)
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
t.Fatalf("the redefined stream: %+v %v", streams, err)
}
}
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
heard := make(chan *nats.Msg, 16)
for _, subject := range broker.BusAdvisories {
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
t.Fatal(err)
}
}
api, _ := jetstream.New(conn)
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
if err != nil {
t.Fatal(err)
}
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
if err != nil {
t.Fatal(err)
}
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
t.Fatal(err)
}
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
t.Fatal(err)
}
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
time.Sleep(300 * time.Millisecond)
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
t.Fatal(err)
}
kinds := map[string]string{}
deadline := time.After(5 * time.Second)
for len(kinds) < 2 {
select {
case m := <-heard:
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
kinds[a.Kind] = a.Said
}
case <-deadline:
t.Fatalf("the bus said only %v", kinds)
}
}
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
t.Fatalf("%v", kinds)
}
}
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
// machine read as behind right after a push sent it the current build — it says the digest-named
// directory it runs from, and the mesh holds a commit).
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
}}
delivered := deliveredVersions(m)
if !slices.Equal(delivered, []string{"31045596c83a"}) {
t.Fatalf("%v", delivered)
}
commit := "1545b00a9f0c"
for _, c := range []struct {
reported string
behind bool
}{
{"31045596c83a", false}, // the live case: current, and was called behind
{"0123456789ab", true}, // another delivery
{"1545b00a", false}, // placed by hand, stamped with the commit
{"", false}, // not said
} {
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
}
}
if engineBehind("31045596c83a", nil, commit) {
t.Error("behind a mesh that holds no delivered build")
}
}
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
asked := 0
for _, p := range probeRegistry {
for _, v := range p.Asks {
asked++
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
}
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
}
}
}
if asked == 0 {
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
}
// And a probe asking what it did not declare is refused before anything is sent.
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
!strings.Contains(err.Error(), "does not declare") {
t.Fatalf("an undeclared question was asked: %v", err)
}
}
// subjectMatches is the bus's matching of a permission pattern against a subject.
func subjectMatches(pattern, subject string) bool {
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
for i, tok := range p {
if tok == ">" {
return len(s) > i
}
if i >= len(s) || (tok != "*" && tok != s[i]) {
return false
}
}
return len(p) == len(s)
}
cmd/mesh-controller/durations.go
-170
View File
@@ -1,170 +0,0 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
//
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
// build's outcome — and summarised here per machine, repository or module.
// recordBuildDuration measures one build from its ask to its outcome heard.
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
if asked.IsZero() || result.ID == "" {
return
}
subject := result.Module
if subject == "" {
subject = result.Repository
}
detail := "built"
if result.Failed != "" {
detail = "failed: " + firstLine(result.Failed)
}
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
}
}
// durationSummary is one subject's measurements of one kind.
type durationSummary struct {
Kind string `json:"kind"`
Subject string `json:"subject"`
Count int `json:"count"`
Median string `json:"median"`
P90 string `json:"p90"`
Max string `json:"max"`
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
// time: three times the slowest apply (S2), three times the median word interval (S1).
Suggests string `json:"suggests,omitempty"`
}
func summarise(ds []inventory.Duration) []durationSummary {
type key struct{ kind, subject string }
by := map[key][]time.Duration{}
for _, d := range ds {
k := key{d.Kind, d.Subject}
by[k] = append(by[k], d.Took)
}
var out []durationSummary
for k, took := range by {
slices.Sort(took)
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
switch k.kind {
case inventory.DurationApply:
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
case inventory.DurationHeartbeatGap:
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
}
out = append(out, s)
}
sort.Slice(out, func(i, j int) bool {
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
if ki != kj {
return ki < kj
}
return out[i].Subject < out[j].Subject
})
return out
}
func round(d time.Duration) string {
switch {
case d < time.Second:
return d.Round(time.Millisecond).String()
case d < time.Minute:
return d.Round(100 * time.Millisecond).String()
default:
return d.Round(time.Second).String()
}
}
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
func durationsCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("durations", flag.ContinueOnError)
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
days := set.Int("days", 14, "how many days back")
asJSON := set.Bool("json", false, "the summary as data")
all := set.Bool("all", false, "every measurement rather than the summary")
if _, err := parseAround(set, args); err != nil {
return err
}
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
if err != nil {
return err
}
if *all {
body, err := json.MarshalIndent(ds, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
summary := summarise(ds)
if *asJSON {
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
if len(summary) == 0 {
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
"plan-tier and build durations as it hears them\n", *days)
return nil
}
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
for _, s := range summary {
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
if s.Suggests != "" {
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
}
}
return nil
}
// forgettingOldDurations removes what is older than a month, at start and daily after.
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
for {
if n, err := inv.ForgetOldDurations(ctx); err != nil {
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
} else if n > 0 {
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
}
select {
case <-ctx.Done():
return
case <-time.After(24 * time.Hour):
}
}
}
cmd/mesh-controller/epoch_send_test.go
-115
View File
@@ -1,115 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said
// it reads one, and to no other: an older node-engine refuses a key it does not know, whole.
// bodiesDelivery records each send as the mesh does, and keeps the bodies.
type bodiesDelivery struct {
recordedDelivery
bodies map[string][]byte
}
func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
b.bodies[s.node] = body
return b.recordedDelivery.declare(ctx, s, body)
}
func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
epoch := uint64(57)
was := epochForActs
epochForActs = func(context.Context) (uint64, error) { return epoch, nil }
t.Cleanup(func() { epochForActs = was })
anchor, err := inv.NodeByName(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil {
t.Fatal(err)
}
carried := func(node string) (epoch float64, has bool) {
var envelope map[string]any
if err := json.Unmarshal(d.bodies[node], &envelope); err != nil {
t.Fatal(err)
}
epoch, has = envelope["epoch"].(float64)
return epoch, has
}
if e, has := carried("anchor"); !has || e != 57 {
t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"])
}
if _, has := carried("laptop"); has {
t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"])
}
// A new holder of the lease is not a change of the machine: neither reads as behind.
epoch = 58
would, err := wouldSend(ctx, open, mustNodes(t, open))
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
sent, err := inv.Outstanding(ctx, node)
if err != nil {
t.Fatal(err)
}
if would[node] != sent {
t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node)
}
}
}
// A process that may not act composes nothing and sends nothing: its number is not taken.
func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
was := epochForActs
epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }
t.Cleanup(func() { epochForActs = was })
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "")
if err != nil {
t.Fatal(err)
}
if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") {
t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused)
}
anchor, _ := open.inventory.NodeByName(ctx, "anchor")
if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 {
t.Fatalf("a controller without the lease took sequence %d", seq)
}
// And at the send itself: the gate every declaration passes.
gate := link.ActingGate
link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") }
t.Cleanup(func() { link.ActingGate = gate })
if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil ||
!strings.Contains(err.Error(), "lost the lease") {
t.Fatalf("a declaration was let through the gate: %v", err)
}
}
cmd/mesh-controller/facts.go
-645
View File
@@ -1,645 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/validate"
"github.com/novox/mesh-controller/internal/artifacts"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The facts snapshot (novox/hq to-be 45 §9, ADR 0227 rule 9): what a merge check needs to judge a change
// against the mesh that runs, written by the controller to the artifact store, where the build seat reads
// it. internal/facts says what it holds and what it never holds; this composes it from the store and
// keeps it current.
// factsEvery is how often the snapshot is composed. It is kept when it moved — a machine, an
// assignment, a seat, a setting, a build — or once a day when nothing did, so its age says the
// controller is still writing it (S14).
var factsEvery = 10 * time.Minute
// factsDaily is how old a snapshot of an unchanged mesh may grow before it is written again.
const factsDaily = 24 * time.Hour
// factsStaleAfter is S14's bound: a snapshot older than this is one no check should be fed.
const factsStaleAfter = 48 * time.Hour
// factsExport is what this controller knows of the snapshot it keeps: when the newest was taken, its
// content, and the last attempt's error.
type factsExport struct {
mu sync.Mutex
taken time.Time
content string
digest string
err error
began time.Time
}
// exportedFacts is this process's export, read by S14.
var exportedFacts = &factsExport{}
func (e *factsExport) last() (taken time.Time, digest string, began time.Time, err error) {
e.mu.Lock()
defer e.mu.Unlock()
return e.taken, e.digest, e.began, e.err
}
func (e *factsExport) kept(f snapshot.Facts, content, digest string) {
e.mu.Lock()
defer e.mu.Unlock()
e.taken, e.content, e.digest, e.err = f.Taken, content, digest, nil
}
func (e *factsExport) failed(err error) {
e.mu.Lock()
defer e.mu.Unlock()
e.err = err
}
// exportingFacts keeps the snapshot current for as long as this controller holds the lease: ctx ends
// when it stops acting.
func exportingFacts(ctx context.Context, open *stores, busVersion func() string) {
exportedFacts.mu.Lock()
exportedFacts.began = time.Now()
exportedFacts.mu.Unlock()
// What the store holds already, so a restarted controller neither writes an unchanged snapshot again
// nor reads its age as zero.
if address, err := factsStore(ctx, open); err == nil {
if body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag); err == nil {
if f, err := snapshot.Decode(body); err == nil {
content, _ := f.Content()
exportedFacts.kept(f, content, digest)
}
}
}
failing := ""
first := time.NewTimer(time.Minute)
defer first.Stop()
tick := time.NewTicker(factsEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-first.C:
case <-tick.C:
}
wrote, err := exportFacts(ctx, open, busVersion(), false)
why := ""
if err != nil {
why = err.Error()
exportedFacts.failed(err)
}
if why != failing {
if why != "" {
fmt.Printf("the facts snapshot cannot be kept: %s\n", why)
} else {
fmt.Println("the facts snapshot is kept again")
}
failing = why
}
if wrote != "" {
fmt.Printf("the facts snapshot moved and is kept as %s\n", short(strings.TrimPrefix(wrote, "sha256:")))
}
}
}
// exportFacts composes the snapshot and keeps it when it moved, or when the one kept is a day old, or
// when told to. Answers the digest it kept, empty when it kept nothing.
func exportFacts(ctx context.Context, open *stores, busVersion string, force bool) (string, error) {
f, err := gatherFacts(ctx, open, busVersion)
if err != nil {
return "", err
}
content, err := f.Content()
if err != nil {
return "", err
}
exportedFacts.mu.Lock()
unchanged := content == exportedFacts.content && time.Since(exportedFacts.taken) < factsDaily
exportedFacts.mu.Unlock()
if unchanged && !force {
return "", nil
}
body, err := f.Encode()
if err != nil {
return "", err
}
address, err := factsStore(ctx, open)
if err != nil {
return "", err
}
digest, err := (artifacts.Store{Address: address}).PutTagged(ctx, snapshot.Repository, snapshot.Tag, snapshot.MediaType, body)
if err != nil && digest == "" {
return "", err
}
exportedFacts.kept(f, content, digest)
return digest, err
}
// factsStore is the artifact store as this controller reaches it.
func factsStore(ctx context.Context, open *stores) (string, error) {
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
}
address, err := artifactStoreAddress(ctx, open.inventory, shelf, "")
if err != nil {
return "", err
}
if address == "" {
return "", errors.New("the artifact store is not on the private network, so there is nowhere to keep the facts")
}
return address, nil
}
// gatherFacts composes one snapshot from the store: read only, nothing made, nothing sent.
func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot.Facts, error) {
inv := open.inventory
f := snapshot.Facts{Format: snapshot.Format, Taken: time.Now().UTC(), Controller: snapshot.Build{Version: version}}
f.Versions.Bus = busVersion
storeVersion, err := inv.ServerVersion(ctx)
if err != nil {
return snapshot.Facts{}, fmt.Errorf("the store will not say its version: %w", err)
}
f.Versions.Store = storeVersion
nodes, err := inv.Nodes(ctx)
if err != nil {
return snapshot.Facts{}, err
}
overlays, err := inv.Overlays(ctx)
if err != nil {
return snapshot.Facts{}, err
}
place := map[string]inventory.Overlay{}
for _, o := range overlays {
place[o.Name] = o
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return snapshot.Facts{}, err
}
shelf := map[string]catalogue.Manifest{}
for _, e := range entries {
shelf[e.Manifest.Module] = e.Manifest
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
return snapshot.Facts{}, err
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return snapshot.Facts{}, err
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return snapshot.Facts{}, err
}
holdings, err := inv.Holdings(ctx)
if err != nil {
return snapshot.Facts{}, err
}
// **Every name first**, so text read afterwards — a setting naming a machine, a problem naming a
// site — has it replaced wherever it appears.
scrub := snapshot.NewScrubber()
domains := map[string]string{}
for _, n := range nodes {
scrub.Machine(n.Name)
if n.Account != "" && n.Account != "root" {
scrub.Account(n.Account)
}
d, err := inv.PublicDomainOf(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
domains[n.Name] = scrub.Domain(d)
}
for _, o := range overlays {
scrub.Site(o.Site)
}
// What a merge check runs in and reads beside it, as the build seat would be asked for it.
if held, err := inv.Held(ctx); err == nil {
for language, reference := range builder.ToolchainsOf(held) {
if f.Versions.Toolchains == nil {
f.Versions.Toolchains = map[string]string{}
}
f.Versions.Toolchains[language] = catalogue.Recorded(reference)
}
} else {
return snapshot.Facts{}, err
}
for _, e := range entries {
if dir, core := coreModules[e.Manifest.Module]; core && !e.Provided && e.Source.Repository != "" {
for d, ref := range besideRefs(dir, current[e.Manifest.Module].Commit) {
if f.Beside == nil {
f.Beside = map[string]string{}
}
f.Beside[d] = ref
}
}
}
if c, ok := current["mesh-controller"]; ok {
f.Controller.Commit = c.Commit
}
gens, gensErr := generators(ctx, open)
hostShelf := shelf[hostModule]
meshWide := map[string]bool{}
engines := map[string]bool{}
for _, n := range nodes {
m := snapshot.Machine{Name: scrub.Machine(n.Name), Length: len(n.Name), Adopted: n.Adopted,
AccountHome: scrub.Text(n.AccountHome), NodeEngine: n.HostVersion, PublicDomain: domains[n.Name]}
switch n.Account {
case "", "root":
m.Account = n.Account
default:
m.Account = scrub.Account(n.Account)
}
if n.HostVersion != "" {
engines[n.HostVersion] = true
}
m.System = systemOf(hostShelf, n.HostVersion)
m.Libc = libcOf(m.System)
reported, err := inv.DescribedOf(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
m.Architecture, m.Kernel = reported.Architecture, reported.Kernel
outward, err := inv.OutwardLinksOf(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
for _, l := range outward {
m.OutwardLinks = append(m.OutwardLinks, scrub.Text(l))
}
capabilities, err := inv.Profile(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
for _, c := range capabilities {
kept := snapshot.Capability{Name: c.Name, Present: c.Present}
// The detail only where it is a version: everything else a detector says — a ruleset, a
// device, a path — is the machine's own business and no check reads it.
if c.Present && (c.Name == "container-runtime" || c.Name == "package-manager") {
kept.Detail = scrub.Text(c.Detail)
}
m.Capabilities = append(m.Capabilities, kept)
}
if o, ok := place[n.Name]; ok {
m.Site, m.Hub, m.Public, m.OnNetwork = scrub.Site(o.Site), o.Hub, o.Endpoint != "", o.Address != ""
}
assigned, err := inv.Assigned(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
m.Assigned = assigned
sent, known, err := inv.SentBuilds(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
if known && slices.Contains(assigned, broker.RuntimeModule) {
m.NodeTools = sent[broker.RuntimeModule]
}
pins, err := inv.PinsFor(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
for provision, c := range pins {
m.Pins = append(m.Pins, snapshot.Pin{Provision: provision, Machine: scrub.Machine(c.Node), Module: c.Module})
}
for _, module := range assigned {
held, err := inv.SecretsOf(ctx, n.Name, module)
if err != nil {
return snapshot.Facts{}, err
}
for _, h := range held {
if h.Origin == inventory.OriginAccepted {
m.Accepted = append(m.Accepted, snapshot.Accepted{Module: module, Name: h.Name,
Provider: scrub.Machine(h.Provider), Local: h.Local})
}
}
layers, err := inv.SettingsFor(ctx, n.Name, module)
if err != nil {
return snapshot.Facts{}, err
}
for _, layer := range layers {
if layer.From == catalogue.MeshWideLayer {
if !meshWide[module] {
meshWide[module] = true
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
}
continue
}
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
}
}
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
if ctx.Err() != nil {
return snapshot.Facts{}, ctx.Err()
}
f.Machines = append(f.Machines, m)
}
for e := range engines {
f.Versions.NodeEngines = append(f.Versions.NodeEngines, e)
}
// Seats and their holders, and from them the roles a machine is named by.
roles := map[string][]string{}
seats := map[string]*snapshot.Seat{}
for _, h := range holdings {
key := h.Claim + "\x00" + h.Scope
s, ok := seats[key]
if !ok {
s = &snapshot.Seat{Name: h.Claim, Scope: h.Scope}
seats[key] = s
}
s.Holders = append(s.Holders, snapshot.Holder{Machine: scrub.Machine(h.Node), Module: h.Module})
if h.Scope == catalogue.ScopeMesh {
role := "holds " + h.Claim
if h.Claim == catalogue.ControllerSeatName {
role = "the control node"
}
roles[h.Node] = append(roles[h.Node], role)
}
}
for _, s := range seats {
f.Seats = append(f.Seats, *s)
}
for i := range f.Machines {
for _, n := range nodes {
if scrub.Machine(n.Name) != f.Machines[i].Name {
continue
}
f.Machines[i].Roles = roles[n.Name]
if f.Machines[i].Hub {
f.Machines[i].Roles = append(f.Machines[i].Roles, "the hub")
}
}
}
// Every module, as the mesh holds it, and where it is built from.
newest := map[string]inventory.Source{}
count := map[string]int{}
for _, e := range entries {
raw, err := json.Marshal(e.Manifest)
if err != nil {
return snapshot.Facts{}, err
}
mod := snapshot.Module{Name: e.Manifest.Module, Repository: e.Source.Repository, Path: e.Source.Path,
Commit: e.Source.BuiltFrom, Provided: e.Provided, RollOut: current[e.Manifest.Module].RollOut,
Manifest: raw}
for _, r := range read[e.Manifest.Module] {
mod.Reads = append(mod.Reads, r.Repository)
}
f.Modules = append(f.Modules, mod)
if e.Provided || e.Source.Repository == "" {
continue
}
count[e.Source.Repository]++
if was, ok := newest[e.Source.Repository]; !ok || e.Source.Seen.After(was.Seen) {
newest[e.Source.Repository] = e.Source
}
}
for repository, s := range newest {
commit := s.Head
if commit == "" {
commit = s.BuiltFrom
}
f.Sources = append(f.Sources, snapshot.Source{Repository: repository, Commit: commit, Modules: count[repository]})
}
for _, e := range edges {
f.Edges = append(f.Edges, snapshot.Edge{From: e.From, To: e.To, Kind: e.Kind})
}
f.Sorted()
return f, nil
}
// declarationFacts is how one machine's declaration composes now, as the next push would compose it and
// without making anything (D1's composition), and whether the node-engine's validator takes it.
func declarationFacts(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
gensErr error, scrub *snapshot.Scrubber) snapshot.Declaration {
var d snapshot.Declaration
if gensErr != nil {
d.Problems = []string{scrub.Text("the private network cannot be computed: " + oneLine(gensErr.Error()))}
return d
}
declared, problems, err := composedAndValidated(ctx, open, node, gens, Foreseeing)
if err != nil {
d.Problems = []string{scrub.Text(oneLine(err.Error()))}
return d
}
for _, p := range problems {
d.Problems = append(d.Problems, scrub.Text(p))
}
d.Composes = len(problems) == 0
if body, err := declared.Body(); err == nil {
d.Digest = fmt.Sprintf("sha256:%x", sha256.Sum256(body))
}
// Scrubbed like every other word: a resource is named after the machine a grant is for.
for _, r := range resourceNames(declared.Resources) {
d.Resources = append(d.Resources, scrub.Text(r))
}
for module, why := range declared.leftOutWhy {
if d.LeftOut == nil {
d.LeftOut = map[string]string{}
}
d.LeftOut[module] = scrub.Text(why)
}
for _, o := range declared.withheld {
d.Withheld = append(d.Withheld, scrub.Text(o.String()))
}
for _, u := range declared.unbound {
d.Unbound = append(d.Unbound, scrub.Text(u.String()))
}
sort.Strings(d.Withheld)
sort.Strings(d.Unbound)
return d
}
// composedAndValidated composes one machine's declaration — as a push would (Allocating) or as the next
// push will without making anything (Foreseeing) — with the order it was last sent, and runs the
// node-engine's own validator over the body. An error is that it did not compose; problems are what the
// validator refuses.
func composedAndValidated(ctx context.Context, open *stores, node string, gens map[string]catalogue.Generator,
choosing Choosing) (sendable, []string, error) {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
return sendable{}, nil, err
}
declared, err := declarationWith(ctx, open, node, plan, settings, gens, choosing)
if err != nil {
return sendable{}, nil, err
}
record, err := open.inventory.NodeByName(ctx, node)
if err != nil {
return sendable{}, nil, err
}
if declared.Sequence, err = open.inventory.Sequence(ctx, record.ID); err != nil {
return sendable{}, nil, err
}
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
return sendable{}, nil, err
}
body, err := declared.Body()
if err != nil {
return sendable{}, nil, err
}
return declared, validate.Declaration(body), nil
}
// resourceNames are a declaration's resources as `type:id`, sorted.
func resourceNames(resources []map[string]any) []string {
out := make([]string, 0, len(resources))
for _, r := range resources {
kind, _ := r["type"].(string)
id, _ := r["id"].(string)
out = append(out, kind+":"+id)
}
sort.Strings(out)
return out
}
// systemOf is the system a node-engine of that version was built for, read from the build the mesh
// holds: the artifact a resource delivered into `versions/<version>` came from is named for its system
// (`host-arch`). Empty when the version is not a delivered one — an engine placed by hand.
func systemOf(host catalogue.Manifest, version string) string {
if version == "" {
return ""
}
for _, r := range host.Resources {
path, _ := r["path"].(string)
if !strings.HasSuffix(path, "/versions/"+version) {
continue
}
source, _ := r["source"].(string)
for _, part := range strings.Split(source, "/") {
if system, ok := strings.CutPrefix(part, "host-"); ok && system != "" {
return system
}
}
}
return ""
}
// libcOf is the C library of a system the node-engine is built for.
func libcOf(system string) string {
switch system {
case "arch":
return "glibc"
case "alpine":
return "musl"
case "android":
return "bionic"
}
return ""
}
// factsCommand is `facts`: what the controller keeps, and keeping it now.
//
// facts the snapshot the artifact store holds: when, which, how many machines
// facts show the same, whole, as JSON
// facts export compose and keep one now
// facts compose compose one and print it, keeping nothing
func factsCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("facts", flag.ContinueOnError)
rest, err := parseAround(set, args)
if err != nil {
return err
}
what := ""
if len(rest) > 0 {
what = rest[0]
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
switch what {
case "", "show":
address, err := factsStore(ctx, open)
if err != nil {
return err
}
body, digest, err := (artifacts.Store{Address: address}).GetTagged(ctx, snapshot.Repository, snapshot.Tag)
if err != nil {
return err
}
if what == "show" {
_, err := os.Stdout.Write(body)
return err
}
f, err := snapshot.Decode(body)
if err != nil {
return err
}
fmt.Printf("the facts snapshot kept as %s:%s is %s, taken %s (%s ago) by controller %s\n",
snapshot.Repository, snapshot.Tag, short(strings.TrimPrefix(digest, "sha256:")),
f.Taken.Format(time.RFC3339), ago(time.Since(f.Taken)), orNone(f.Controller.Commit))
fmt.Printf(" %d machine(s), %d module(s), %d seat(s); the longest machine name is %d characters\n",
len(f.Machines), len(f.Modules), len(f.Seats), f.Longest())
fmt.Printf(" the bus runs %s, the store %s\n", orNone(f.Versions.Bus), orNone(f.Versions.Store))
for _, m := range f.Machines {
state := "composes"
if !m.Declaration.Composes {
state = "does NOT compose: " + strings.Join(m.Declaration.Problems, "; ")
}
fmt.Printf(" %-12s %s; %d module(s); %s\n", m.Name, m.Described(), len(m.Assigned), state)
}
return nil
case "export", "compose":
busVersion := ""
if server, err := connectLink(ctx, nil, nil, nil); err == nil {
busVersion = busVersionOf(server)
server.Close()
}
if what == "compose" {
f, err := gatherFacts(ctx, open, busVersion)
if err != nil {
return err
}
body, err := f.Encode()
if err != nil {
return err
}
_, err = os.Stdout.Write(append(body, '\n'))
return err
}
digest, err := exportFacts(ctx, open, busVersion, true)
if err != nil {
return err
}
fmt.Printf("the facts snapshot is kept as %s:%s, %s\n", snapshot.Repository, snapshot.Tag, digest)
return nil
}
return fmt.Errorf("facts [show|export|compose], not %q", what)
}
// busVersionOf is the bus server's release, as it told this connection.
func busVersionOf(server *link.Server) string {
if bus, ok := server.Bus().(link.OverNATS); ok && bus.Conn != nil {
return bus.Conn.ConnectedServerVersion()
}
return ""
}
cmd/mesh-controller/facts_test.go
-137
View File
@@ -1,137 +0,0 @@
package main
import (
"regexp"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
)
// The facts snapshot (novox/hq to-be 45 §9): composed from the store, every machine under a pseudonym
// of its name's length, and nothing of the installation in it — no secret, no address, no name.
// aMeshWithSecrets is aMesh with a provider and its consumers, a value given by hand, settings carrying
// a password, an address and a machine's name, and a push's worth of credentials made.
func aMeshWithSecrets(t *testing.T) (*stores, []string) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"},
Resources: []map[string]any{{"id": "config", "type": "file", "path": "/etc/files/config.json",
"mode": "0600", "content": "{}", "merge": "json"}}})
for _, a := range [][2]string{{"anchor", "objects"}, {"laptop", "files"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
secrets := []string{"Hunter2-Is-Not-A-Password-9f8e7d", "0123456789abcdefABCDEF0123456789zz"}
if err := open.inventory.SetSettings(ctx, "", "files", map[string]any{
"admin_password": secrets[0], "upstream": "10.77.0.9", "hub": "anchor"}); err != nil {
t.Fatal(err)
}
if err := open.inventory.SetSettings(ctx, "laptop", "files", map[string]any{
"note": "reach me at 192.168.1.135, token " + secrets[1]}); err != nil {
t.Fatal(err)
}
// A push's worth of composition, which makes the pair credential the consumer is sent.
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"laptop", "anchor"} {
if _, _, err := composedAndValidated(ctx, open, node, gens, Allocating); err != nil {
t.Fatalf("%s does not compose: %v", node, err)
}
}
issued, err := open.inventory.SecretsFrom(ctx, "anchor")
if err != nil || len(issued) == 0 {
t.Fatalf("no credential was made for the consumer: %v", err)
}
for _, s := range issued {
// Sealed, never kept plain (ADR 0004): the sealed blobs are what the store holds, and none may leave.
secrets = append(secrets, s.ForConsumer, s.ForProvider)
}
return open, secrets
}
func TestTheFactsCarryNoSecretNoAddressAndNoName(t *testing.T) {
open, secrets := aMeshWithSecrets(t)
f, err := gatherFacts(t.Context(), open, "2.11.17")
if err != nil {
t.Fatal(err)
}
body, err := f.Encode()
if err != nil {
t.Fatal(err)
}
text := string(body)
for _, s := range secrets {
if s != "" && strings.Contains(text, s) {
t.Errorf("a secret is in the snapshot: %q", s)
}
}
for _, leaked := range []string{"anchor", "laptop", "10.77.0.", "192.168.1.135", ".example:51820"} {
if strings.Contains(text, leaked) {
t.Errorf("%q is in the snapshot", leaked)
}
}
// Every address it carries is a documentation address.
for _, a := range regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b`).FindAllString(text, -1) {
if !strings.HasPrefix(a, "192.0.2.") && !strings.HasPrefix(a, "198.51.100.") && !strings.HasPrefix(a, "203.0.113.") {
t.Errorf("%s is an address outside the documentation ranges", a)
}
}
// What a check needs is there: every machine, its length, its modules, its declaration composing.
if len(f.Machines) != 2 || f.Longest() != len("laptop") {
t.Fatalf("machines %+v, longest %d", f.Machines, f.Longest())
}
anchor := snapshot.Pseudonym("machine", "anchor")
m, ok := f.Machine(anchor)
if !ok || !m.Hub || !strings.Contains(m.Described(), "the hub") || len(m.Name) != len("anchor") {
t.Fatalf("the anchor reads as %+v", m)
}
if !m.Declaration.Composes || m.Declaration.Digest == "" || len(m.Declaration.Resources) == 0 {
t.Errorf("the anchor's declaration reads as %+v", m.Declaration)
}
laptop, _ := f.Machine(snapshot.Pseudonym("machine", "laptop"))
if strings.Join(laptop.Assigned, ",") != "files,mesh-wireguard" && !strings.Contains(strings.Join(laptop.Assigned, ","), "files") {
t.Errorf("the laptop's assignments read as %v", laptop.Assigned)
}
var meshWide map[string]any
for _, s := range f.Settings {
if s.Module == "files" {
meshWide = s.Values
}
}
if meshWide["admin_password"] != snapshot.Withheld || meshWide["hub"] != anchor {
t.Errorf("the mesh-wide settings read as %v", meshWide)
}
if f.Versions.Bus != "2.11.17" || f.Versions.Store == "" {
t.Errorf("versions read as %+v", f.Versions)
}
var objects bool
for _, mod := range f.Modules {
objects = objects || mod.Name == "objects" && len(mod.Manifest) > 0
}
if !objects {
t.Error("the modules the mesh holds are not in the snapshot")
}
// And an unchanged mesh is the same content a moment later.
again, err := gatherFacts(t.Context(), open, "2.11.17")
if err != nil {
t.Fatal(err)
}
a, _ := f.Content()
b, _ := again.Content()
if a != b {
t.Error("two snapshots of an unchanged mesh differ, so it would be written again every ten minutes")
}
}
cmd/mesh-controller/gate.go
-942
View File
@@ -1,942 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/micro"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0236, to-be 45
// §8, ADR 0227 rule 8).
//
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
// the component's health — the core's health definitions, or a module's own — passing three times over
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
//
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
// build goes back to the one the first machine ran before, and that machine is sent it: the ordinary
// path, again. Once per build: the verdict is written before the send, and a verdict once written is
// not written over. Said as a condition, urgent for the core and when it could not be put back, and as
// the event `rolled-back`.
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
var (
// gateSettle is how long the first machine must stay healthy, at the least.
gateSettle = 2 * time.Minute
// gateBound is how long after the apply the build may take to become healthy.
gateBound = 10 * time.Minute
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
gatePasses = 3
gateEvery = 40 * time.Second
)
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
const gateProbe = "DG"
// The kinds a gate raises.
const (
kindRolledBack = "rolled-back"
kindRollbackFailed = "rollback-failed"
)
// coreComponent is the core component a module is, as a witness names it — controller, node-engine,
// node-tools — or empty: the core is judged by its health definitions, anything else by its own health.
func coreComponent(module string) string {
switch module {
case catalogue.ControllerSeatName:
return lease.ComponentController
case hostModule:
return lease.ComponentEngine
case broker.RuntimeModule:
return lease.ComponentNodeTools
}
return ""
}
// health is a judging's word on one machine.
type health int
const (
healthGood health = iota
healthNotYet
healthBroken
)
// served is what one machine's node tools answered the bus's discovery with.
type served struct {
runtime bool
tools map[string]bool
}
// gateFacts is what one judging reads, gathered once for every machine it judges.
type gateFacts struct {
now time.Time
reports map[string]inventory.Reported
// engines is each machine's node-engine build as it last reported it.
engines map[string]string
// served is what the bus's discovery answered; servedErr why it could not be asked.
served map[string]served
servedErr error
// open are the open conditions; openErr why they could not be read (nil keeper: not judged).
open []conditions.Condition
openErr error
judged bool
// rolledBack is what each machine's witnesses say they decided.
rolledBack map[string][]lease.Rollback
// holder is who holds the controller lease, for judging the controller.
holder *lease.Holder
holderErr error
}
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
// variable so a test can hand a judging its facts.
var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
inv := open.inventory
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: witnessed.all()}
reports, err := inv.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
f.reports[r.Node] = r
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return f, err
}
for _, n := range nodes {
f.engines[n.Name] = n.HostVersion
}
if d := doctorFrom; d != nil {
if d.keeper != nil {
f.judged = true
f.open, f.openErr = d.keeper.Open(ctx)
}
if d.js != nil {
f.served, f.servedErr = servedOnTheBus(ctx, d.js.Conn())
} else {
f.servedErr = errors.New("this controller has no bus to ask")
}
} else {
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
}
if theLease != nil {
h, found, err := theLease.holder(ctx)
switch {
case err != nil:
f.holderErr = err
case found:
f.holder = &h
}
if component != lease.ComponentController && f.holderErr != nil {
f.holderErr = nil // read only for the controller's own judging
}
}
return f, nil
}
// judgeHealth is one machine's health for a module's new build, from what one judging read: healthy,
// not yet (with what is wanting), or healthBroken — a witness put it back, or the machine refused or failed
// what it was sent. Pure.
func judgeHealth(module, component string, m catalogue.Manifest, machine string, since time.Time, f gateFacts) (health, string) {
// A witness's verdict made since the build was sent: the build failed its health there. One that
// could not judge at all (unwitnessed) is not a verdict on the build.
for _, r := range f.rolledBack[machine] {
if component == "" || r.Component != component || r.Outcome == lease.OutcomeUnwitnessed || r.At.Before(since) {
continue
}
return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component,
short(r.From), r.Outcome, r.Why)
}
r, said := f.reports[machine]
switch {
case !said || r.At == nil || !r.Current:
return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine)
case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused:
return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome)
case r.Outcome != inventory.OutcomeApplied:
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
}
// **No new condition about it**: about the machine itself, or naming the module on that machine,
// raised since the judging began. The gate's own are not evidence about the build.
if f.judged {
if f.openErr != nil {
return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " +
firstLine(f.openErr.Error())
}
for _, c := range f.open {
if c.Source == gateProbe || c.Raised.Before(since) {
continue
}
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
(c.Subject.Scope == conditions.ScopeMachine && c.Subject.ID == machine)
if !onIt {
continue
}
if c.Subject.Scope == conditions.ScopeMachine || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
}
}
}
switch component {
case lease.ComponentEngine:
// The node-engine has reported its current declaration under its own build.
want := deliveredVersions(m)
if len(want) > 0 && !slices.Contains(want, f.engines[machine]) {
return healthNotYet, fmt.Sprintf("%s's node-engine reports build %s, not the new %s", machine,
orNotKnown(f.engines[machine]), strings.Join(want, " or "))
}
case lease.ComponentNodeTools:
// The node tools are announced and answer.
if f.servedErr != nil {
return healthNotYet, "whether the node tools answer cannot be asked: " + firstLine(f.servedErr.Error())
}
if !f.served[machine].runtime {
return healthNotYet, fmt.Sprintf("the node tools on %s do not answer the bus", machine)
}
case lease.ComponentController:
// The new controller holds the lease and says it is ready.
switch {
case f.holderErr != nil:
return healthNotYet, "who holds the controller lease cannot be read: " + firstLine(f.holderErr.Error())
case f.holder == nil:
return healthNotYet, "no controller holds the lease"
case f.holder.Taken.Before(since.Add(-time.Minute)):
return healthNotYet, fmt.Sprintf("the lease is held since %s, by a controller older than the new build",
f.holder.Taken.UTC().Format(time.RFC3339))
case f.holder.Health == nil || !f.holder.Health.Ready:
why := "the controller holding the lease does not say it is ready"
if f.holder.Health != nil && f.holder.Health.Why != "" {
why += ": " + f.holder.Health.Why
}
return healthNotYet, why
}
default:
// A module's tools answer, where it has any and the machine runs the node tools that serve them.
if len(m.Tools) > 0 {
if f.servedErr != nil {
return healthNotYet, "whether its tools are served cannot be asked: " + firstLine(f.servedErr.Error())
}
if s := f.served[machine]; s.runtime && !s.tools[module] {
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
}
}
}
return healthGood, ""
}
// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is
// the judging's facts without those conditions, on what each names among the modules the send moved,
// and whole what holds the machine back as a whole.
type machineWord struct {
facts gateFacts
on map[string]string
whole string
}
// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
// yet reported applying the node tools it was last sent.
const kindCoreBehind = "core-behind"
// aboutTheMachine sorts the conditions raised about a machine itself since a send was made there
// (novox/hq issue 281). The gate read every one of them as the module's it was kept on: a machine-level
// condition caused by anything else in the send — or by a tier sent one module at a time — failed that
// module, at the bound, with a reason that was never about it.
//
// - one naming a module the send moved is that module's;
// - the core being behind (D10) is the core's: of the node-engine or the node tools when the send
// moved them — inside their settle window, which is the gate's bound — and otherwise no evidence about
// what was sent: a send not yet applied the machine's reports already say, and a newer core build
// that no plan sends is not this send's;
// - anything else holds the machine back as a whole: everything the send moved there waits on it, and
// fails with it, together, at the bound, with that reason.
//
// Pure.
func aboutTheMachine(machine string, moved []string, since time.Time, f gateFacts) machineWord {
w := machineWord{facts: f, on: map[string]string{}}
if !f.judged || f.openErr != nil {
return w
}
var core []string
for _, m := range moved {
if c := coreComponent(m); c == lease.ComponentEngine || c == lease.ComponentNodeTools {
core = append(core, m)
}
}
kept := make([]conditions.Condition, 0, len(f.open))
for _, c := range f.open {
aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine ||
slices.Contains(c.Subject.Also, machine))
if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) {
kept = append(kept, c)
continue
}
said := fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
parts := strings.Split(c.Subject.ID, ".")
var named []string
for _, m := range moved {
if slices.Contains(parts, m) {
named = append(named, m)
}
}
coreBehind := c.Kind == kindCoreBehind || strings.HasSuffix(c.Key, "."+kindCoreBehind)
switch {
case len(named) > 0:
for _, m := range named {
if _, already := w.on[m]; !already {
w.on[m] = said
}
}
case coreBehind && len(core) > 0:
for _, m := range core {
if _, already := w.on[m]; !already {
w.on[m] = said + " (its settle window runs to the gate's bound)"
}
}
case coreBehind:
// Not what the send moved: said nowhere against it.
default:
if w.whole == "" {
w.whole = machine + " as a whole: " + said
}
}
}
w.facts.open = kept
return w
}
// judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass
// counted, a pass missed (and why), or the verdict. Answers the verdict once there is one.
func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
running []string, now time.Time) (string, error) {
g := state.Gate
if g == nil {
// Judged from the send: a witness's verdict, a condition, the bound — all counted from when the
// first machine was sent the build.
start := now
if state.FirstAt != nil {
start = *state.FirstAt
}
var machines []string
for _, n := range state.First {
if slices.Contains(running, n) {
machines = append(machines, n)
}
}
g = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, From: state.Previous,
To: state.Commit, Since: &start}
state.Gate = g
}
pairs := []judged{}
for _, n := range g.Machines {
pairs = append(pairs, judged{module: module, node: n})
}
return judgeMoves(ctx, open, g, pairs, now)
}
// judged is one module on one machine, as a gate judges it.
type judged struct{ module, node string }
// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and
// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is
// wanting, and which modules), or the verdict. Answers the verdict once there is one.
func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) {
if g.Verdict != "" {
return g.Verdict, nil
}
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
return "", nil
}
for _, c := range g.Carried {
if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) {
pairs = append(pairs, judged{module: c.Module, node: c.Node})
}
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
}
facts, err := gatherGateFacts(ctx, open, g.Component)
if err != nil {
return "", err
}
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
// gate happens to be kept on.
byMachine := map[string][]string{}
for _, j := range pairs {
byMachine[j.node] = append(byMachine[j.node], j.module)
}
words := map[string]machineWord{}
for node, moved := range byMachine {
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
}
worst, why := healthGood, ""
var failing []string
broken := map[string]bool{}
for _, j := range pairs {
w := words[j.node]
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
if h == healthGood {
if on, named := w.on[j.module]; named {
h, said = healthNotYet, on
} else if w.whole != "" {
h, said = healthNotYet, w.whole
}
}
if h != healthGood && !slices.Contains(failing, j.module) {
failing = append(failing, j.module)
}
if h == healthBroken {
broken[j.module] = true
}
if h > worst {
worst, why = h, said
} else if h == worst && h != healthGood && why == "" {
why = said
}
}
switch {
case worst == healthBroken:
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
g.Failing = failing
decide(g, inventory.GateFailed, why, now)
case worst == healthNotYet:
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
if now.Sub(*g.Since) > gateBound {
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
}
default:
g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second)), now)
}
}
return g.Verdict, nil
}
// decide sets a gate's verdict.
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
g.Took = now.Sub(*g.Since).Round(time.Second).String()
}
// gatePassed keeps a passing build's verdict, so `plans` and the gate's probe can read it.
func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule) {
g := state.Gate
err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
if err != nil && state.Build != "" {
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
}
passCarried(ctx, open, p, g, module)
fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module,
strings.Join(g.Machines, ", "), g.Why)
if module == catalogue.ControllerSeatName {
carryUserList(ctx, open, p)
}
}
// carryUserList sends the machine holding the bus the user list a new controller composes, once that
// controller passed its gate (ADR 0236). The controller's own grants travel in that list, and the old
// controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new
// controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR
// 0221): then it is said, as a push would say it.
func carryUserList(ctx context.Context, open *stores, p *inventory.Plan) {
holder, behind, err := brokerBehind(ctx, open, nil)
if err != nil || holder == "" || !behind {
if err != nil {
fmt.Printf("%s: whether the bus's user list is behind the new controller cannot be read: %v\n", p.ID, err)
}
return
}
held, err := heldMachines(ctx, open, []string{holder})
if err != nil {
fmt.Printf("%s: whether %s may be sent the new user list cannot be read: %v\n", p.ID, holder, err)
return
}
if why, isHeld := held[holder]; isHeld {
fmt.Printf("%s: the new controller's user list is not carried to %s, which holds the bus: %s — `push %s` "+
"carries it\n", p.ID, holder, strings.Join(why, "; "), holder)
return
}
if _, err := sendRollout(ctx, open, []string{holder}); err != nil {
fmt.Printf("%s: the new controller's user list could not be carried to %s: %v\n", p.ID, holder, err)
return
}
fmt.Printf("%s: carried the new controller's user list to %s, which holds the bus\n", p.ID, holder)
}
// gateFailed stops the plan at a build that failed its gate and puts the previous build back on the
// machines it was judged on — once per build, said as a condition and an event. The plan is saved
// before the send: a controller that is itself the build being put back does not outlive it.
func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
machines []string, why string) {
inv := open.inventory
now := time.Now().UTC()
if state.Gate == nil {
state.Gate = &inventory.PlanGate{Component: coreComponent(module), Machines: machines,
From: state.Previous, To: state.Commit, Since: state.FirstAt}
}
g := state.Gate
if g.Verdict == "" {
if g.Since == nil {
g.Since = &now
}
decide(g, inventory.GateFailed, why, now)
}
if len(g.Machines) == 0 {
g.Machines = machines
}
state.Why = "failed its gate: " + g.Why
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s failed its gate on %s in tier %d: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why)
verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous,
Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why,
Component: g.Component, JudgingFrom: g.Since}
// **A send that changed nothing of the module there is no verdict on its build** (novox/hq issue
// 280). The machine already ran this build, or one that made the same artifacts from the same
// manifest: whatever the gate found wanting, this build did not bring it, and there is nothing to
// put back. On 2026-10-06 such a module was marked failed, and the rollback looked for an earlier
// build of the very commit it had failed — "no build kept" — while the build it had run before was
// kept all along. Said, left as it is, never marked.
if unchangedBy(ctx, inv, module, state.Previous, state.Commit) {
g.Rollback = gateUnchanged
state.Why = "stopped with its send; the send changed nothing of it: " + g.Why
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was left as it was — %s already ran "+
"%s %s, or a build identical to it, before the send, so nothing of it moved and nothing is put back",
strings.Join(g.Machines, ", "), p.Tier, g.Why, module, strings.Join(g.Machines, ", "), module, short(state.Commit))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return
}
if state.Build == "" {
// A plan from before builds were asked by id: nothing to mark, so nothing is put back by the
// mesh — said, for a person.
g.Rollback = inventory.NotRolledBack
p.Note += "; not put back: the plan does not know which build it sent"
sayRollback(ctx, open, module, g, "")
return
}
if err := inv.RecordGate(ctx, verdict); err != nil {
if errors.Is(err, inventory.ErrGateKept) {
// Already judged and acted on, by this controller before a restart or by another: never twice.
if kept, found, _ := inv.GateOf(ctx, state.Build); found {
g.Rollback = kept.Rollback
}
p.Note += "; its rollback was already made once and is not made again"
return
}
g.Rollback = inventory.NotRolledBack
p.Note += "; not put back: its verdict could not be kept, and a rollback that cannot be counted is not made — " + err.Error()
sayRollback(ctx, open, module, g, "")
return
}
// The previous build: the one the first machine ran, from the build records.
notBack := func(why string) {
g.Rollback = inventory.NotRolledBack
p.Note += "; NOT put back: " + why
if err := inv.SetRollback(ctx, state.Build, inventory.NotRolledBack, g.Why+"; not put back: "+why); err != nil {
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
}
sayRollback(ctx, open, module, g, why)
}
if state.Previous == "" {
// A first build there: nothing ran before it, so nothing can be put back, and the machine is left
// with it — said as that, not as a build the mesh lost.
notBack(fmt.Sprintf("this is the first build of %s that %s was sent, or what it was sent before is not known: "+
"there is no earlier build there to put back, so it is left with this one — `unassign` takes it off, "+
"a newer merge replaces it", module, strings.Join(g.Machines, ", ")))
return
}
failed, _, err := inv.BuildByID(ctx, state.Build)
if err != nil {
notBack("the failed build's record cannot be read: " + err.Error())
return
}
previous, found, err := inv.PreviousBuild(ctx, module, state.Previous, failed)
if err != nil {
notBack("the build records cannot be read: " + err.Error())
return
}
if !found {
notBack(fmt.Sprintf("no build of %s from %s, asked before the failed one, is among the %d newest kept to put "+
"back", module, short(state.Previous), inventory.KeptBuilds))
return
}
if err := inv.RestoreModule(ctx, previous); err != nil {
notBack(err.Error())
return
}
g.Rollback = inventory.RollingBack
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err)
}
// Put back with the rest of its send, in one send per machine (issue 281).
if b, batched := ctx.Value(rollbacksKey{}).(*rollbacks); batched {
b.pending = append(b.pending, pendingRollback{module: module, state: state, g: g, previous: previous})
b.modules[module] = true
for _, n := range g.Machines {
if !slices.Contains(b.machines, n) {
b.machines = append(b.machines, n)
}
}
return
}
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
if err != nil {
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
return
}
g.Rollback = inventory.RolledBack
p.Note += fmt.Sprintf("; put back to %s on %s", short(previous.Commit), strings.Join(sent, ", "))
if err := inv.SetRollback(ctx, state.Build, inventory.RolledBack, g.Why); err != nil {
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
}
sayRollback(ctx, open, module, g, "")
}
// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one
// send judges what it moved as one, and what it found wanting goes back in one send per machine — not
// in a send for each module, which is the churn that failed the gate in the first place.
type rollbacks struct {
modules map[string]bool
machines []string
pending []pendingRollback
}
type pendingRollback struct {
module string
state *inventory.PlanModule
g *inventory.PlanGate
previous inventory.Build
}
type rollbacksKey struct{}
// batchingRollbacks is a context under which gateFailed registers what it puts back and leaves the send
// to sendRollbacks.
func batchingRollbacks(ctx context.Context) (context.Context, *rollbacks) {
b := &rollbacks{modules: map[string]bool{}}
return context.WithValue(ctx, rollbacksKey{}, b), b
}
// sendRollbacks sends what a failed send put back, once to each machine, and says each module's rollback.
func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *rollbacks) {
if len(b.pending) == 0 {
return
}
inv := open.inventory
sort.Strings(b.machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
var back []string
for _, r := range b.pending {
if err != nil {
why := fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
"sends it", short(r.previous.Commit), strings.Join(r.g.Machines, ", "), err, firstOf(r.g.Machines))
r.g.Rollback = inventory.NotRolledBack
if err := inv.SetRollback(ctx, r.state.Build, inventory.NotRolledBack, r.g.Why+"; not put back: "+why); err != nil {
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
}
sayRollback(ctx, open, r.module, r.g, why)
continue
}
r.g.Rollback = inventory.RolledBack
back = append(back, r.module+" to "+short(r.previous.Commit))
if err := inv.SetRollback(ctx, r.state.Build, inventory.RolledBack, r.g.Why); err != nil {
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
}
sayRollback(ctx, open, r.module, r.g, "")
}
if err != nil {
p.Note += fmt.Sprintf("; NOT put back: sending %s was refused: %v", strings.Join(b.machines, ", "), err)
return
}
p.Note += fmt.Sprintf("; put back %s on %s, in one send", strings.Join(back, ", "), strings.Join(sent, ", "))
}
// gateUnchanged is a failed gate's word on a module its send changed nothing of (novox/hq issue 281):
// left as it was, never marked failed, and so no verdict on its build — `plans retry` asks it again.
const gateUnchanged = "unchanged"
// unchangedBy says whether a send moving a module from one build to another changed nothing of it: the
// same commit, builds made from the same source, or builds that made the same artifacts from the same
// manifest. Not known is changed.
func unchangedBy(ctx context.Context, inv *inventory.Inventory, module, from, to string) bool {
if from == "" || to == "" {
return false
}
if sameCommit(from, to) {
return true
}
// Made from the same source (issue 280), or the same artifacts from the same manifest.
f := moveFacts{}
var err error
if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
return false
}
if f.fps, err = inv.Fingerprints(ctx); err != nil {
return false
}
return f.identical(module, from, to)
}
// rolledBackEvent is the body of `rolled-back` (ADR 0236): a contract, like a condition's events.
type rolledBackEvent struct {
Event string `json:"event"`
At time.Time `json:"at"`
Module string `json:"module"`
Component string `json:"component,omitempty"`
Machines []string `json:"machines"`
From string `json:"from,omitempty"`
To string `json:"to,omitempty"`
Why string `json:"why"`
// Rollback is rolled-back, or not-rolled-back with NotWhy.
Rollback string `json:"rollback"`
NotWhy string `json:"not_why,omitempty"`
Show string `json:"show"`
}
// sayRollback raises the gate's condition at once — the probe keeps it from then — and says the event.
func sayRollback(ctx context.Context, open *stores, module string, g *inventory.PlanGate, notWhy string) {
o := gateObservation(module, g.Component, g.Machines, g.Rollback, g.Why, notWhy, g.To, g.From)
fmt.Println(o.Summary)
d := doctorFrom
if d == nil {
return
}
if d.keeper != nil {
o.Source = gateProbe
if _, err := d.keeper.Observe(ctx, o); err != nil {
fmt.Printf("the gate's condition %s could not be kept: %v\n", o.Key(), err)
}
}
if d.teller == nil {
return
}
body, err := json.Marshal(rolledBackEvent{Event: link.KeyRolledBack, At: time.Now().UTC(), Module: module,
Component: g.Component, Machines: g.Machines, From: g.To, To: g.From, Why: g.Why, Rollback: g.Rollback,
NotWhy: notWhy, Show: conditions.Condition{Key: o.Key()}.Show()})
if err != nil {
return
}
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyRolledBack, body); err != nil {
fmt.Printf("%s's rollback could NOT be said on the bus: %v\n", module, err)
}
}
// gateObservation is a failed gate as a condition: `core.<component>.<machine>.rolled-back` for the
// core (to-be 45 §2), `build.<module>.<machine>.rolled-back` for any other module; `rollback-failed`
// when it could not be put back. Urgent for the core and for a build left in place; a warning for a
// module put back, which runs what it ran before.
func gateObservation(module, component string, machines []string, rollback, why, notWhy, failed, previous string) conditions.Observation {
machine := strings.Join(machines, ",")
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: module + "." + machine, Kind: kindRolledBack,
Token: kindRolledBack, Machine: firstOf(machines), Severity: conditions.Warning, Source: gateProbe,
Summary: fmt.Sprintf("%s's build %s failed its gate on %s and was put back to %s: %s", module, short(failed),
machine, short(previous), why)}
if component != "" {
o.Scope, o.ID, o.Severity = conditions.ScopeCore, component+"."+machine, conditions.Urgent
}
if len(machines) > 1 {
o.Also = machines[1:]
}
if rollback != inventory.RolledBack && rollback != inventory.RollingBack {
o.Kind, o.Token, o.Severity = kindRollbackFailed, kindRollbackFailed, conditions.Urgent
o.Resolver = conditions.ResolverOperator
o.Summary = fmt.Sprintf("%s's build %s failed its gate on %s and was NOT put back: %s — %s", module, short(failed),
machine, why, orNone(notWhy))
}
return o
}
// probeGates is DG: no build that failed its gate is left without its condition, and no witness's
// rollback goes unsaid. Each module whose newest verdict is a failure keeps its condition; a newer build
// that passes clears it. Each core component a machine's witness says it put back is `core.<component>.
// <machine>.rolled-back`, urgent, until the machine's reports stop saying it.
func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
latest, err := d.open.inventory.LatestGates(ctx)
if err != nil {
return nil, err
}
var out []conditions.Observation
for _, v := range latest {
if v.Verdict != inventory.GateFailed {
continue
}
notWhy := ""
if v.Rollback == inventory.NotRolledBack {
notWhy = v.Why
}
out = append(out, gateObservation(v.Module, v.Component, v.Machines, v.Rollback, v.Why, notWhy, v.Commit, v.Previous))
}
for node, list := range witnessed.all() {
for _, r := range list {
out = append(out, witnessObservation(node, r))
}
}
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
out = append(out, backlogObservation()...)
return sortedFound(dedupeObservations(out)), nil
}
// dedupeObservations keeps one observation per key, the first.
func dedupeObservations(list []conditions.Observation) []conditions.Observation {
seen := map[string]bool{}
var out []conditions.Observation
for _, o := range list {
if seen[o.Key()] {
continue
}
seen[o.Key()] = true
out = append(out, o)
}
return out
}
// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules'
// tools are served where. A variable so a test needs no runtime.
var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) {
inbox := conn.NewRespInbox()
sub, err := conn.SubscribeSync(inbox)
if err != nil {
return nil, err
}
defer func() { _ = sub.Unsubscribe() }()
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
}
out := map[string]served{}
add := func(node string) served {
s, ok := out[node]
if !ok {
s = served{tools: map[string]bool{}}
}
return s
}
deadline := time.Now().Add(discoveryPatience)
for time.Now().Before(deadline) {
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
msg, err := sub.NextMsgWithContext(wait)
cancel()
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
break
}
var info micro.Info
if json.Unmarshal(msg.Data, &info) != nil {
continue
}
if info.Name == broker.RuntimeModule {
node := info.Metadata["node"]
if node == "" {
node = info.ID
}
s := add(node)
s.runtime = true
out[node] = s
}
for _, e := range info.Endpoints {
if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" {
continue
}
node := e.Metadata["node"]
if node == "" {
node = info.ID
}
s := add(node)
s.tools[e.Metadata["module"]] = true
out[node] = s
}
}
return out, nil
}
// gateLines is a plan's gates as `plans <id>` says them: the rollout's record.
func gateLine(g *inventory.PlanGate) string {
if g == nil {
return ""
}
what := "judging"
switch g.Verdict {
case inventory.GatePassed:
what = "passed"
case inventory.GateFailed:
what = "FAILED"
}
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
if g.Component != "" {
line += " (core: " + g.Component + ")"
}
if g.From != "" || g.To != "" {
line += fmt.Sprintf(", %s → %s", short(orNone(g.From)), short(g.To))
}
if g.Took != "" {
line += ", after " + g.Took
}
if g.Why != "" {
line += ": " + g.Why
} else if g.Last != "" {
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
} else if g.Verdict == "" {
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
}
if g.Rollback != "" {
line += "; " + g.Rollback
}
return line
}
// witnessObservation is a witness's verdict as a condition (ADR 0236, the host's contract):
// `core.<component>.<node>.<outcome>`, urgent for rolled-back, not-reversible, restore-failed and
// halted, a warning for nothing-to-restore and unwitnessed.
func witnessObservation(node string, r lease.Rollback) conditions.Observation {
severity := conditions.Warning
if lease.Urgent(r.Outcome) {
severity = conditions.Urgent
}
outcome := r.Outcome
if outcome == "" {
outcome = lease.OutcomeRolledBack
}
what := map[string]string{
lease.OutcomeRolledBack: "and put back " + short(orNone(r.To)),
lease.OutcomeNotReversible: "and left it: it is not reversible",
lease.OutcomeNothingToRestore: "and had nothing to put back",
lease.OutcomeRestoreFailed: "and could not put the previous build back",
lease.OutcomeUnwitnessed: "and could not judge it at all",
lease.OutcomeHalted: "and gave up: the build before it fails too",
}[outcome]
return conditions.Observation{Scope: conditions.ScopeCore, ID: r.Component + "." + node, Kind: kindRolledBack,
Token: outcome, Machine: node, Severity: severity,
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
}
cmd/mesh-controller/gate_test.go
-551
View File
@@ -1,551 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).
// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
// sent and reports it — with the gate's bounds shortened to judge in three steps.
type gateMesh struct {
open *stores
sent [][]string
health map[string]health // per machine, what a judging finds; healthy when unsaid
keeper *conditions.Keeper
told *conditions.Told
}
func aGateMesh(t *testing.T) *gateMesh {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
g := &gateMesh{open: open, health: map[string]health{}}
g.keeper, _ = withConditionsInMemory(t)
g.told = &conditions.Told{}
was := doctorFrom
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
t.Cleanup(func() { doctorFrom = was })
for _, b := range []inventory.Build{
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
} {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
b.Manifest = manifest
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
register := func(commit string, asked time.Time) {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
Head: commit, Asked: asked}); err != nil {
t.Fatal(err)
}
}
register("c1", time.Now().Add(-2*time.Hour))
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, n, "app"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
}
register("c2", time.Now().Add(-time.Minute))
// Every send: recorded with the build the module is at, applied and reported by the machine.
n := 0
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
g.sent = append(g.sent, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
// What a judging reads: the store's reports, and a health the test says per machine.
wasGather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
if g.health[r.Node] == healthBroken {
r.Outcome = inventory.OutcomeFailed
}
f.reports[r.Node] = r
}
for node, h := range g.health {
if h == healthNotYet {
f.rolledBack[node] = nil
r := f.reports[node]
r.Current = false
f.reports[node] = r
}
}
return f, nil
}
t.Cleanup(func() { gatherGateFacts = wasGather })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
built := time.Now().UTC()
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
Build: "build-2"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
return g
}
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
t.Helper()
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
if err != nil {
t.Fatal(err)
}
return p
}
// A module's build that fails its gate on the first machine is put back there — the previous build
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
// condition and an event, never registered or rolled back again.
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
advancePlans(ctx, g.open) // the first machine is sent the new build
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v, not the first machine alone", g.sent)
}
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
}
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
gateEvery = 0
advancePlans(ctx, g.open)
p := g.plan(t)
gate := p.Modules["app"].Gate
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
gate.Rollback != inventory.RolledBack {
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
}
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
}
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
t.Fatalf("anchor was last sent %v, not the previous build", sent)
}
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
t.Fatalf("laptop was sent the failed build: %v", sent)
}
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
}
// Said: a condition for the operator, and the event.
open, err := g.keeper.Open(ctx)
if err != nil {
t.Fatal(err)
}
var key string
for _, c := range open {
if c.Kind == kindRolledBack {
key = c.Key
}
}
if key != "build.app.anchor.rolled-back" {
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
}
saidIt := false
for _, e := range g.told.Said() {
saidIt = saidIt || e.Event == link.KeyRolledBack
}
if !saidIt {
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
}
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
// heard again is not registered.
advancePlans(ctx, g.open)
state := p.Modules["app"]
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
if len(g.sent) != 2 {
t.Fatalf("sent again after the rollback: %v", g.sent)
}
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"})})
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
t.Fatalf("the failed build was registered again: %v", err)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("the module moved to %s", current["app"].Commit)
}
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
}
// The probe keeps the condition while the newest verdict is the failure.
obs, err := probeGates(ctx, doctorFrom)
if err != nil || len(obs) != 1 || obs[0].Key() != key {
t.Fatalf("the gate's probe found %+v %v", obs, err)
}
}
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
// the rest are sent, the verdict kept, and the plan done.
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
gateEvery = 0
for i := 0; i < 6; i++ {
advancePlans(ctx, g.open)
}
p := g.plan(t)
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("sent %v", g.sent)
}
gate := p.Modules["app"].Gate
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
}
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
}
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
}
}
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
advancePlans(ctx, g.open)
g.health["anchor"] = healthNotYet
gateEvery = 0
advancePlans(ctx, g.open)
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
}
gateBound = -time.Second
advancePlans(ctx, g.open)
p := g.plan(t)
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
}
}
// The health a judging finds, per core component and for a module.
func TestTheHealthDefinitions(t *testing.T) {
now := time.Now()
since := now.Add(-time.Minute)
applied := func(node string) gateFacts {
at := now
return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
}
m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}
f := applied("anchor")
f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
t.Errorf("a module whose tools are not served is %v (%s)", h, why)
}
f.served["anchor"].tools["app"] = true
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
t.Errorf("a module applied with its tools served is %v (%s)", h, why)
}
// A condition raised about it on that machine since it was sent: not yet healthy.
f.judged = true
f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
t.Errorf("a module with a new condition about it is %v", h)
}
f.open[0].Raised = since.Add(-time.Hour)
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
t.Errorf("a condition older than the send counted against it: %v", h)
}
// A witness that put it back: broken.
f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
t.Errorf("a component a witness put back is %v", h)
}
// The node tools answer, or not.
f = applied("anchor")
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
t.Errorf("node tools not answering are %v", h)
}
f.served["anchor"] = served{runtime: true}
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
t.Errorf("node tools answering are %v", h)
}
// The controller: the lease held since the send, by a controller that says it is ready.
f = applied("control")
f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
t.Errorf("a lease held by a controller older than the build is %v", h)
}
f.holder.Taken = now
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
t.Errorf("a new controller holding the lease and ready is %v", h)
}
f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
!strings.Contains(why, "first run") {
t.Errorf("a controller not ready is %v (%s)", h, why)
}
// A machine that refused what it was sent: broken.
f = applied("anchor")
r := f.reports["anchor"]
r.Outcome = inventory.OutcomeRefused
f.reports["anchor"] = r
if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
t.Errorf("a refusal is %v", h)
}
}
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
t.Fatal(err)
}
u, err := inv.UpgradeOf(ctx, "nats")
if err != nil || u.RollOut || u.From != catalogue.FromBus {
t.Fatalf("the bus's policy is %+v %v", u, err)
}
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
t.Fatalf("a person rolled the bus out: %v", err)
}
var sent [][]string
was := sendRollout
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
sent = append(sent, names)
return names, nil
}
t.Cleanup(func() { sendRollout = was })
now := time.Now().UTC()
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
advancePlans(ctx, open)
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
}
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
}
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
t.Fatalf("a send to the bus's machine was not refused: %v", err)
}
// A rebuild that made the same artifacts is no move.
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
b.Asked, b.At = time.Now(), time.Now()
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
Reference: "registry/nats@sha256:new"}}}); err != nil {
t.Fatal(err)
}
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
// first by the bus machine's backup holder.
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
}
wasSnapshot := takeBusSnapshot
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
}
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
return node + "'s restore point of " + module, nil
}
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
t.Fatalf("the step sent %v, not the bus's machine", sent)
}
s, found, err := inv.LatestBusStep(ctx)
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
t.Fatalf("the step is %+v %v %v", s, found, err)
}
}
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := asksRecorded(t)
for _, name := range []string{"gone", "kept"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
}
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
if len(*asked) != 0 {
t.Fatalf("a deleted module was asked to build: %v", *asked)
}
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
if _, still := shelf["gone"]; still {
t.Fatal("a deleted module nothing holds was not forgotten")
}
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
asked0 := time.Now().UTC().Add(-time.Minute)
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
p, _ := inv.PlanByID(ctx, "plan-deleted")
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
}
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
b, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return b
}
func nodeID(t *testing.T, open *stores, name string) string {
t.Helper()
n, err := open.inventory.NodeByName(context.Background(), name)
if err != nil {
t.Fatal(err)
}
return n.ID
}
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
// gone with the first report that no longer carries it.
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
open := aMesh(t)
d := &doctor{open: open}
at := time.Now().UTC()
witnessed.heard("control", []lease.Rollback{
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
Why: "the new controller did not take the lease within 60s", At: at},
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
}, at)
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
obs, err := probeGates(t.Context(), d)
if err != nil {
t.Fatal(err)
}
got := map[string]conditions.Severity{}
for _, o := range obs {
got[o.Key()] = o.Severity
}
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
"core.node-tools.control.unwitnessed": conditions.Warning}
if !reflect.DeepEqual(got, want) {
t.Fatalf("the witness's verdicts are %v", got)
}
witnessed.heard("control", nil, time.Now())
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
}
}
cmd/mesh-controller/given.go
-90
View File
@@ -1,90 +0,0 @@
package main
import (
"context"
"encoding/json"
"log"
"strings"
"sync"
"time"
"github.com/novox/mesh-controller/internal/link"
)
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
//
// The serving controller hears every report; a clean one about the declaration a machine was last
// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is
// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
// so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log
// is read as the count of repairs a healer is wanted for.
// SecretReplaced is the controller seat's fact that a value given by hand was replaced
// (link.KeySecretReplaced). Never the value: neither the old one, which the mesh cannot read,
// nor the new one, sealed to the machine as it was made.
type SecretReplaced struct {
Node string `json:"node"`
Module string `json:"module"`
Name string `json:"name"`
Given time.Time `json:"given"`
// Sent are the machines sent so the module starts again on the new value; Unsent says why
// they could not be, in which case the next push carries it.
Sent []string `json:"sent"`
Unsent string `json:"unsent,omitempty"`
Why string `json:"why"`
}
// givenEvents is where the serving controller states it; nil in a command.
var givenEvents link.Bus
// replacing keeps one replacement per machine at a time: two reports arriving together find the
// same rows, and the store's claim makes one of them the replacer, but the sends need not race.
var replacing sync.Map
// startedWell says a report is a machine's clean account of a declaration: everything applied,
// nothing failed or refused. Whether it is the declaration last sent is the store's to answer.
func startedWell(report link.Report) bool {
return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil
}
const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
// replaceGiven replaces what the report makes due, sends the machines, and says so.
func replaceGiven(ctx context.Context, open *stores, report link.Report) {
if _, busy := replacing.LoadOrStore(report.Node, true); busy {
return
}
defer replacing.Delete(report.Node)
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared)
if err != nil {
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err)
}
for _, r := range replaced {
said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
Sent: r.Machines, Why: givenWhy}
log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; sending %s",
r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
if err := sendTo(ctx, open, r.Machines); err != nil {
said.Sent, said.Unsent = nil, err.Error()
log.Printf("the new %q of %s is sealed and not yet delivered to %s — the next push carries it: %v",
r.Name, r.Module, strings.Join(r.Machines, ", "), err)
}
stateReplaced(ctx, said)
}
}
func stateReplaced(ctx context.Context, said SecretReplaced) {
if givenEvents == nil {
return
}
body, err := json.Marshal(said)
if err != nil {
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
return
}
stating, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := givenEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySecretReplaced, body); err != nil {
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
}
}
cmd/mesh-controller/given_test.go
-45
View File
@@ -1,45 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A rotation asked through the seat carries why to the command, which records it in the hand-act
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password --why leaked into logs --cause leaked" {
t.Fatalf("%v %v", argv, err)
}
argv, err = argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password"})
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
t.Fatalf("without why: %v %v", argv, err)
}
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
t.Fatalf("why beside a provision was passed over: %v", argv)
}
}
// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that
// the machine is there is not (novox/hq ADR 0228).
func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) {
for _, c := range []struct {
report link.Report
good bool
}{
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true},
{link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true},
{link.Report{Node: "anchor"}, false},
{link.Report{Node: "anchor", Applied: []string{"x"}}, false},
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false},
{link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false},
} {
if got := startedWell(c.report); got != c.good {
t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good)
}
}
}
cmd/mesh-controller/grant_follows_binding_test.go
-197
View File
@@ -1,197 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 274: a provider is granted exactly the consumers whose own resolution binds them to
// it — not every consumer a pair credential from it was ever made for.
// grantOf is the grant of one provision to one consuming module, and whether there is one at all.
func grantOf(grants []catalogue.Grant, provision, consumer, module string) (catalogue.Grant, bool) {
for _, g := range grants {
if g.Provision == provision && g.Consumer == consumer && (g.From == module || g.From == "") {
return g, true
}
}
return catalogue.Grant{}, false
}
// The morning after issue 273, through the stores: a consumer was bound to the store on another
// machine, a credential from there was made, and a person pinned it back to the store beside it. Both
// credentials are on record. The store it left is no longer granted it — so it retires it and keeps
// its data (ADR 0230) — and says so; the store it is bound to keeps its grant; and the credential from
// the store it left stays on record.
func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
for _, m := range storeManifests() {
register(t, open, m)
}
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
t.Fatal(err)
}
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
t.Fatal(err)
}
for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
// Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded.
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
t.Fatal(err)
}
plan, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
t.Fatalf("pinned to the anchor and bound to %s", n.From)
}
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
t.Fatal(err)
}
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound)
}
// Pinned back beside its data, as the operator did.
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil {
t.Fatal(err)
}
plan, _, err = planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" {
t.Fatalf("pinned back to the laptop and bound to %s", n.From)
}
if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
t.Fatal(err)
}
holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop")
if err != nil {
t.Fatal(err)
}
if len(holders) != 2 {
t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders)
}
// The store it left: no longer granted, and said.
grants, _, unbound, err = grantsFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" {
t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g)
}
if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" ||
strings.Join(unbound[0].BoundTo, ",") != "laptop" {
t.Fatalf("the consumer that moved is not the one said: %+v", unbound)
}
planned, settings, err := planFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, "anchor", planned, settings)
if err != nil {
t.Fatal(err)
}
if got := declared.Received["store"]["postgres-database"]; len(got) != 0 {
t.Fatalf("the anchor's store is still told about %+v", got)
}
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") ||
!strings.Contains(said, "cleanup delete") {
t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said)
}
// The store it is bound to: granted.
grants, _, unbound, err = grantsFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 {
t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound)
}
// And the credential from the store it left is kept: the key to the login and data held there.
if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 {
t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err)
}
}
// A consumer whose resolution cannot be read is an error, never a consumer bound nowhere — withdrawing
// a grant on that reading would take its access away (issue 152).
func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
for _, m := range storeManifests() {
register(t, open, m)
}
for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
if _, _, err := planFor(ctx, open, "laptop"); err != nil {
t.Fatal(err)
}
// The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and
// that is not its set failing to compose.
laptop, err := inv.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil {
t.Fatal(err)
}
if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) {
t.Fatalf("the seam this test relies on moved: %v", err)
}
grants, _, unbound, err := grantsFor(ctx, open, "anchor")
if err == nil {
t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound)
}
if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") {
t.Fatalf("the error does not say whose resolution could not be read: %v", err)
}
}
// The rule itself, on a resolution: bound is the provider a need for exactly that credential is
// answered by, never one answered by a record, and a different local name is a different credential.
func TestBindsFromIsTheProviderOfThatCredential(t *testing.T) {
r := catalogue.Resolution{Needs: []catalogue.Needed{
{Name: "postgres-database", For: "board", From: "laptop"},
{Name: "postgres-database", For: "board", From: "laptop", Local: "reports"},
{Name: "postgres-database", For: "wiki", From: "anchor"},
{Name: "a-licence", For: "board", From: "the-licence", ByRecord: true},
}}
s := inventory.Secret{Name: "postgres-database", ConsumerModule: "board"}
if got := r.BindsFrom(s.Name, s.ConsumerModule, s.Local); strings.Join(got, ",") != "laptop" {
t.Fatalf("board's credential is bound to %v", got)
}
if got := r.BindsFrom("postgres-database", "board", "archive"); len(got) != 0 {
t.Fatalf("a local name nothing asks for is bound to %v", got)
}
if got := r.BindsFrom("a-licence", "board", ""); len(got) != 0 {
t.Fatalf("a need answered by a record is bound to %v", got)
}
}
cmd/mesh-controller/handacts.go
-268
View File
@@ -1,268 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// Acts done by hand, and why (novox/hq to-be 45 §7).
//
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
// required, because the installer and the lab push by command line as a step of what they do, and a
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
// (Phase 1).
// handActVerb is one verb that writes the hand-act log, and whether what it records is a repair.
type handActVerb struct {
Verb string
// Decision says why an act of this verb is a person's decision by design rather than a repair a
// healer could take over; empty for a repair.
Decision string
// DecidedFor limits Decision to these causes; empty, it holds for every act of the verb.
DecidedFor []string
}
// causeLeakedInLogs is the cause a rotation after a value was printed into a log gives.
const causeLeakedInLogs = "leaked-in-logs"
// handActVerbs is every verb that writes the hand-act log (novox/hq to-be 45 §7). **S15 reads it**:
// an act recorded by a verb whose entry names a decision is the mesh working as decided, never a
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
var handActVerbs = []handActVerb{
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
// default (ADR 0236) exists to end.
{Verb: "push"},
{Verb: "plans stop"},
{Verb: "plans close"},
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
// not trusted with it — either is a repair the owner should have made.
{Verb: "plans go"},
{Verb: "broker consumer-reset"},
// Silencing the same condition twice says the condition, or what it watches, wants mending.
{Verb: "conditions silence"},
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts.
{Verb: "hand-act record"},
// A person's decisions by design.
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
"person starts (ADR 0236)"},
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
"person releases it (ADR 0236)"},
// A leak is judged by a person — which value was exposed, to whom — and its rotation is the answer
// to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the
// module that prints them, an issue against it, not a healer that rotates. A rotation for any other
// cause — a credential that stopped working — counts: a schedule or a healer could take it over.
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
DecidedFor: []string{causeLeakedInLogs}},
}
// personsDecision is whether an act in the log is a person's decision by design, by the verb that
// recorded it (handActVerbs).
func personsDecision(a link.HandAct) bool {
for _, v := range handActVerbs {
if v.Verb != a.Verb {
continue
}
return v.Decision != "" && (len(v.DecidedFor) == 0 || slices.Contains(v.DecidedFor, a.Cause))
}
return false
}
// handActFlags are the flags every repairing verb takes.
type handActFlags struct {
why, cause, condition *string
}
func addHandActFlags(set *flag.FlagSet) handActFlags {
return handActFlags{
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
}
}
// given is whether a reason was given.
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
// require refuses an act without a reason, before anything is done.
func (f handActFlags) require(verb string) error {
if f.given() {
return nil
}
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
"log, novox/hq to-be 45 §7). Nothing was done", verb)
}
// handActConn is the serving controller's connection, for what it reads of the log itself; a
// command dials its own.
var handActConn *nats.Conn
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
func onTheBus(f func(*nats.Conn) error) error {
if handActConn != nil {
return f(handActConn)
}
address, err := broker.BusAddress()
if err != nil {
return err
}
js, err := broker.Dial(address)
if err != nil {
return fmt.Errorf("cannot reach the bus: %w", err)
}
defer js.Close()
return f(js.Conn())
}
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
// whose bus is down is exactly the mesh somebody is repairing by hand.
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
if !f.given() {
return
}
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
err := onTheBus(func(conn *nats.Conn) error {
written, err := link.RecordHandAct(ctx, conn, act)
act = written
return err
})
if err != nil {
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
return
}
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
}
// handActCommand is `hand-act record` and `hand-acts`.
func handActCommand(ctx context.Context, args []string) error {
if len(args) > 0 && args[0] == "record" {
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
f := addHandActFlags(set)
positionals, err := parseAround(set, args[1:])
if err != nil {
return err
}
what := strings.TrimSpace(strings.Join(positionals, " "))
if what == "" {
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
}
if err := f.require("hand-act record"); err != nil {
return err
}
if strings.TrimSpace(*f.cause) == "" {
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
"act for the same reason will use — it is how a repair done twice is found")
}
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
return onTheBus(func(conn *nats.Conn) error {
written, err := link.RecordHandAct(ctx, conn, act)
if err != nil {
return fmt.Errorf("the act could not be recorded: %w", err)
}
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
written.Why, written.Cause)
return nil
})
}
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
}
if len(args) > 0 && args[0] == "list" {
args = args[1:]
}
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
days := set.Int("days", 14, "how many days back")
asJSON := set.Bool("json", false, "as data")
if _, err := parseAround(set, args); err != nil {
return err
}
return onTheBus(func(conn *nats.Conn) error {
now := time.Now()
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
if err != nil {
return err
}
repeated := link.RepeatedCauses(repairs(acts), now)
if *asJSON {
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
if len(acts) == 0 {
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
return nil
}
for i := len(acts) - 1; i >= 0; i-- {
a := acts[i]
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
if a.Condition != "" {
fmt.Printf(", condition %s", a.Condition)
}
fmt.Println(")")
}
if len(repeated) > 0 {
causes := make([]string, 0, len(repeated))
for c, n := range repeated {
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
}
sort.Strings(causes)
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
strings.Join(causes, ", "))
}
return nil
})
}
// repairs are the acts that are not a person's decision by design: what S15 counts.
func repairs(acts []link.HandAct) []link.HandAct {
out := make([]link.HandAct, 0, len(acts))
for _, a := range acts {
if !personsDecision(a) {
out = append(out, a)
}
}
return out
}
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
// the log could not be read, which status says rather than reading as none.
func handActsThisWeek(ctx context.Context) (int, string) {
n := -1
err := onTheBus(func(conn *nats.Conn) error {
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
n = len(acts)
return err
})
if err != nil {
return -1, err.Error()
}
return n, ""
}
cmd/mesh-controller/handacts_test.go
-94
View File
@@ -1,94 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
// the verb.
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
}{
{"push", map[string]any{"node": "anchor"}},
{"push", map[string]any{}},
{"plans", map[string]any{"close": "plan-1"}},
{"plans", map[string]any{"stop": "plan-1"}},
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
{"command", map[string]any{"command": "push anchor"}},
{"command", map[string]any{"command": "plans close plan-1"}},
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
} {
argv, err := argvFor(c.verb, c.args)
if c.verb == "plans" && err == nil {
// The seat composes the command line; the command refuses it, before opening anything.
err = plansCommand(context.Background(), argv[1:])
}
if err == nil || !strings.Contains(err.Error(), "why") {
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
}
}
for _, args := range [][]string{{"EVENTS", "controller"}} {
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
t.Errorf("consumer-reset without why: %v", err)
}
}
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
!strings.Contains(err.Error(), "--why") {
t.Errorf("hand-act record without why: %v", err)
}
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
!strings.Contains(err.Error(), "--cause") {
t.Errorf("hand-act record without a cause: %v", err)
}
}
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
func TestARepairByHandCarriesItsReason(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want string
}{
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
"plans close plan-1 --why the report will not come"},
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
}
}
}
// The summary of durations says, per kind and subject, what a bound would be set from.
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
var ds []inventory.Duration
for i := 1; i <= 10; i++ {
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
Took: time.Duration(i) * time.Second})
}
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
got := summarise(ds)
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
t.Fatalf("%+v", got)
}
if !strings.Contains(got[1].Suggests, "3m0s") {
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
}
}
cmd/mesh-controller/healers.go
-876
View File
@@ -1,876 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The healers (novox/hq to-be 45 §7, ADR 0227 rule 7, Phase 3).
//
// **A known failure heals itself, under a brake, and every repair is said.** Research 031 counted the
// repairs people made by hand in six days: a push to unstick a plan waiting on a report (four times),
// a controller restarted to make an object again (twice), a plan closed (twice), a consumer re-made from
// now (once). Each was the ordinary path, taken again by a person who noticed. A healer is that act,
// registered against the one condition kind it answers, so the mesh takes it itself:
//
// - **its repair is the ordinary path again** — the send a push makes, the plan's own close, the
// assertion every send makes, the consumer reset the verb makes — never a withdrawal, a deletion of
// data or a recreation of it;
// - **its budget** is how many acts it may take against one thing in a window, and **its settle** how
// long after an act it leaves the observation to say whether it worked;
// - **success is never the healer's to say.** The condition clears when the watchdog or the probe that
// raised it no longer sees it. A healer marking its own work done would be the second opinion of a
// fact that rule 1 forbids;
// - **a spent budget stops it**: the condition is the operator's, urgent, with every attempt in
// `tried`, and no healer touches it again until observation clears it;
// - **every act is said**: begun in the store before it is made (so a controller dying mid-act has
// still spent it), kept in the condition's `tried` as `healer Hn`, and said on the bus as the
// controller seat's `healer-acted`. A heal is not a hand act and is never in the hand-act log —
// which is how S15 can tell a repair the mesh made from one a person had to.
//
// **And the mesh-wide brake**: more than healBrakeLimit acts in an hour, all healers together, and every
// healer stops — an urgent condition says so — until an hour has passed with none. A healer looping is
// then at most a dozen acts, said, and never the incident itself.
//
// Only the controller holding the lease heals, under its epoch: a controller serving without the lease
// (S12) heals nothing, because a repair is the one act that can always wait for the lease.
// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour.
const (
healBrakeLimit = 12
healBrakeWindow = time.Hour
)
// healEvery is how often the healers look at what is open.
var healEvery = 30 * time.Second
// What raises the healers' own conditions, and their kinds.
const (
sourceHealers = "healers"
kindHealersBraked = "healers-braked"
kindConsumerBehind = "consumer-behind"
kindHealersBlind = "probe-failed"
)
// healerRow is one row of the registry.
type healerRow struct {
ID string
// Kinds are the condition kinds it answers: from the signals table, the probe registry, or an event.
Kinds []string
// Condition, Repair, Then, From are the row in words, as to-be 45 §7 and `healers` say it.
Condition string
Repair string
Then string
From string
// Budget acts against one budget key within Window; Settle after an act before the next, or the
// escalation, so the observation has had its turn to say whether it worked.
Budget int
Window time.Duration
Settle time.Duration
// ActsIn is where the repair runs: the controller, or a module that repairs its own (H5).
ActsIn string
// Event is what each act is said as.
Event string
// applies says whether this condition is one the healer may act on, and what its budget is
// counted against; why when it is not. Reads, never acts. Nil where the repair is a module's.
applies func(ctx context.Context, h *healing, c conditions.Condition) (budget string, ok bool, why string, err error)
// repair is the act: what it did in words, what came of it, or an error when it could not be done.
repair func(ctx context.Context, h *healing, c conditions.Condition) (act, said string, err error)
}
// actsInController is a healer whose repair the controller makes.
const actsInController = "controller"
// healerRegistry is to-be 45 §7's table, compiled in. **The registry is the design's live form**: a
// test generated from it holds every row to a condition kind the mesh raises, a budget, a brake and its
// event (healers_test.go).
var healerRegistry = []healerRow{
{ID: "H1", Kinds: []string{"sent-not-reported"},
Condition: "a machine was sent a declaration and has not reported it (S2)",
Repair: "ask the machine's node-engine to say again what it last applied (the `report` verb); if what " +
"comes back is not the declaration it was sent, or nothing comes, send it its current declaration " +
"again — what a push of that one machine does, never moving a build a policy or a plan holds back",
Then: "resolver operator, urgent", From: "a push by hand to unstick a plan waiting on a report (230, 257, 264, 267)",
Budget: 2, Window: 6 * time.Hour, Settle: 3 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAMachine, repair: repairReport},
{ID: "H2", Kinds: []string{"stalled"},
Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " +
"exists — or already finished — every module of every tier built or failed, and every one that " +
"rolls out sent (S3); or a delivery held past its state's bound where mesh-delivery's table lets H2 " +
"take a transition (D14, novox/hq ADR 0239)",
Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " +
"done; what it asked still builds and registers — or, for a delivery, mesh-delivery's `close`, which " +
"reads its walk again and takes only the transition its table names",
Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)",
Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAStalePlan, repair: repairPlan},
{ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"},
Condition: "a seat's holder that does not answer (D3), or a durable consumer the mesh expects and the " +
"bus does not hold (D6, S9)",
Repair: "assert the bus's streams, consumers and seat workers again — the assertion every send makes " +
"(issue 208)",
Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)",
Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAnObject, repair: repairObjects},
{ID: "H4", Kinds: []string{kindConsumerBehind},
Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable",
Repair: "re-make the consumer to deliver from now — `broker consumer-reset` (issue 248); what it drops " +
"is caught up where the table says",
Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)",
Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAResettableConsumer, repair: repairConsumer},
{ID: "H5", Kinds: []string{kindProviderFailing},
Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing, " +
"credentials-rejected)",
Repair: "the provider repairs it itself through the server's own bootstrap command, checks again and " +
"says what it did (ADR 0224 §5); the controller keeps its word as the condition",
Then: "announced failing by the provider, braked from ten minutes doubling to six hours (ADR 0224 §5)",
From: "the identity provider's admin reset through its bootstrap command (179)",
// Its budget and brake are the module's own: ten minutes, doubling, to six hours.
Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute,
ActsIn: "the provider module (keycloak), ADR 0224 §5", Event: "provisioner.failing, provisioner.recovered"},
}
// healerFor is the healer registered for a kind, and false when none acts in the controller.
func healerFor(kind string) (healerRow, bool) {
for _, r := range healerRegistry {
if r.repair != nil && slices.Contains(r.Kinds, kind) {
return r, true
}
}
return healerRow{}, false
}
// healerNamedFor is any healer registered for a kind, wherever it acts: what S15 says beside a cause.
func healerNamedFor(kind string) string {
for _, r := range healerRegistry {
if slices.Contains(r.Kinds, kind) {
return r.ID
}
}
return ""
}
// healing is the healers' runner and what their repairs reach.
type healing struct {
open *stores
keeper *conditions.Keeper
teller conditions.Teller
// js is the bus, for the repairs that assert or reset its objects; nil where there is none.
js *broker.JetStream
// epoch is the lease's gate; acting says this controller is the one acting, not one standing by.
epoch func(ctx context.Context) (uint64, error)
acting func() bool
now func() time.Time
say func(format string, args ...any)
// The acts, as seams a test replaces: asking a machine to report, sending it again, asserting the
// bus's objects, resetting a consumer.
askReport func(ctx context.Context, node string) error
sendAgain func(ctx context.Context, node string) error
assertObjects func(ctx context.Context) error
resetConsumer func(stream, name string) (string, error)
// reportWait is how long H1 waits for the machine's report after asking.
reportWait time.Duration
mu sync.Mutex
// declined is why each condition was last passed over, so it is said once and `healers` can show it.
declined map[string]string
paused string
}
// newHealing is the serving controller's runner, its acts the real ones.
func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Teller, js *broker.JetStream) *healing {
h := &healing{open: open, keeper: keeper, teller: teller, js: js, acting: link.Holding,
epoch: func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) },
now: time.Now, say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
reportWait: 45 * time.Second, declined: map[string]string{}}
h.askReport = func(ctx context.Context, node string) error {
if h.js == nil {
return errors.New("this controller is not on the bus")
}
return askToReport(ctx, h.js.Conn(), node)
}
h.sendAgain = func(ctx context.Context, node string) error {
// **Never what a policy or a plan holds back** (ADR 0221): a send by the mesh itself is a push
// that did not name the machine — a person's word sends a held build, a healer's does not.
held, err := heldMachines(ctx, open, []string{node})
if err != nil {
return err
}
if why := held[node]; len(why) > 0 {
return fmt.Errorf("not sent again: it would move what a policy or a plan holds back (ADR 0221) — %s; "+
"`push %s` sends it, on a person's word", strings.Join(why, "; "), node)
}
return sendTo(ctx, open, []string{node})
}
h.assertObjects = func(ctx context.Context) error {
if h.js == nil {
return errors.New("this controller is not on the bus")
}
return assertOnSend(ctx, open.inventory, h.js, " ")
}
h.resetConsumer = func(stream, name string) (string, error) {
if h.js == nil {
return "", errors.New("this controller is not on the bus")
}
before, after, err := h.js.ResetConsumer(stream, name)
if err != nil {
return "", err
}
return fmt.Sprintf("it was %d behind with %d unacknowledged; it delivers from now, %d pending", before.Pending,
before.AckPending, after.Pending), nil
}
return h
}
// askToReport asks one machine's node-engine to say again what it last applied (to-be 45 §6). On core
// NATS, fired and flushed: the answer is the machine's ordinary report, read from the store.
func askToReport(ctx context.Context, conn *nats.Conn, node string) error {
body, err := json.Marshal(map[string]any{"asked": time.Now().UTC(), "by": "the controller's healer H1"})
if err != nil {
return err
}
if err := conn.Publish(broker.AskReportSubject(node), body); err != nil {
return err
}
flushing, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
return conn.FlushWithContext(flushing)
}
// keep runs the healers until ctx ends.
func (h *healing) keep(ctx context.Context) {
tick := time.NewTicker(healEvery)
defer tick.Stop()
for {
h.tick(ctx)
select {
case <-ctx.Done():
return
case <-tick.C:
}
}
}
// tick is one look at what is open, and every act it calls for.
func (h *healing) tick(ctx context.Context) {
if h.acting != nil && !h.acting() {
return
}
epoch, err := h.epoch(ctx)
switch {
case err != nil:
h.pause(fmt.Sprintf("this controller may not act: %v", err))
return
case epoch == 0:
h.pause("this controller serves without the lease (S12): a repair waits for it")
return
}
inv := h.open.inventory
now := h.now()
heals, err := inv.HealsSince(ctx, now.Add(-24*time.Hour))
if err != nil {
// Blind: nothing is done, and that is said — never read as "no heals, budgets whole".
h.reconcile(ctx, []conditions.Observation{{Scope: conditions.ScopeProbe, ID: "healers", Token: "failed",
Kind: kindHealersBlind, Severity: conditions.Warning,
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
Said: firstLine(err.Error())}})
return
}
open, err := h.keeper.Open(ctx)
if err != nil {
h.say("the healers cannot read the open conditions, and act on nothing: %v", err)
return
}
acts := actsWithin(heals, now.Add(-healBrakeWindow))
if braked, said := brakeHolds(acts, open, now); braked {
h.reconcile(ctx, []conditions.Observation{brakeObservation(acts, said)})
h.pause("the mesh-wide brake holds: " + said)
return
}
h.reconcile(ctx, nil)
h.resume()
for _, c := range open {
row, ok := healerFor(c.Kind)
if !ok || c.Escalated() {
continue
}
budget, applies, why, err := row.applies(ctx, h, c)
if err != nil {
h.decline(c.Key, row.ID, "could not tell whether it applies: "+err.Error())
continue
}
if !applies {
h.decline(c.Key, row.ID, why)
continue
}
h.forget(c.Key)
spent := spentOn(heals, row, budget, now)
if n := len(spent); n > 0 && now.Sub(spent[n-1].At) < row.Settle {
continue // its last act is still the observation's to judge
}
if len(spent) >= row.Budget {
h.escalate(ctx, row, c, budget, spent)
continue
}
if len(acts) >= healBrakeLimit {
return // the brake takes hold on the next look, said there
}
if h.act(ctx, row, c, budget, len(spent)) {
acts = append(acts, inventory.Heal{At: now})
}
}
}
// act is one healer's act on one condition: begun in the store, made, finished, kept in the
// condition's tried and said. False when it could not even be begun.
func (h *healing) act(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent int) bool {
inv := h.open.inventory
begun, err := inv.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
BudgetKey: budget, Act: row.Repair, At: h.now()})
if err != nil {
h.say("healer %s did not act on %s: %v", row.ID, c.Key, err)
return false
}
act, said, err := row.repair(ctx, h, c)
outcome := inventory.HealActed
if err != nil {
outcome, said = inventory.HealFailed, err.Error()
}
if act == "" {
act = row.Repair
}
finishing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if ferr := inv.FinishHeal(finishing, begun.ID, outcome, act+" — "+said); ferr != nil {
h.say("healer %s acted on %s and could not record what came of it: %v", row.ID, c.Key, ferr)
}
budgetWords := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window)
attempt := conditions.Attempt{What: act, Outcome: outcome + ": " + said + " (" + budgetWords + ")",
By: "healer " + row.ID}
if _, _, terr := h.keeper.Tried(finishing, c.Key, attempt, conditions.ResolverHealer(row.ID)); terr != nil {
h.say("healer %s acted on %s and could not keep it in the condition: %v", row.ID, c.Key, terr)
}
h.tell(finishing, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: act, Outcome: outcome,
Said: said, Budget: budgetWords, Epoch: begun.Epoch})
h.say("healer %s %s %s: %s — %s (%s)", row.ID, outcome, c.Key, act, said, budgetWords)
return true
}
// escalate is a spent budget: the condition is the operator's now, urgent, with what was tried. Said
// once: an escalated condition is passed over by every healer until observation clears it.
func (h *healing) escalate(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent []inventory.Heal) {
var tried []string
for _, s := range spent {
tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.At.UTC().Format("15:04 MST"), firstLine(s.Said)))
}
said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window,
c.Key, strings.Join(tried, "; "))
if _, err := h.open.inventory.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
BudgetKey: budget, Act: "handed the condition to the operator", Outcome: inventory.HealEscalated, Said: said,
At: h.now()}); err != nil {
h.say("healer %s could not record handing %s to the operator, and does not: %v", row.ID, c.Key, err)
return
}
attempt := conditions.Attempt{What: "handed to the operator: nothing in the mesh will repair it now",
Outcome: inventory.HealEscalated + ": " + said, By: "healer " + row.ID}
if _, _, err := h.keeper.Escalate(ctx, c.Key, attempt); err != nil {
h.say("healer %s could not hand %s to the operator: %v", row.ID, c.Key, err)
}
h.tell(ctx, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator",
Outcome: inventory.HealEscalated, Said: said, Budget: fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window)})
h.say("healer %s handed %s to the operator: %s", row.ID, c.Key, said)
}
// healerActed is the body of `healer-acted` (to-be 45 §7): the healer, the condition, the act and its
// outcome, and where the budget stands. **A contract**, like a condition's events: the operator-channel's
// holder may say it.
type healerActed struct {
Event string `json:"event"`
At time.Time `json:"at"`
Healer string `json:"healer"`
By string `json:"by"`
Condition string `json:"condition"`
Kind string `json:"kind"`
Act string `json:"act"`
// Outcome is acted, failed or escalated. Acted says the act was made, not that it worked: the
// condition clearing says that.
Outcome string `json:"outcome"`
Said string `json:"said"`
Budget string `json:"budget"`
Epoch uint64 `json:"epoch,omitempty"`
Show string `json:"show"`
}
// tell says a heal on the bus. Not said is said in the log: the act and the condition's tried still
// hold it.
func (h *healing) tell(ctx context.Context, e healerActed) {
e.Event, e.At, e.By = link.KeyHealerActed, h.now().UTC(), "healer "+e.Healer
e.Show = "mesh-controller.conditions key=" + e.Condition
if h.teller == nil {
return
}
body, err := json.Marshal(e)
if err != nil {
return
}
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := h.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyHealerActed, body); err != nil {
h.say("healer %s %s %s, and that could NOT be said on the bus: %v", e.Healer, e.Outcome, e.Condition, err)
}
}
// reconcile keeps the healers' own conditions: the brake, and their being blind.
func (h *healing) reconcile(ctx context.Context, observed []conditions.Observation) {
if err := h.keeper.Reconcile(ctx, sourceHealers, observed); err != nil {
h.say("the healers' own conditions could not be kept: %v", err)
}
}
func (h *healing) pause(why string) {
h.mu.Lock()
defer h.mu.Unlock()
if h.paused != why {
h.say("the healers act on nothing: %s", why)
}
h.paused = why
}
func (h *healing) resume() {
h.mu.Lock()
defer h.mu.Unlock()
if h.paused != "" {
h.say("the healers act again")
}
h.paused = ""
}
// decline remembers why a healer passed a condition over, said once.
func (h *healing) decline(key, healer, why string) {
h.mu.Lock()
defer h.mu.Unlock()
if h.declined[key] != why {
h.say("healer %s leaves %s alone: %s", healer, key, why)
}
h.declined[key] = why
}
func (h *healing) forget(key string) {
h.mu.Lock()
defer h.mu.Unlock()
delete(h.declined, key)
}
// actsWithin are the heals since a moment that were acts — begun, made or failed; an escalation is a
// saying, not an act, and the brake does not count it.
func actsWithin(heals []inventory.Heal, since time.Time) []inventory.Heal {
var out []inventory.Heal
for _, h := range heals {
if !h.At.Before(since) && h.Outcome != inventory.HealEscalated {
out = append(out, h)
}
}
return out
}
// spentOn is one healer's acts against one budget key within its window, oldest first.
func spentOn(heals []inventory.Heal, row healerRow, budget string, now time.Time) []inventory.Heal {
var out []inventory.Heal
for _, h := range actsWithin(heals, now.Add(-row.Window)) {
if h.Healer == row.ID && h.BudgetKey == budget {
out = append(out, h)
}
}
return out
}
// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake
// already said and an act within the hour still — it lets go an hour after the last act, not the first.
func brakeHolds(acts []inventory.Heal, open []conditions.Condition, now time.Time) (bool, string) {
said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), healBrakeWindow, healBrakeLimit)
if len(acts) >= healBrakeLimit {
return true, said
}
held := slices.ContainsFunc(open, func(c conditions.Condition) bool { return c.Kind == kindHealersBraked })
if held && len(acts) > 0 {
last := acts[len(acts)-1].At
return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said,
last.Add(healBrakeWindow).UTC().Format("15:04 MST"))
}
return false, said
}
// brakeObservation is the brake, as the urgent condition that says it.
func brakeObservation(acts []inventory.Heal, said string) conditions.Observation {
counts := map[string]int{}
for _, a := range acts {
if a.Healer != "" {
counts[a.Healer+" on "+a.ConditionKey]++
}
}
var most []string
for what, n := range counts {
most = append(most, fmt.Sprintf("%s ×%d", what, n))
}
sort.Strings(most)
return conditions.Observation{Scope: conditions.ScopeMesh, ID: "healers", Token: "braked", Kind: kindHealersBraked,
Severity: conditions.Urgent,
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
"healer looping, not repairing — " + said,
Said: strings.Join(most, ", ")}
}
// --- H1 ----------------------------------------------------------------------------------------------
// appliesToAMachine is H1's: a machine named, its budget counted against the condition.
func appliesToAMachine(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
if c.Subject.Machine == "" {
return "", false, "it names no machine", nil
}
return c.Key, true, "", nil
}
// repairReport is H1: ask the machine to report; if what comes back is not what it was sent, or nothing
// comes, send it again.
func repairReport(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
node := c.Subject.Machine
inv := h.open.inventory
// The store's clock, as the report's time is: not the runner's, which a test moves by hand.
asked := time.Now()
askErr := h.askReport(ctx, node)
current, heard := false, false
if askErr == nil {
deadline := time.Now().Add(h.reportWait)
for {
r, found, err := lastReportOf(ctx, inv, node)
if err != nil {
return "", "", err
}
if found && r.At != nil && r.At.After(asked) {
heard, current = true, r.Current
if current {
break
}
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
return "", "", ctx.Err()
case <-time.After(min(2*time.Second, h.reportWait/4+time.Millisecond)):
}
}
}
if current {
return "asked " + node + "'s node-engine to say again what it last applied",
"it reported the declaration it was sent: its report had not reached the mesh", nil
}
why := "it said nothing within " + h.reportWait.String() + " — its node-engine may be older than the report verb"
switch {
case askErr != nil:
why = "it could not be asked: " + askErr.Error()
case heard:
why = "it reported a declaration other than the one it was sent"
}
if err := h.sendAgain(ctx, node); err != nil {
return "asked " + node + " to report, then sent it its current declaration again", why + "; the send failed",
err
}
return "asked " + node + " to report, then sent it its current declaration again", why + "; sent again", nil
}
// lastReportOf is one machine's last report beside its last send.
func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (inventory.Reported, bool, error) {
reports, err := inv.LastReports(ctx)
if err != nil {
return inventory.Reported{}, false, err
}
for _, r := range reports {
if r.Node == node {
return r, true, nil
}
}
return inventory.Reported{}, false, nil
}
// --- H2 ----------------------------------------------------------------------------------------------
// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty
// when it is neither, which is not H2's to repair.
func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) {
if p.Release != nil {
return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236)
}
recent, err := inv.RecentPlans(ctx, 50)
if err != nil {
return "", "", err
}
for _, newer := range recent {
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded {
continue
}
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
"and branch", newer.ID, newer.Repository, short(newer.Commit)), nil
}
for _, tier := range p.Tiers {
for _, m := range tier {
s := p.Modules[m]
if s == nil || (s.State != "built" && s.State != "failed") {
return "", "", nil
}
if s.State == "built" && s.SentAt == nil {
u, err := inv.UpgradeOf(ctx, m)
if err != nil {
return "", "", err
}
if u.RollOut {
return "", "", nil
}
}
}
}
return inventory.PlanDone, "finished: every module of every tier is built or failed, and every one that rolls " +
"out was sent — nothing is left to wait on", nil
}
// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished.
func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
if c.Subject.Scope == conditions.ScopeDelivery {
return appliesToAStalledDelivery(ctx, h, c)
}
p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID)
if err != nil {
return "", false, "", err
}
if !p.Open() {
return "", false, "the plan is " + p.State + " already: its condition clears on the next look", nil
}
state, _, err := planStale(ctx, h.open.inventory, p)
if err != nil {
return "", false, "", err
}
if state == "" {
return "", false, "its wait is neither superseded nor finished: it waits on something still to come, " +
"which its own signal says", nil
}
return c.Key, true, "", nil
}
// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set.
func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
if c.Subject.Scope == conditions.ScopeDelivery {
return repairDelivery(ctx, h, c)
}
inv := h.open.inventory
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return "", "", err
}
defer release()
p, err := inv.PlanByID(ctx, c.Subject.ID)
if err != nil {
return "", "", err
}
if !p.Open() {
return "closed nothing", "the plan is " + p.State + " already", nil
}
state, why, err := planStale(ctx, inv, p)
if err != nil {
return "", "", err
}
if state == "" {
return "closed nothing", "its wait is no longer superseded or finished", nil
}
p.State = state
p.Note = fmt.Sprintf("closed by healer H2 at tier %d: %s", p.Tier, why)
if state != inventory.PlanDone {
sayUnsent(&p, func(m string) bool {
u, err := inv.UpgradeOf(ctx, m)
return err == nil && u.RollOut
})
}
if err := inv.SavePlan(ctx, &p); err != nil {
return "", "", err
}
return "closed the plan " + p.ID + " (" + state + ")", why, nil
}
// --- H3 ----------------------------------------------------------------------------------------------
// appliesToAnObject is H3's: every holder or consumer the probe or the bus names, its budget per object.
func appliesToAnObject(_ context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
if h.assertObjects == nil {
return "", false, "this controller is not on the bus", nil
}
return c.Key, true, "", nil
}
// repairObjects is H3: the send's own assertion of every stream, consumer and seat worker.
func repairObjects(ctx context.Context, h *healing, _ conditions.Condition) (string, string, error) {
if err := h.assertObjects(ctx); err != nil {
return "", "", err
}
return "asserted the bus's streams, consumers and seat workers again",
"every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there", nil
}
// --- H4 ----------------------------------------------------------------------------------------------
// resettable is why a consumer may be reset by the mesh itself, from the stream table; empty when not.
func resettable(stream, name string) string {
for _, c := range broker.MeshConsumers() {
if c.Stream == stream && c.Name == name {
return c.Resettable
}
}
return ""
}
// consumerOf is the stream and consumer a bus condition names: `<stream>.<consumer>`.
func consumerOf(c conditions.Condition) (string, string, bool) {
stream, name, found := strings.Cut(c.Subject.ID, ".")
return stream, name, found && stream != "" && name != ""
}
// appliesToAResettableConsumer is H4's: only a consumer the stream table marks resettable.
func appliesToAResettableConsumer(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
stream, name, ok := consumerOf(c)
if !ok {
return "", false, "it names no consumer", nil
}
if resettable(stream, name) == "" {
return "", false, fmt.Sprintf("%s on %s is not marked resettable in the stream table: what a reset drops, "+
"nothing would catch up", name, stream), nil
}
return c.Key, true, "", nil
}
// repairConsumer is H4: `broker consumer-reset`, made by the mesh.
func repairConsumer(_ context.Context, h *healing, c conditions.Condition) (string, string, error) {
stream, name, _ := consumerOf(c)
said, err := h.resetConsumer(stream, name)
if err != nil {
return "", "", err
}
return fmt.Sprintf("re-made %s on %s to deliver from now", name, stream),
said + "; " + resettable(stream, name), nil
}
// --- the verb ----------------------------------------------------------------------------------------
// healersCommand is `healers`: the registry, what the healers did lately, and the brake.
func healersCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("healers", flag.ContinueOnError)
daysFlag := set.Int("days", 7, "how many days of acts back")
jsonFlag := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("healers [--days N] [--json]")
}
days, asJSON := *daysFlag, *jsonFlag
if days <= 0 {
return fmt.Errorf("healers --days takes a number of days, not %d", days)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
now := time.Now()
heals, err := open.inventory.HealsSince(ctx, now.Add(-time.Duration(days)*24*time.Hour))
if err != nil {
return fmt.Errorf("what the healers did cannot be read: %w", err)
}
conds, condErr := openConditions(ctx)
braked, said := brakeHolds(actsWithin(heals, now.Add(-healBrakeWindow)), conds, now)
brake := map[string]any{"holds": braked, "said": said, "limit": healBrakeLimit, "window": healBrakeWindow.String()}
if condErr != nil {
brake["conditions"] = "the open conditions could not be read, so whether the brake was said is not known: " +
condErr.Error()
}
var rows []map[string]any
for _, r := range healerRegistry {
rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair,
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
"then": r.Then, "acts-in": r.ActsIn, "event": r.Event, "from": r.From})
}
if heals == nil {
heals = []inventory.Heal{}
}
if asJSON {
return printJSON(map[string]any{"healers": rows, "heals": heals, "brake": brake, "days": days,
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"})
}
for _, r := range healerRegistry {
fmt.Printf("%s %s\n → %s\n budget %d within %s; then %s (in %s)\n", r.ID, r.Condition, r.Repair, r.Budget,
r.Window, r.Then, r.ActsIn)
}
fmt.Printf("\nthe brake: %s — %s\n\n", map[bool]string{true: "HOLDS, every healer has stopped", false: "off"}[braked], said)
if len(heals) == 0 {
fmt.Printf("no healer acted in the last %d day(s)\n", days)
return nil
}
for i := len(heals) - 1; i >= 0; i-- {
x := heals[i]
fmt.Printf("%s %s %-9s %s\n %s\n", x.At.Local().Format("2006-01-02 15:04"), x.Healer, x.Outcome, x.ConditionKey,
firstLine(x.Said))
}
return nil
}
// forgettingOldHeals removes heals past their keeping once a day, by the controller acting only.
func forgettingOldHeals(ctx context.Context, inv *inventory.Inventory) {
for {
if link.Holding() {
if n, err := inv.ForgetOldHeals(ctx); err != nil {
fmt.Printf("heals older than %s could not be removed: %v\n", inventory.HealsKeptFor, err)
} else if n > 0 {
fmt.Printf("removed %d heal(s) older than %s\n", n, inventory.HealsKeptFor)
}
}
select {
case <-ctx.Done():
return
case <-time.After(24 * time.Hour):
}
}
}
// healsCount is what the healers did, counted for `status`.
type healsCount struct {
Acts int `json:"acts"`
Escalated int `json:"escalated"`
}
// countHeals counts acts and escalations.
func countHeals(heals []inventory.Heal) *healsCount {
out := &healsCount{}
for _, h := range heals {
if h.Outcome == inventory.HealEscalated {
out.Escalated++
} else {
out.Acts++
}
}
return out
}
cmd/mesh-controller/healers_test.go
-640
View File
@@ -1,640 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"slices"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
// checked"): **every row is walked.** Its kinds are ones the mesh raises — a row of the signals table, a
// probe of the self-check, or a provider's event — it has a budget, a window and a settle inside it,
// what happens when the budget is spent, and the event each act is said as; a healer the controller runs
// has its applies and its repair, and an induced failure below that sees it act, say so and brake. A
// row added without one fails, so the registry cannot grow a healer nobody has seen act.
// raisedKinds is every condition kind the mesh raises, and what raises it.
func raisedKinds() map[string]string {
out := map[string]string{kindProviderFailing: "the provisioner.failing event (ADR 0224)"}
for _, r := range signalsTable {
for _, k := range kindsOf(r) {
out[k] = r.Row
}
}
for _, p := range probeRegistry {
out[p.Kind] = p.ID
for _, k := range p.Raises {
out[k] = p.ID
}
}
return out
}
// inducedFailures are the healers seen acting in this file, by id: a row without one fails.
var inducedFailures = map[string]string{
"H1": "TestH1AsksAMachineToReportAndSendsItAgain",
"H2": "TestH2ClosesAPlanAnotherHasTakenOver",
"H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget",
"H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers",
}
func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) {
kinds := raisedKinds()
source, err := os.ReadFile("healers_test.go")
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
answered := map[string]string{}
for _, r := range healerRegistry {
t.Run(r.ID, func(t *testing.T) {
if seen[r.ID] {
t.Fatalf("%s is in the registry twice", r.ID)
}
seen[r.ID] = true
if len(r.Kinds) == 0 || r.Condition == "" || r.Repair == "" || r.Then == "" || r.From == "" || r.ActsIn == "" {
t.Fatalf("%s does not say what it answers, what it does, what then, and which hand act it replaces: %+v", r.ID, r)
}
for _, k := range r.Kinds {
if _, ok := kinds[k]; !ok {
t.Errorf("%s answers %q, which nothing in the mesh raises", r.ID, k)
}
if other, twice := answered[k]; twice {
t.Errorf("%q is answered by %s and %s: one healer per kind", k, other, r.ID)
}
answered[k] = r.ID
}
if r.Budget <= 0 || r.Window <= 0 || r.Settle <= 0 || r.Settle > r.Window {
t.Errorf("%s has no budget it can spend: %d within %s, settled after %s", r.ID, r.Budget, r.Window, r.Settle)
}
if r.Event == "" {
t.Errorf("%s says nothing when it acts", r.ID)
}
if r.ActsIn != actsInController {
if r.repair != nil || r.applies != nil {
t.Errorf("%s acts in %s and the controller would act for it too", r.ID, r.ActsIn)
}
return
}
if r.repair == nil || r.applies == nil {
t.Fatalf("%s acts in the controller and has no repair or no applies", r.ID)
}
if r.Event != link.KeyHealerActed || !slices.Contains(broker.ControllerStates, r.Event) {
t.Errorf("%s is said as %q, which the controller's grant does not permit", r.ID, r.Event)
}
if inducedFailures[r.ID] == "" {
t.Errorf("%s has no induced failure: a healer nobody has seen act", r.ID)
} else if !strings.Contains(string(source), "func "+inducedFailures[r.ID]+"(t *testing.T)") {
t.Errorf("%s's induced failure %s is not a test in this file", r.ID, inducedFailures[r.ID])
}
})
}
for id := range inducedFailures {
if !seen[id] {
t.Errorf("an induced failure for %s, which the registry does not have", id)
}
}
if healBrakeLimit <= 0 || healBrakeWindow <= 0 {
t.Error("the mesh-wide brake holds nothing")
}
}
// heard keeps the healer-acted events said.
type heard struct {
mu sync.Mutex
acts []healerActed
}
func (h *heard) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
if seat != conditions.Seat || event != link.KeyHealerActed {
return errors.New("said under the wrong seat or name: " + seat + " " + event)
}
var e healerActed
if err := json.Unmarshal(body, &e); err != nil {
return err
}
h.mu.Lock()
defer h.mu.Unlock()
h.acts = append(h.acts, e)
return nil
}
func (h *heard) said() []healerActed {
h.mu.Lock()
defer h.mu.Unlock()
return append([]healerActed(nil), h.acts...)
}
// testClock is a moment a test moves by hand.
type testClock struct {
mu sync.Mutex
at time.Time
}
func (c *testClock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.at
}
func (c *testClock) pass(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.at = c.at.Add(d)
}
// healingOn is a runner over a mesh's stores and its condition store, under epoch 57, every act a
// fake that fails the test unless the test gives it.
func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) {
t.Helper()
if conditionsFrom == nil {
t.Fatal("the mesh has no condition store")
}
told, clock := &heard{}, &testClock{at: time.Now()}
// The store's gate, as the serving controller's is the lease's (stores.go).
open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
h := &healing{open: open, keeper: conditionsFrom, teller: told,
epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now,
say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond,
declined: map[string]string{}}
h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil }
h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil }
h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil }
h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil }
return h, told, clock
}
// sentNotReported raises S2 for a machine, as the watchdog does.
func sentNotReported(t *testing.T, node string) string {
t.Helper()
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "sent-not-reported", Machine: node,
Severity: conditions.Warning, Summary: node + " was sent a declaration and has not reported it", Source: "S2"}
if _, err := conditionsFrom.Observe(t.Context(), o); err != nil {
t.Fatal(err)
}
return o.Key()
}
// **H1, the commonest hand act** (031/01 §f: a push by hand to unstick a plan waiting on a report, four
// times): the machine is asked to report; a report that names what it was sent is all it takes, and one
// that does not — or none — is a send of its current declaration again. Each act kept in `tried` as
// `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more.
func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, clock := healingOn(t, open)
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil {
t.Fatal(err)
}
key := sentNotReported(t, "laptop")
// First: the report had not reached the mesh, and asking for it brings it.
asked := 0
h.askReport = func(ctx context.Context, node string) error {
asked++
if node != "laptop" {
t.Errorf("asked %s", node)
}
_, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied,
At: time.Now().Add(time.Second), Declared: "d2"})
return err
}
h.tick(ctx)
c, found, err := conditionsFrom.Get(ctx, key)
if err != nil || !found {
t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err)
}
if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") ||
c.Resolver != "healer:H1" {
t.Fatalf("after the first act: asked %d, %+v", asked, c)
}
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed ||
acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 {
t.Fatalf("the act was not said as healer-acted: %+v", acts)
}
// Within its settle, nothing more: the observation has its turn.
clock.pass(time.Minute)
h.tick(ctx)
if asked != 1 {
t.Fatalf("acted again inside the settle: asked %d", asked)
}
// Second: the machine says nothing, so it is sent again.
clock.pass(3 * time.Minute)
sent := 0
h.askReport = func(context.Context, string) error { asked++; return nil }
h.sendAgain = func(_ context.Context, node string) error { sent++; return nil }
h.tick(ctx)
if asked != 2 || sent != 1 {
t.Fatalf("the second act: asked %d, sent %d", asked, sent)
}
c, _, _ = conditionsFrom.Get(ctx, key)
if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") {
t.Fatalf("tried %+v", c.Tried)
}
// The budget is spent: the operator's, urgent, said; and no healer touches it again.
clock.pass(4 * time.Minute)
h.tick(ctx)
c, _, _ = conditionsFrom.Get(ctx, key)
if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 ||
!strings.Contains(c.Tried[2].Outcome, "budget of 2") {
t.Fatalf("not handed to the operator: %+v", c)
}
if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated {
t.Fatalf("the escalation was not said: %+v", acts)
}
clock.pass(time.Hour)
h.tick(ctx)
if asked != 2 || sent != 1 || len(told.said()) != 3 {
t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said()))
}
heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour))
if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed ||
heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 {
t.Fatalf("the heals kept: %+v %v", heals, err)
}
// And a heal is not a hand act: what S15 counts never sees it.
for _, x := range heals {
if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") {
t.Errorf("a heal reads as a hand act: %+v", x)
}
}
}
// **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing.
func TestNoHealerActsWithoutTheLease(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, _ := healingOn(t, open)
sentNotReported(t, "laptop")
h.epoch = func(context.Context) (uint64, error) { return 0, nil }
h.tick(ctx)
h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }
h.tick(ctx)
h.epoch = func(context.Context) (uint64, error) { return 57, nil }
h.acting = func() bool { return false }
h.tick(ctx)
if len(told.said()) != 0 {
t.Fatalf("a healer acted without the lease: %+v", told.said())
}
}
// **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour
// after the last.
func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, clock := healingOn(t, open)
for i := 0; i < healBrakeLimit; i++ {
if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent",
Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed,
At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil {
t.Fatal(err)
}
}
sentNotReported(t, "laptop")
h.tick(ctx) // the fakes fail the test if anything acts
braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked")
if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") {
t.Fatalf("the brake was not said: %+v %v", braked, err)
}
if len(told.said()) != 0 {
t.Fatalf("a healer acted under the brake: %+v", told.said())
}
// Past the hour of the first act, still held: it lets go an hour after the last.
clock.pass(30 * time.Minute)
h.tick(ctx)
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held {
t.Fatal("the brake let go before an hour had passed since the last act")
}
clock.pass(31 * time.Minute)
asked := 0
h.askReport = func(context.Context, string) error { asked++; return nil }
h.sendAgain = func(context.Context, string) error { return nil }
h.tick(ctx)
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held {
t.Fatal("the brake held an hour after the last act")
}
if asked != 1 {
t.Fatalf("the healers did not act again after the brake let go: asked %d", asked)
}
}
// **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is
// still to come: that one is its own signal's, not a healer's.
func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, _ := healingOn(t, open)
created := time.Now().UTC().Add(-time.Hour)
older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created,
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}}
waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created,
State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}}
newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute),
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}
for _, p := range []*inventory.Plan{&older, &waiting, &newer} {
if err := open.inventory.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
}
for _, id := range []string{"plan-old", "plan-other"} {
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled",
Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil {
t.Fatal(err)
}
}
h.tick(ctx)
closed, err := open.inventory.PlanByID(ctx, "plan-old")
if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") ||
!strings.Contains(closed.Note, "plan-new") {
t.Fatalf("the superseded plan: %+v %v", closed, err)
}
if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding {
t.Fatalf("a plan still waiting was closed: %+v", other)
}
if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf {
t.Fatalf("a plan H2 does not repair was touched: %+v", c)
}
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" {
t.Fatalf("said %+v", acts)
}
// Finished: every module built, none rolled out unsent — closed as done.
done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created,
State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}}
if err := open.inventory.SavePlan(ctx, &done); err != nil {
t.Fatal(err)
}
if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") {
t.Fatalf("a finished plan reads %q %q %v", state, why, err)
}
}
// **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said
// and left — what a reset drops, nothing would catch up for it.
func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, _, _ := healingOn(t, open)
marked := 0
for _, c := range broker.MeshConsumers() {
if c.Resettable != "" {
marked++
if c.Stream != broker.EventsStream || c.Name != broker.ControllerName {
t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream)
}
}
}
if marked != 1 {
t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked)
}
for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} {
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil {
t.Fatal(err)
}
}
h.tick(ctx) // the fake reset fails the test if it is called
reset := ""
h.resetConsumer = func(stream, name string) (string, error) {
reset = stream + "." + name
return "it was 1500 behind", nil
}
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller",
Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil {
t.Fatal(err)
}
h.tick(ctx)
if reset != "EVENTS.controller" {
t.Fatalf("reset %q", reset)
}
}
// **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
url := testbus.URL(t)
open := aMesh(t)
ctx := t.Context()
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
_ = js.Context().DeleteStream(s)
}
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
t.Fatal(err)
}
h, told, clock := healingOn(t, open)
h.js = js
h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") }
d := &doctor{open: open, js: js}
probe := func() {
t.Helper()
found, err := probeConsumers(ctx, d)
if err != nil {
t.Fatal(err)
}
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
t.Fatal(err)
}
}
const key = "bus.NODES.laptop.missing"
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
t.Fatal(err)
}
probe()
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" {
t.Fatalf("the deleted consumer was not raised: %+v", c)
}
h.tick(ctx)
if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil {
t.Fatalf("H3 did not make the consumer again: %v", err)
}
c, still, _ := conditionsFrom.Get(ctx, key)
if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" {
t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c)
}
probe()
if _, still, _ := conditionsFrom.Get(ctx, key); still {
t.Fatal("the probe's next run did not clear what H3 repaired")
}
// Kept in the history as the keeper says it, a moment later.
var cleared *conditions.Event
for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) {
history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute))
if err != nil {
t.Fatal(err)
}
for i := range history {
if history[i].Key == key && history[i].Change == conditions.ChangeCleared {
cleared = &history[i]
}
}
}
if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 {
t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared)
}
// Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops.
clock.pass(10 * time.Minute)
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
t.Fatal(err)
}
probe()
h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil }
h.tick(ctx)
c, _, _ = conditionsFrom.Get(ctx, key)
if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 {
t.Fatalf("the spent budget was not handed to the operator: %+v", c)
}
acts := told.said()
if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated {
t.Fatalf("said %+v", acts)
}
clock.pass(2 * time.Hour)
h.tick(ctx)
if len(told.said()) != 2 {
t.Fatal("a healer acted on what the operator holds")
}
}
// **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
// receives what comes next: a reset that left the controller deaf would be the incident.
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
url := testbus.URL(t)
open := aMesh(t)
ctx := t.Context()
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
_ = js.Context().DeleteStream(s)
}
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
t.Fatal(err)
}
// Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none.
events := make(chan *nats.Msg, 64)
sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
followed := broker.ControllerFollows[0]
for i := 0; i < consumerFarBehind+200; i++ {
if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil {
t.Fatal(err)
}
}
d := &doctor{open: open, js: js}
probe := func() []conditions.Observation {
t.Helper()
found, err := probeConsumers(ctx, d)
if err != nil {
t.Fatal(err)
}
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
t.Fatal(err)
}
return found
}
probe()
const key = "bus.EVENTS.controller.behind"
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind {
t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c)
}
h, told, _ := healingOn(t, open)
h.resetConsumer = func(stream, name string) (string, error) {
before, after, err := js.ResetConsumer(stream, name)
if err != nil {
return "", err
}
return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) +
" pending now", nil
}
h.tick(ctx)
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed {
t.Fatalf("said %+v", acts)
}
if found := probe(); len(found) != 0 {
t.Fatalf("after the reset the probe still finds %+v", found)
}
if _, still, _ := conditionsFrom.Get(ctx, key); still {
t.Fatal("the probe's next run did not clear what H4 repaired")
}
// What comes next still reaches the controller.
for len(events) > 0 {
<-events
}
if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil {
t.Fatal(err)
}
deadline := time.After(10 * time.Second)
for {
select {
case m := <-events:
if strings.Contains(string(m.Data), "after") {
return
}
_ = m.Ack()
case <-deadline:
t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf")
}
}
}
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop"))
if err != nil {
t.Fatal(err)
}
if err := askToReport(t.Context(), conn, "laptop"); err != nil {
t.Fatal(err)
}
msg, err := asked.NextMsg(5 * time.Second)
if err != nil || !strings.Contains(string(msg.Data), "healer H1") {
t.Fatalf("the machine heard %v, %v", msg, err)
}
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") ||
slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") {
t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err)
}
}
cmd/mesh-controller/health_probes.go
-226
View File
@@ -1,226 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The core components' health definitions, as probes in the self-check's registry (novox/hq to-be 45
// §8, §4 `H-*`). The same definitions judge a core component's new build on its first machine (the
// gate, gate.go); here they are run against every machine on the self-check's schedule, so a core
// component that stops being healthy between upgrades is said too.
//
// controller holds the lease, and says it is ready: its self-check ran, `status` answered in bound
// node-engine has reported its current declaration, under a build the mesh delivered
// node tools announced, and answer the bus's discovery
// bus every stream and durable consumer the mesh defines is there, and a request crosses the
// bus to every machine's node tools and back
const kindCoreUnhealthy = "core-unhealthy"
// probeControllerHealth is H-controller: the lease is held, fresh, by a controller that says it is
// ready — or one that started less than the witness's bound ago.
func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
h, found, err := theLease.holder(ctx)
if err != nil {
return nil, err
}
unhealthy := func(why string) []conditions.Observation {
node := d.host
if found && h.Host != "" {
node = h.Host
}
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: lease.ComponentController + "." + node,
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Urgent,
Summary: "the controller is not healthy: " + why}}
}
switch {
case !found:
return unhealthy("nobody holds the controller lease"), nil
case time.Since(h.Renewed) > lease.FreshWithin:
return unhealthy(fmt.Sprintf("the lease was last renewed %s ago", time.Since(h.Renewed).Round(time.Second))), nil
case (h.Health == nil || !h.Health.Ready) && time.Since(h.Taken) > lease.ReadyWithin:
why := "the controller holding the lease does not say it is ready"
if h.Health != nil && h.Health.Why != "" {
why += ": " + h.Health.Why
}
return unhealthy(why), nil
}
return nil, nil
}
// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the
// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send.
func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
return machinesHealth(ctx, d, hostModule)
}
// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them
// announced, answering the bus's discovery.
func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
return machinesHealth(ctx, d, broker.RuntimeModule)
}
// machinesHealth judges a component on every machine running it and heard from, by its health
// definition, as the gate does — with no condition taken as the build's doing.
func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) {
inv := d.open.inventory
running, err := inv.Running(ctx, component)
if err != nil {
return nil, err
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
f, err := gatherGateFacts(ctx, d.open, coreComponent(component))
if err != nil {
return nil, err
}
f.judged = false
heard := heardMachines(d)
var out []conditions.Observation
for _, node := range running {
if !heard[node] {
continue // a machine not heard from is S1's
}
if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound {
continue // inside the bound of a send: S2's, and the gate's
}
// What the machine did with what it was sent is not the component's health: the node-engine's is
// that it reported its current declaration at all, the node tools' that they answer.
if r := f.reports[node]; r.Current || component == broker.RuntimeModule {
now := time.Now()
r.Current, r.Outcome = true, inventory.OutcomeApplied
if r.At == nil {
r.At = &now
}
f.reports[node] = r
}
if component == hostModule {
// A node-engine reporting a build the mesh delivered, current or previous, is the version
// split's (D10), not ill health; a build the mesh did not deliver is.
f.engines[node] = deliveredOr(shelf[component], f.engines[node])
}
h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f)
if h == healthGood {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node,
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning,
Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)})
}
return sortedFound(out), nil
}
// deliveredOr is the reported engine build, or the current delivered one when the report names any
// build at all: what H-engine judges is that it reports, not which.
func deliveredOr(m catalogue.Manifest, reported string) string {
if reported == "" {
return reported
}
if v := deliveredVersions(m); len(v) > 0 {
return v[0]
}
return reported
}
// componentWords is a core component as a sentence names it.
func componentWords(component string) string {
switch component {
case hostModule:
return "node-engine"
case broker.RuntimeModule:
return "node tools"
case catalogue.ControllerSeatName:
return "controller"
}
return component
}
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a
// request crosses it to every machine's node tools and back.
func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
problems, err := busHealth(ctx, d)
if err != nil {
return nil, err
}
if len(problems) == 0 {
return nil, nil
}
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: "mesh", Token: "unhealthy",
Kind: kindCoreUnhealthy, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the bus is not healthy: %s", strings.Join(problems, "; ")),
Said: strings.Join(problems, "; ")}}, nil
}
// busHealth is the bus's health definition, as what is wanting: nothing when healthy. What the bus's
// planned step checks after the bus is replaced, too.
var busHealth = func(ctx context.Context, d *doctor) ([]string, error) {
if d.js == nil {
return nil, errors.New("this controller has no bus to ask")
}
streams, consumers, err := expectedBusObjects(ctx, d.open.inventory)
if err != nil {
return nil, err
}
js := d.js.Context()
var problems []string
for _, s := range streams {
if _, err := js.StreamInfo(s.Name, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
problems = append(problems, "the stream "+s.Name+" is missing")
} else if err != nil {
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
}
}
for _, c := range consumers {
_, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
if errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound) {
problems = append(problems, consumerWords(c)+" is missing")
} else if err != nil {
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
}
}
// The round trip: every machine heard from that runs the node tools answers across the bus.
running, err := d.open.inventory.Running(ctx, broker.RuntimeModule)
if err != nil {
return nil, err
}
heard := heardMachines(d)
var expected []string
for _, n := range running {
if heard[n] {
expected = append(expected, n)
}
}
if len(expected) > 0 {
answered, err := servedOnTheBus(ctx, d.js.Conn())
if err != nil {
return nil, err
}
var silent []string
for _, n := range expected {
if !answered[n].runtime {
silent = append(silent, n)
}
}
// One machine's tools silent is that machine's (H-tools); none answering is the bus.
if len(silent) == len(expected) {
slices.Sort(silent)
problems = append(problems, "no request crossed the bus and came back: no machine's node tools answered ("+
strings.Join(silent, ", ")+")")
}
}
return problems, nil
}
cmd/mesh-controller/held_back.go
+1 -24
View File
@@ -120,10 +120,6 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
if err != nil {
return nil, err
}
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, err
}
for _, node := range names {
plan, _, err := planFor(ctx, open, node)
if err != nil {
@@ -137,25 +133,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
if err != nil {
return nil, err
}
// A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it
// runs.
same := map[string]string{}
for m, was := range sent {
same[m] = was
if f.identical(m, was, current[m].Commit) {
same[m] = current[m].Commit
}
}
why := heldBack(node, modules, same, known, current, plans)
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
for _, mv := range f.moves(node, modules, same, known, false) {
if planStillToSend(plans, mv.Module, node) == "" {
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
}
}
if len(why) > 0 {
sort.Strings(why)
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
out[node] = why
}
}
@@ -254,7 +232,6 @@ func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held
return sendable{}, err
}
reportUnhostable(node, plan)
reportKept(held, plan)
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
if err == nil {
reportLeftOut(node, declared)
cmd/mesh-controller/held_back_test.go
+3 -21
View File
@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
r.declared = append(r.declared, s.node)
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
}
// aResolver is a module built from a repository, at a commit, with something on the machine that
@@ -193,10 +193,6 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
t.Fatal(err)
}
}
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
@@ -218,7 +214,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
}
before := digestOfLaptop()
// The change merges; the policy is record. `push anchor` sends the anchor...
// The change merges; the policy is the default, record. `push anchor` sends the anchor...
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
d.declared = nil
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
@@ -264,8 +260,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
t.Fatalf("the push did not say both:\n%s", said.String())
}
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
// (novox/hq ADR 0236).
// A policy that rolls out: the laptop is a consequence like any other, and sent.
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
t.Fatal(err)
}
@@ -274,19 +269,6 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
compose, d, "", &said); err != nil {
t.Fatal(err)
}
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
}
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
t.Fatal(err)
}
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
}
cmd/mesh-controller/holdings.go
-71
View File
@@ -6,8 +6,6 @@ import (
"sort"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -92,72 +90,3 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
}
return said, nil
}
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
//
// **The holders on record, and only the sole claimant when there are none** — the same answer the
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
// listed here. A holder off the private network is left out: a resolver named at an address nothing
// answers is a lookup that waits out its timeout on every name.
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
var replicated []catalogue.Seat
for _, s := range catalogue.Seats() {
if s.Replicated && s.Scope == catalogue.ScopeMesh {
replicated = append(replicated, s)
}
}
if len(replicated) == 0 {
return nil, nil
}
recorded, err := inv.Holdings(ctx)
if err != nil {
return nil, err
}
places, err := onTheNetwork(ctx, inv, shelf)
if err != nil {
return nil, err
}
address := map[string]string{}
for _, p := range places {
address[p.Name] = p.Address
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, err
}
out := map[string]map[string]string{}
for _, seat := range replicated {
var nodes []string
for _, h := range recorded {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
nodes = append(nodes, h.Node)
}
}
if len(nodes) == 0 {
var derived []string
for _, e := range entries {
for _, c := range e.Manifest.Claims {
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
derived = append(derived, e.On...)
}
}
}
if len(derived) == 1 {
nodes = derived
}
}
at := map[string]string{}
for _, n := range nodes {
if address[n] != "" {
at[overlay.InternalName(n)] = address[n]
}
}
out[seat.Name] = at
}
return out, nil
}
cmd/mesh-controller/identity_bound_test.go
-150
View File
@@ -1,150 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
// one consumer's identity.
// `module check` refuses the pull request that introduces an overflow, naming the module.
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
dir := t.TempDir()
write := func(name, body string) string {
p := filepath.Join(dir, name+".json")
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
return p
}
objects := write("objects", `{"module":"objects","version":"1",
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
resolver := write("resolver", `{"module":"resolver","version":"1",
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
var out bytes.Buffer
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
}
out.Reset()
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
if err == nil {
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
}
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
strings.Contains(out.String(), "networkmanager wants") {
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
}
}
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
// resolver provision, and a second consumer there overflows an object store's access key. The
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
// by push and by `status`, and the keyless consumer is granted with its long name.
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
Requires: []string{"wildcard-resolution"}})
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
// The consumer's machine resolves, and says which of its modules no provider will grant.
consumer, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
over := consumer.Overflowing()
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
}
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
// networkmanager, and no push to the provider could go through.
plan, settings, err := planFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
if err != nil {
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
}
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
}
grants, _, _, err := grantsFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
var keyless bool
for _, g := range grants {
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
}
if !keyless {
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
}
var granted []string
for _, c := range declared.Received["objects"]["s3-bucket"] {
granted = append(granted, c.From)
}
if strings.Join(granted, ",") != "files" {
t.Fatalf("the object store grants %v; files and only files fit", granted)
}
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
!strings.Contains(said, "left out of anchor's grants") {
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
}
// And `status` names it, and does not call the mesh well while it stands.
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
}
if asked.well() {
t.Fatal("a mesh with a consumer left out of its grants reads as well")
}
shown := printed(t, func() error { return printStatus(asked) })
if !strings.Contains(shown, "identified too long for a provision they require") ||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
t.Fatalf("status does not say it:\n%s", shown)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
Overflowing []catalogue.Overflow `json:"overflowing"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
doc.Overflowing[0].Bound.Max != 20 {
t.Fatalf("the document does not carry it: %v\n%s", err, body)
}
}
cmd/mesh-controller/issue_204_test.go
+4 -4
View File
@@ -45,11 +45,11 @@ func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *test
// fails leaves nothing composed for that machine, and the others are still composed.
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
calls := 0
allot := func(node string) (order, error) {
allot := func(node string) (int64, error) {
if node == "anchor" {
return order{}, context.DeadlineExceeded
return 0, context.DeadlineExceeded
}
return order{sequence: 7}, nil
return 7, nil
}
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
calls++
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
}
cancel() // the sender is going away: its context is cancelled between the send and the record
body := []byte(`{"declaration":1,"resources":[]}`)
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
digest, err := recordSent(ctx, inv, "anchor", body, nil)
if err != nil {
// NodeByName on the cancelled context may itself refuse; the record must still be possible
// through the detached context, so look the node up again on a live one.
cmd/mesh-controller/lease_command_test.go
-68
View File
@@ -1,68 +0,0 @@
package main
import (
"testing"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/lease"
)
// A command run at a shell (novox/hq to-be 45 §6): under the holder's epoch while a controller holds the
// lease, read at the moment it acts; under a lease of its own while none does, given back as it ends.
func TestACommandActsUnderTheHoldersEpochOrItsOwn(t *testing.T) {
url, kv := aBusForTheLease(t)
t.Setenv(broker.NATSVar, url)
ctx := t.Context()
// Nobody holds it: the command takes it, and gives it back.
cmd := &actor{}
own, err := cmd.epoch(ctx)
if err != nil || own == 0 {
t.Fatalf("a command with nobody holding the lease acts as %d (%v)", own, err)
}
if h, found, _ := lease.Current(ctx, kv); !found || h.Epoch != own {
t.Fatalf("the command's lease is not on the bus: %+v", h)
}
cmd.release()
if _, found, _ := lease.Current(ctx, kv); found {
t.Fatal("the command did not give its lease back as it ended")
}
// A controller holds it: a command acts under that epoch.
l, err := lease.Open(ctx, mustJetStream(t, url), broker.LeaseBucket, lease.Options{Holder: lease.Holder{Instance: "serving"}})
if err != nil {
t.Fatal(err)
}
held, err := l.TryTake(ctx)
if err != nil {
t.Fatal(err)
}
borrower := &actor{}
defer borrower.release()
if got, err := borrower.epoch(ctx); err != nil || got != held {
t.Fatalf("a command acts as %d (%v), want the holder's %d", got, err, held)
}
// The holder lets go: the command does not go on under an epoch nobody holds.
l.Release(ctx)
if _, err := borrower.epoch(ctx); err == nil {
t.Fatal("a command acted under an epoch nobody holds any more")
}
}
// mustJetStream is a connection of its own to the test bus.
func mustJetStream(t *testing.T, url string) jetstream.JetStream {
t.Helper()
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
js, err := jetstream.New(conn)
if err != nil {
t.Fatal(err)
}
return js
}
cmd/mesh-controller/lease_test.go
-173
View File
@@ -1,173 +0,0 @@
package main
import (
"context"
"errors"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/testbus"
)
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
// serving controllers over one store and one bus. The second waits while the first holds the lease; on
// a handover it takes it at a higher epoch, the record says which held what and how each ended; and the
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
// moment it lost it.
//
// MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus)
// aBusForTheLease is the test bus with the controller's lease bucket new.
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
t.Helper()
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
js, err := jetstream.New(conn)
if err != nil {
t.Fatal(err)
}
_ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket)
if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil {
t.Fatal(err)
}
kv, err := js.KeyValue(t.Context(), broker.LeaseBucket)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) })
return url, kv
}
func TestTwoControllersOneActs(t *testing.T) {
url, kv := aBusForTheLease(t)
inv := inventory.ForTest(t)
ctx := t.Context()
// Controller A takes the lease.
a := &actor{}
aCtx, stopA := context.WithCancel(ctx)
defer stopA()
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
if err != nil {
t.Fatal(err)
}
epochA, err := a.epoch(ctx)
if err != nil || epochA == 0 {
t.Fatalf("A holds no epoch: %d, %v", epochA, err)
}
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
b := &actor{}
bCtx, stopB := context.WithCancel(ctx)
defer stopB()
tookB := make(chan (<-chan struct{}), 1)
go func() {
lost, err := b.serveUnderTheLease(bCtx, inv, url)
if err != nil {
t.Errorf("B: %v", err)
close(tookB)
return
}
tookB <- lost
}()
select {
case <-tookB:
t.Fatal("B took the lease while A held it")
case <-time.After(3 * time.Second):
}
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
t.Fatalf("B, waiting, may act: %v", err)
}
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
stopA()
a.release()
var lostB <-chan struct{}
select {
case lostB = <-tookB:
case <-time.After(10 * time.Second):
t.Fatal("B did not take the lease A gave back")
}
select {
case <-lostA:
default:
t.Fatal("A, having given the lease back, is not told it no longer holds it")
}
epochB, err := b.epoch(ctx)
if err != nil || epochB <= epochA {
t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err)
}
if _, err := a.epoch(ctx); err == nil {
t.Fatal("A acts after giving the lease back")
}
ea, _, _ := inv.EpochOf(ctx, epochA)
eb, _, _ := inv.EpochOf(ctx, epochB)
if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance {
t.Fatalf("the record of the handover reads %+v then %+v", ea, eb)
}
// Something else writes the lease's key — a third controller on a clock that read it as expired:
// B's next renewal is refused, and B stops acting at once.
if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil {
t.Fatal(err)
}
select {
case <-lostB:
case <-time.After(2 * lease.RenewEvery):
t.Fatal("B was not told it lost the lease")
}
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
t.Fatalf("B acts after losing the lease: %v", err)
}
// Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition
// is not raised.
inv.ActsUnder(b.epoch)
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
t.Fatal(err)
}
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
State: inventory.PlanBuilding}
if err := inv.SavePlan(ctx, &saved); err == nil {
t.Fatal("B wrote a plan after losing the lease")
}
store := conditions.NewInMemory()
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
defer keeper.Close(context.Background())
if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x",
Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil {
t.Fatal("B raised a condition after losing the lease")
}
if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost {
t.Fatalf("B's epoch does not say it was lost: %+v", ended)
}
}
// A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease:
// it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can.
func TestAControllerTheBusRefusesTheLeaseServesUnleasedAndSaysSo(t *testing.T) {
a := &actor{serving: true, unleased: "the bus refused the lease's key"}
if epoch, err := a.epoch(context.Background()); err != nil || epoch != 0 {
t.Fatalf("an unleased controller answers %d, %v: it acts, claiming no epoch", epoch, err)
}
if st := a.standing(); st.Unleased == "" || st.Held {
t.Fatalf("its standing says %+v", st)
}
// One that neither holds nor is unleased — still waiting — acts on nothing.
waiting := &actor{serving: true}
if _, err := waiting.epoch(context.Background()); !errors.Is(err, lease.ErrNotHeld) {
t.Fatalf("a controller waiting for the lease may act: %v", err)
}
}
cmd/mesh-controller/main.go
+2 -78
View File
@@ -55,9 +55,6 @@ func run() error {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
// 45 §6), so the next controller takes it at once rather than after its age.
defer theLease.release()
switch args[0] {
case "build":
@@ -95,9 +92,6 @@ func run() error {
return collectionCommand(ctx, args[1:])
case "plans":
return plansCommand(ctx, args[1:])
case "delivery":
// The verbs the delivery's owner asks with (novox/hq ADR 0239).
return deliveryCommand(ctx, args[1:])
case "pin":
return pinCommand(ctx, args[1:], true)
case "unpin":
@@ -117,20 +111,8 @@ func run() error {
return brokerCommand(ctx, args[1:])
case "serve":
return serve(ctx)
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9).
case "facts":
return factsCommand(ctx, args[1:])
// The merge gate: every machine of the snapshot composed with a change (novox/hq to-be 45 §9).
case "merge-gate":
return mergeGateCommand(ctx, args[1:])
// A pull request's merge check run here exactly as the build seat runs it (novox/hq issue 286).
case "check-here":
return checkHereCommand(ctx, args[1:])
case "upgrade":
return upgradeCommand(ctx, args[1:])
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
case "bus":
return busCommand(ctx, args[1:])
case "declare":
return declare(ctx, args[1:])
case "overlay":
@@ -163,31 +145,6 @@ func run() error {
return seatCommand(ctx, args[1:])
case "status":
return statusCommand(ctx, args[1:])
// Acts done by hand, and why (novox/hq to-be 45 §7).
case "hand-act":
return handActCommand(ctx, args[1:])
case "hand-acts":
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
case "durations":
return durationsCommand(ctx, args[1:])
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
case "conditions":
return conditionsCommand(ctx, args[1:])
case "doctor":
return doctorCommand(ctx, args[1:])
// What the healers did, and their brake (novox/hq to-be 45 §7).
case "healers":
return healersCommand(ctx, args[1:])
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
// (novox/hq ADR 0230).
case "retire":
return retireCommand(ctx, args[1:])
case "cleanup":
return cleanupCommand(ctx, args[1:])
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
case "data":
return dataCommand(ctx, args[1:])
case "version":
fmt.Println(version)
return nil
@@ -234,16 +191,9 @@ func usage() {
upgrade <name> roll-out [--together] ...send it to the machines running it
upgrade <name> record ...record that they are behind, and send nothing
status [--json] what is wrong, what is quiet, and what is out of date
retire [--json] every provider waiting for a person to approve a retirement (ADR 0230)
retire approve <node> <module> --why <text> retire what it waits with: access off, data kept
retire reject <node> <module> --why <text> keep them active; a warning stays open
cleanup [list] [--json] every retired consumer per provider: age, size, why
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
seats [--json] every seat this mesh defines, what it delivers, and who holds it
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
board [--listen ADDR] the same three questions, as a page that holds nothing
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
assign <node> <module>... put modules on a node, judged together (ADR 0207)
@@ -275,34 +225,19 @@ func usage() {
kill <id> end a build where it runs; recorded failed, killed by hand
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
hand-act record <what> --why <text> --cause <word> [--condition <key>]
record an act done by hand outside the mesh (to-be 45 §7)
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
conditions [--scope S] [--severity S] [--machine M] [--json]
what is wrong now: every open condition, urgent first (to-be 45 §2)
conditions show <key> one condition whole, with its evidence
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
conditions history [--days N] [--key K] every raising, change and clearing lately
doctor [run|probes|signals] [--json]
the self-check: the last verdict, a run now, the probes, the signals' ages
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
durations [--kind K] [--days N] [--json]
apply, heartbeat, plan-tier and build durations, per machine or module
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
builder issue <name> a broker account for a build machine, scoped to build work,
delivered as the builder module's broker secret (module add it first)
licence add|list|use|key model access, under the name a person calls it
licence manager <name> <node> the node that holds a refreshable licence's refresh token
licence refresh <name> mint a new access token and seal it to every holder
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
pin <node> <provision> <from-node> <module>
which provider this one gets a provision from: the module, and its node
unpin <node> <provision> put that question back
plan <node> [--files|--json] what that node would run, and why
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
that need it; --why records it in the hand-act log
push [<node>] [--behind] send a node everything it should be, or only those that need it
version what this binary is
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
@@ -344,12 +279,6 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
// became of a build nobody was watching.
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
// recorded or registered.
if result.Check != nil || result.Checked != nil {
checked(ctx, result)
return nil
}
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
// unmerged branch was heard here like any build, registered, and its definition reached a machine
// before anyone had reviewed it.
@@ -361,9 +290,6 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
asked, _ := link.BuildAskedAt(result.ID)
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
// Phase 0). Said if lost; never a reason not to take the build in.
recordBuildDuration(ctx, b.inv, result, asked)
switch {
case err != nil && result.Failed != "":
fmt.Printf("%s: %v\n", result.ID, err)
@@ -390,7 +316,5 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
// A module registered may be one a machine is now behind: `status` is composed again.
statusFrom.nudge()
return nil
}
cmd/mesh-controller/merge_gate.go
-1290
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/merge_gate_test.go
-414
View File
@@ -1,414 +0,0 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
"github.com/novox/mesh-controller/internal/inventory"
)
// The merge gate (novox/hq to-be 45 §9): a change judged against every machine of the snapshot, by the
// incidents it is written from. Each case raises a mesh, takes its snapshot, and judges a pull request's
// tree against it in throwaway stores of its own.
// catalogueMesh is two machines and a catalogue repository: a resolver and an object store on the
// anchor, and on the laptop a network manager requiring the resolver, an album requiring the object
// store, and a login manager. Every module is registered from novox/mesh-catalog, as the mesh's are.
func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
manifests := map[string]string{
"objects": `{"module":"objects","version":"1",
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
"resolver": `{"module":"resolver","version":"1",
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`,
"networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`,
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
"lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"],
"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`,
}
for name, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
}
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"},
{"laptop", "album"}, {"laptop", "lemurs"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
f, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
return f, manifests
}
// aTree is a checkout of the catalogue repository holding these manifests.
func aTree(t *testing.T, manifests map[string]string) string {
t.Helper()
dir := t.TempDir()
for name, raw := range manifests {
at := filepath.Join(dir, "modules", name)
if err := os.MkdirAll(at, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(raw), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict {
t.Helper()
admin := os.Getenv("MESH_TEST_POSTGRES")
in := mergeCheckInput{facts: f, admin: admin, changed: changed}
if tree != "" {
in.repository, in.tree = "novox/mesh-catalog", tree
}
v, err := judgeChange(t.Context(), in)
if err != nil {
t.Fatal(err)
}
return v
}
func withEdit(manifests map[string]string, name, raw string) map[string]string {
out := map[string]string{}
for k, v := range manifests {
out[k] = v
}
if raw == "" {
delete(out, name)
} else {
out[name] = raw
}
return out
}
// The mesh as it is passes: every machine composes in the gate's store as the controller composed it.
func TestTheMeshAsItIsPassesTheGate(t *testing.T) {
f, manifests := catalogueMesh(t)
for _, m := range f.Machines {
if !m.Declaration.Composes {
t.Fatalf("the mesh itself does not compose: %+v", m.Declaration)
}
}
v := gateJudged(t, f, aTree(t, manifests))
if v.Verdict != "pass" {
t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report())
}
for _, m := range v.Machines {
if !m.Base.Composes || !m.Change.Composes {
t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change)
}
}
}
// **Issue 263**: a change makes the network manager require the object store's provision; on a machine
// whose name is six characters its identity is 26 against a bound of 20. Refused in the pull request,
// naming the module, and not on the anchor after it merged.
func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) {
f, manifests := catalogueMesh(t)
tree := aTree(t, withEdit(manifests, "networkmanager",
`{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`))
v := gateJudged(t, f, tree)
if v.Verdict != "fail" {
t.Fatalf("the overflow passed the gate:\n%s", v.Report())
}
report := v.Report()
if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") {
t.Errorf("the refusal does not name the module and the provision:\n%s", report)
}
}
// **Issue 236**: a login manager's service that omits its state passed the catalogue check and was
// refused whole by the node-engine on the first machine. A change to a module a machine runs, and a
// module nobody runs yet, are both refused before merge, naming the machine and the module.
func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) {
f, manifests := catalogueMesh(t)
broken := `{"module":"lemurs","version":"1","capabilities":["systemd"],
"resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}`
v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken)))
if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") {
t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report())
}
laptop := snapshot.Pseudonym("machine", "laptop")
if !strings.Contains(v.Report(), laptop) {
t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report())
}
// And as a new module, which no machine runs: tried on one that could.
newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`)
newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service")
v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer)))
if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") {
t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report())
}
}
// **Version skew**: a manifest the controller judging it cannot read — the one the mesh runs, for a
// catalogue change — fails here, not at registration after the merge.
func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) {
f, manifests := catalogueMesh(t)
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album",
`{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`)))
if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") {
t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report())
}
}
// A module a machine runs, removed from its source, fails until it is unassigned (ADR 0236).
func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) {
f, manifests := catalogueMesh(t)
v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", "")))
if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") {
t.Fatalf("removing a module a machine runs passed:\n%s", v.Report())
}
}
// **Issues 278 and 280**: a file in no held module's directory read as shared code, and a merge rebuilt the
// catalogue. A file is a module's only by being inside it — no build reads one outside — so it rebuilds
// nothing, and the gate says why; a merge that does rebuild widely is warned of.
func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) {
f, manifests := catalogueMesh(t)
was := wideRebuild
wideRebuild = 2
t.Cleanup(func() { wideRebuild = was })
for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} {
v := gateJudged(t, f, aTree(t, manifests), file)
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" {
t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict)
}
if !strings.Contains(v.Report(), "read by no module's build") {
t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report())
}
}
v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go")
if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" {
t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict)
}
// With the reference module's definition in the tree, its directory is a module, held or not.
withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`)
v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts")
if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 {
t.Fatalf("a file of a module nobody holds reads %+v", v.Width)
}
v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json")
if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" {
t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict)
}
}
// **A delivery group is judged as one future state** (novox/hq ADR 0239): a catalogue change that needs a
// provision only another repository's change adds fails alone and passes composed with it; and a group
// whose other head breaks what the first needs fails, naming it.
func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) {
f, manifests := catalogueMeshWithAnApp(t)
needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`)
catTree := aTree(t, needsTheApp)
alone := gateJudged(t, f, catTree, "modules/album/module.json")
if alone.Verdict != "fail" {
t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report())
}
app := t.TempDir()
if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1",
"provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil {
t.Fatal(err)
}
in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog",
tree: catTree, changed: []string{"modules/album/module.json"},
group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}}
together, err := judgeChange(t.Context(), in)
if err != nil {
t.Fatal(err)
}
if together.Verdict == "fail" {
t.Fatalf("the group composed together fails:\n%s", together.Report())
}
if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" {
t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules)
}
}
// catalogueMeshWithAnApp is catalogueMesh with an application built from a repository of its own, at its
// root, on the anchor.
func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
manifests := map[string]string{
"objects": `{"module":"objects","version":"1",
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`,
"album": `{"module":"album","version":"1","requires":["s3-bucket"]}`,
}
for name, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
}
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
f, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
return f, manifests
}
// busMesh is aMesh with a module on the bus on the laptop, its account issued as `module issue` issues
// one: minted, and its credential held as the module's own secret named broker.
func busMesh(t *testing.T) snapshot.Facts {
t.Helper()
open := aMesh(t)
ctx := t.Context()
m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1",
"own-secrets":{"broker":"/var/lib/speaker/broker"}}`))
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "laptop", "speaker"); err != nil {
t.Fatal(err)
}
user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username()
password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule,
Node: "laptop", Module: "speaker"})
if err != nil {
t.Fatal(err)
}
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker",
`{"user":"`+user+`","password":"`+password+`"}`); err != nil {
t.Fatal(err)
}
f, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
for _, mc := range f.Machines {
if !mc.Declaration.Composes {
t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration)
}
}
return f
}
// **Issue 285**: every machine running a module on the bus failed to compose in the gate's store, with
// the change and without — the store held the module's credential and no account for it, which
// composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the
// mesh issued is raised with its credential, so the machine composes in the gate as on the mesh.
func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) {
f := busMesh(t)
v := gateJudged(t, f, "")
if v.Verdict != "pass" {
t.Fatalf("the mesh as it is does not pass:\n%s", v.Report())
}
for _, m := range v.Machines {
if !m.Base.Composes {
t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems)
}
}
if !strings.Contains(v.Summary, "2 of 2 compose") {
t.Errorf("the summary does not say every machine composes: %s", v.Summary)
}
}
// **Issue 285**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged
// against a machine that is not the mesh's — broken against broken, which passes whatever the change does.
// That is an error, never a pass.
func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) {
f := busMesh(t)
for i := range f.Machines {
// A fact the snapshot does not carry: the module's credential, gone from the laptop's.
var kept []snapshot.Accepted
for _, a := range f.Machines[i].Accepted {
if a.Name != "broker" {
kept = append(kept, a)
}
}
f.Machines[i].Accepted = kept
}
v := gateJudged(t, f, "")
if v.Verdict != "error" {
t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report())
}
if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") {
t.Errorf("the error does not name what could not be raised: %v", v.Errors)
}
}
// A path the snapshot withheld is given a path of its own where an access or a place needs one: a bare
// "withheld" is no absolute path, and a machine whose setting composes on the mesh would not in the gate.
func TestAWithheldPathIsStoodInForByAPath(t *testing.T) {
got := standInPaths(map[string]any{
"accesses": map[string]any{"races": "withheld", "films": "/media/films", "shows": "/withheld"},
"places": map[string]any{"config": map[string]any{"path": "withheld", "owner": "1000:1000"}},
"puid": 1000,
})
accesses := got["accesses"].(map[string]any)
if accesses["races"] == accesses["shows"] || !strings.HasPrefix(accesses["races"].(string), "/") ||
!strings.HasPrefix(accesses["shows"].(string), "/") || accesses["films"] != "/media/films" {
t.Errorf("accesses: %v", accesses)
}
place := got["places"].(map[string]any)["config"].(map[string]any)
if !strings.HasPrefix(place["path"].(string), "/") || place["owner"] != "1000:1000" || got["puid"] != 1000 {
t.Errorf("places: %v", got)
}
}
// **Issue 286**: a check run by hand clones beside a change what the seat clones — one rule, read by the
// controller's ask and by the facts — and finds the repository it checks from its origin.
func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) {
for dir, refs := range map[string]map[string]string{
"mesh-catalog": {"mesh-catalog": "main"},
"mesh-host": {"mesh-host": "c0ffee"},
"mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"},
} {
got := besideRefs(dir, "c0ffee")
for d, ref := range refs {
if got[d] != ref {
t.Errorf("beside %s, %s is cloned at %q, not %q", dir, d, got[d], ref)
}
}
}
for owner, want := range map[string][3]string{
"ssh://git@git.example:222/novox/mesh-host.git": {"novox", "mesh-host", "ssh://git@git.example:222/novox"},
"git@git.example:novox/mesh-tools.git": {"novox", "mesh-tools", "git@git.example:novox"},
"http://git.example/novox/hq": {"novox", "hq", "http://git.example/novox"},
} {
o, r, p := ownerRepoOf(owner)
if [3]string{o, r, p} != want {
t.Errorf("%s reads as %s %s %s", owner, o, r, p)
}
}
}
cmd/mesh-controller/mesh_for_test.go
-20
View File
@@ -1,14 +1,11 @@
package main
import (
"context"
"crypto/ecdh"
"crypto/rand"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/novox/mesh-controller/internal/conditions"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
@@ -40,9 +37,6 @@ func aMesh(t *testing.T) *stores {
t.Fatal(err)
}
t.Cleanup(open.Close)
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
withConditionsInMemory(t)
for _, m := range provided {
if err := open.inventory.Provide(t.Context(), m); err != nil {
t.Fatal(err)
@@ -112,17 +106,3 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
}
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
t.Helper()
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
before := conditionsFrom
conditionsFrom = k
t.Cleanup(func() {
conditionsFrom = before
k.Close(context.Background())
})
return k, store
}
cmd/mesh-controller/missed_merges.go
-189
View File
@@ -1,189 +0,0 @@
package main
import (
"context"
"fmt"
"sync"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
//
// The forge's poll announces every merge on the events stream, and the controller acts on what its
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
// server moved the consumer past it — a fault of consumers with several filters in the server the
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
// built from that repository stayed behind and were built by hand.
//
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
// would still move something was never acted on — it is said, and acted on now.
const (
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
mergeGrace = 10 * time.Minute
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
mergeLookBack = 24 * time.Hour
// mergeCatchUpEvery is how often the stream is read back.
mergeCatchUpEvery = 5 * time.Minute
)
// merges is what reads back the forge's announcements; the link server, or a test's list.
type merges interface {
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
}
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
f := following{open}
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
return nil, nil, err
}
read, err := open.inventory.ReadRepositories(ctx)
return entries, read, err
}
failing := ""
tick := time.NewTicker(mergeCatchUpEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
}
watchedMerges.begin()
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
fmt.Printf(format+"\n", args...)
})
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
// says that instead, and leaves what the last one found standing.
watchedMerges.end(time.Now(), err)
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
why := ""
if err != nil {
why = err.Error()
}
if why != failing {
if why != "" {
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
} else {
fmt.Println("merges the bus may not have handed over are looked for again")
}
failing = why
}
}
}
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
// move something is said and acted on, oldest first.
//
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
// because acting on an earlier missed merge of the same repository may have moved what a later one
// would have.
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
if err != nil {
return err
}
entries, read, err := catalogued(ctx)
if err != nil {
return err
}
for _, a := range all {
if now.Sub(a.At) < mergeGrace {
continue
}
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
continue
}
if entries, read, err = catalogued(ctx); err != nil {
return err
}
moves := wouldMove(a.SourceMoved, entries, read)
if len(moves) == 0 {
continue
}
var names []string
for _, e := range moves {
names = append(names, e.Manifest.Module)
}
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
At: a.At, Modules: names})
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
isAre(len(names)))
if err := act(ctx, a.SourceMoved); err != nil {
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
a.Owner, a.Repo, a.Commit, err)
continue
}
if entries, read, err = catalogued(ctx); err != nil {
return err
}
}
return nil
}
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
type missedMerge struct {
Owner, Repo, Base, Commit string
// At is when the bus took the announcement.
At time.Time
Modules []string
}
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
// the fault. The next pass, finding it acted on, clears it.
type mergeWatch struct {
mu sync.Mutex
passed time.Time
err error
finding []missedMerge
missed []missedMerge
}
var watchedMerges = &mergeWatch{}
func (w *mergeWatch) begin() {
w.mu.Lock()
defer w.mu.Unlock()
w.finding = nil
}
func (w *mergeWatch) found(m missedMerge) {
w.mu.Lock()
defer w.mu.Unlock()
w.finding = append(w.finding, m)
}
func (w *mergeWatch) end(at time.Time, err error) {
w.mu.Lock()
defer w.mu.Unlock()
w.passed, w.err = at, err
if err == nil {
w.missed = w.finding
}
}
// last is when the last pass ended, what the last pass that read found, and what the last pass
// could not read.
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
w.mu.Lock()
defer w.mu.Unlock()
return w.passed, append([]missedMerge(nil), w.missed...), w.err
}
cmd/mesh-controller/missed_merges_test.go
-180
View File
@@ -1,180 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// announcedList is the events stream's merges as a test gives them.
type announcedList []link.AnnouncedMerge
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
var out []link.AnnouncedMerge
for _, m := range a {
if !m.At.Before(since) {
out = append(out, m)
}
}
return out, nil
}
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
type aCatalogue struct {
entries []inventory.Entry
acted []string
fail error
}
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
return append([]inventory.Entry(nil), c.entries...), nil, nil
}
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
return func(_ context.Context, m link.SourceMoved) error {
if c.fail != nil {
return c.fail
}
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
for _, moved := range wouldMove(m, c.entries, nil) {
for i := range c.entries {
if c.entries[i].Manifest.Module == moved.Manifest.Module {
c.entries[i].Source.Head = m.Commit
c.entries[i].Source.Seen = now()
}
}
}
return nil
}
}
func at(s string) time.Time {
t, err := time.Parse(time.RFC3339, s)
if err != nil {
panic(err)
}
return t
}
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
return link.AnnouncedMerge{
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
MergedAt: mergedAt, Paths: paths,
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
At: at(onTheBus),
}
}
func built(module, repo, path, commit, seen string) inventory.Entry {
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
return e
}
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
// events stream; the bus never handed it to the controller, which acted on the merges around it and
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
// move something — so it is said and acted on, once, and only after the controller's own consumer
// has had its time with it.
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
cat := &aCatalogue{entries: []inventory.Entry{
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
// Acted on when it was announced: looked at after it was merged.
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
}}
stream := announcedList{
// Nothing the mesh holds is built from the records repository.
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
// Acted on: its module was looked at since.
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
// Never handed over.
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
"node-tools/internal/console/console.go"),
}
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
clock := at("2026-10-05T22:50:00Z")
now := func() time.Time { return clock }
pass := func() {
t.Helper()
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
}
pass()
if len(cat.acted) != 0 {
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
}
clock = at("2026-10-05T22:58:00Z")
pass()
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
}
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
t.Fatalf("the missed merge was not said as one: %q", said)
}
clock = at("2026-10-05T23:03:00Z")
pass()
if len(cat.acted) != 1 || len(said) != 1 {
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
}
}
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
// that could not be acted on is said and tried again on the next pass.
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
cat := &aCatalogue{
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
fail: errors.New("the store is restarting"),
}
stream := announcedList{
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
"modules/plex/module.json", "modules/plex/x.ts"),
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
}
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
clock := at("2026-10-05T22:00:00Z")
now := func() time.Time { return clock }
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
t.Fatalf("a failed catch-up was not said: %q", said)
}
cat.fail = nil
clock = at("2026-10-05T22:05:00Z")
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
}
}
// A merge older than the look-back is left to the operator: a controller that did not run this
// missed it, and acting on it days later would be a surprise rebuild.
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
clock := at("2026-10-05T22:00:00Z")
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
func(string, ...any) {}); err != nil {
t.Fatal(err)
}
if len(cat.acted) != 0 {
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
}
}
cmd/mesh-controller/modules.go
+6 -22
View File
@@ -40,12 +40,7 @@ func providedModules() []catalogue.Manifest {
// and neither could be swapped for anything, which is the test of whether a thing is a
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
// to live, and now a module says where it wants it — `facts` in its own manifest.
//
// **And the bundle that required it is gone** (novox/hq ADR 0226): `networking` named this
// module's requirement and nothing else, so every machine carried two modules for one
// network. A machine is assigned the private network itself; what a release stops shipping
// is retired at the next start (stores.go, Inventory.RetireUnshipped).
overlay.Manifest(),
overlay.Manifest(), overlay.DomainManifest(),
} {
var m catalogue.Manifest
b, _ := json.Marshal(raw)
@@ -64,19 +59,8 @@ func moduleCommand(ctx context.Context, args []string) error {
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
if args[0] == "check" {
set := flag.NewFlagSet("module check", flag.ContinueOnError)
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
"judge each module's identity on a machine name this many characters long")
given, err := parseAround(set, args[1:])
if err != nil {
return err
}
if *longest < 1 {
return errors.New("--longest-machine-name is a length, at least 1")
}
var paths []string
for _, a := range given {
for _, a := range args[1:] {
if info, err := os.Stat(a); err == nil && info.IsDir() {
under, err := manifestsUnder(a)
if err != nil {
@@ -87,7 +71,7 @@ func moduleCommand(ctx context.Context, args []string) error {
}
paths = append(paths, a)
}
return moduleCheckFor(paths, *longest, os.Stdout)
return moduleCheck(paths, os.Stdout)
}
open, err := openStores(ctx)
if err != nil {
@@ -510,8 +494,8 @@ const theBrokerSeat = "mesh-broker"
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
// has adopted it) does the hub stand in, which is where the foundation is by convention.
//
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the private network
// — not "has an address", which is true of every placed machine and says nothing about
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the networking
// module — not "has an address", which is true of every placed machine and says nothing about
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
@@ -677,7 +661,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
// durable subscription nobody reads.
if consumer, needed := broker.ConsumerFor(broker.Principal{
Kind: broker.KindModule, Node: node, Module: m.Module,
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
}); needed {
if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
cmd/mesh-controller/nodes.go
+3 -35
View File
@@ -6,7 +6,6 @@ import (
"errors"
"flag"
"fmt"
"github.com/novox/mesh-controller/internal/conditions"
"strings"
"time"
@@ -388,7 +387,7 @@ func brokerCommand(ctx context.Context, args []string) error {
return busAccounts(ctx, args[1:])
}
if len(args) > 0 && args[0] == "consumer-reset" {
return consumerReset(ctx, args[1:])
return consumerReset(args[1:])
}
if len(args) == 0 || args[0] != "show" {
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
@@ -415,21 +414,10 @@ func brokerCommand(ctx context.Context, args []string) error {
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
func consumerReset(ctx context.Context, args []string) error {
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
// A repair by hand, which says why (novox/hq to-be 45 §7).
why := addHandActFlags(set)
args, err := parseAround(set, args)
if err != nil {
return err
}
func consumerReset(args []string) error {
if len(args) != 2 {
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
}
if err := why.require("broker consumer-reset"); err != nil {
return err
}
why.record(ctx, "broker consumer-reset", args)
address, err := broker.BusAddress()
if err != nil {
return err
@@ -517,26 +505,6 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
fmt.Printf(" public domain %s\n", domain)
}
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
// is something not working now and they are a description. Unreadable is said, not passed over.
open, err := openConditions(ctx)
if err != nil {
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
}
var here []conditions.Condition
for _, c := range open {
if concerns(c, name) {
here = append(here, c)
}
}
if len(here) > 0 {
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
for _, line := range conditionLines(here, time.Now()) {
fmt.Printf(" %s\n", line)
}
}
held, err := inv.Profile(ctx, name)
if err != nil {
return err
cmd/mesh-controller/order_test.go
+4 -71
View File
@@ -1,7 +1,6 @@
package main
import (
"encoding/json"
"reflect"
"strings"
"testing"
@@ -144,7 +143,7 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
}{
{"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"},
{"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"},
{"a file at the root no build reads (issue 280)", merge([]string{"tsconfig.json"}, false), ""},
{"a file they share", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
@@ -153,9 +152,9 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), ""},
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""},
{"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"},
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), "gitea,keycloak"},
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), "gitea,keycloak"},
{"the root's files still", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
} {
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
@@ -244,69 +243,3 @@ func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) {
t.Error("a module following another branch was matched")
}
}
// novox/hq issues 278 and 280: a merge touching the code of a module the mesh does not hold — the
// catalogue's reference module, whose manifest it left alone — read as shared and rebuilt every module
// built from the repository (103 of them on 2026-10-06, 88 byte-identical); so did a merge-check.sh added at
// the root. No build reads a file outside its module's directory, so neither rebuilds anything, said by the
// announcer or not.
func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) {
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
gitea := fromRepo("gitea", repo, "modules/gitea")
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
known := []inventory.Entry{gitea, keycloak}
candidates := []inventory.Entry{gitea, keycloak}
merge := func(paths, modules []string, said bool) link.SourceMoved {
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Paths: paths,
ModuleDirs: modules, ModuleDirsSaid: said}
}
named := func(entries []inventory.Entry) string {
var names []string
for _, e := range entries {
names = append(names, e.Manifest.Module)
}
return strings.Join(names, ",")
}
showcase := []string{"modules/showcase/index.ts"}
for _, c := range []struct {
what string
m link.SourceMoved
want string
}{
{"a module held by none, said", merge(showcase, []string{"modules/showcase"}, true), ""},
{"beside a held one's change", merge(append([]string{"modules/gitea/index.ts"}, showcase...),
[]string{"modules/gitea", "modules/showcase"}, true), "gitea"},
{"deeper inside it", merge([]string{"modules/showcase/daemon/index.ts"}, []string{"modules/showcase"}, true), ""},
{"a directory holding no manifest, read by no build", merge([]string{"modules/lib/x.go"}, nil, true), ""},
{"one of two files in no module", merge([]string{"modules/showcase/index.ts", "modules/lib/x.go"},
[]string{"modules/showcase"}, true), ""},
{"the root is never a module directory", merge([]string{"tsconfig.json"}, []string{"", "/", "."}, true), ""},
{"not said: still no build reads it", merge(showcase, []string{"modules/showcase"}, false), ""},
{"an old announcer saying nothing", merge(showcase, nil, false), ""},
{"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""},
} {
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
}
}
}
// What the announcer says reaches the controller as it is sent: the two fields, by their names on the
// wire, and an older announcement without them reads as not said.
func TestTheAnnouncerSaysWhichDirectoriesAreModules(t *testing.T) {
var m link.SourceMoved
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc",`+
`"paths":["modules/showcase/index.ts"],"module_dirs":["modules/showcase"],"module_dirs_said":true}`), &m); err != nil {
t.Fatal(err)
}
if !m.ModuleDirsSaid || !reflect.DeepEqual(m.ModuleDirs, []string{"modules/showcase"}) {
t.Fatalf("the announcer's word was lost: %+v", m)
}
var old link.SourceMoved
if err := json.Unmarshal([]byte(`{"owner":"novox","repo":"mesh-catalog","merge_commit_sha":"abc","paths":["x"]}`), &old); err != nil {
t.Fatal(err)
}
if old.ModuleDirsSaid || old.ModuleDirs != nil {
t.Fatalf("an older announcement says nothing about module directories: %+v", old)
}
}
cmd/mesh-controller/plan.go
+39 -169
View File
@@ -90,12 +90,6 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
if err != nil {
return catalogue.Resolution{}, nil, err
}
// Where each of its consumers of a provision that keeps data was last sent (novox/hq ADR 0232):
// a resolution that would answer one from anywhere else keeps it there, and says so.
world.Bound, err = inv.BindingsFor(ctx, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
@@ -378,19 +372,13 @@ func declarationFor(ctx context.Context, open *stores, node string,
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
// machine "waiting" means — the read needs no number to be right about that.
type Choosing int
type Choosing bool
const (
// Reading is every question: what is assigned is used, and nothing is created.
Reading Choosing = iota
// Allocating is the send path: what is not assigned yet is assigned now and kept.
Allocating
// Foreseeing is Reading, asked ahead of a send (the self-check's D1, novox/hq issue 275): nothing
// is created, and an own secret the next send WOULD make is composed with a stand-in and named
// (sendable.foreseen) rather than failing the composition — while one the send would be refused
// (a bus credential nobody issued, a given secret under an old key) is refused here as it would
// be there. Never sent: the stand-in is not a sealed value.
Foreseeing
Allocating Choosing = true
// Reading is every question: what is assigned is used, and nothing is created.
Reading Choosing = false
)
func declarationWith(ctx context.Context, open *stores, node string,
@@ -411,8 +399,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
}
out := sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
unbound: with.Unbound, foreseen: composed.Foreseen}
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
// 0221). Read only on the send path: a question about what would be sent records nothing.
if choosing == Allocating {
@@ -432,35 +419,10 @@ func declarationWith(ctx context.Context, open *stores, node string,
names = append(names, m.Module)
}
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
out.Bindings = boundToData(plan, composed.LeftOut)
}
return out, nil
}
// boundToData is every binding of this machine's consumers to a provision that keeps their data
// (novox/hq ADR 0232), as a send records it. Not a consumer left out of the declaration: the machine
// is not told anything new about it, so nothing about where it is bound has been sent.
func boundToData(plan catalogue.Resolution, leftOut map[string]string) []inventory.Binding {
var out []inventory.Binding
seen := map[[2]string]bool{}
for _, n := range plan.Needs {
if !n.KeepsData || n.ByRecord || n.Module == "" {
continue
}
if _, left := leftOut[n.For]; left {
continue
}
key := [2]string{n.For, n.Name}
if seen[key] {
continue
}
seen[key] = true
out = append(out, inventory.Binding{Machine: plan.Node, Consumer: n.For, Provision: n.Name,
Provider: catalogue.Chosen{Node: n.From, Module: n.Module}})
}
return out
}
// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
// issue 259): the module's current build for each module in it, and for a module left out of it
// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
@@ -502,39 +464,6 @@ func reportLeftOut(node string, declared sendable) {
"what the machine holds for it is kept and its containers are untouched. %s\n",
node, m, declared.leftOutWhy[m])
}
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
// 0225): the machine is sent everything else, and the consumer is named.
for _, o := range declared.withheld {
fmt.Printf("%s: %s\n", node, o)
}
// And whom it no longer serves because they are bound elsewhere (novox/hq issue 274).
for _, u := range declared.unbound {
fmt.Printf("%s: %s\n", node, u)
}
}
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
//
// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is
// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted
// by `module issue` and sealed by it; a push that made a random one would deliver a file the process
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
if name != "broker" {
return nil
}
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return err
} else if !minted {
return fmt.Errorf(
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
"`module issue %s --node %s`, then push again",
module, node, user, module, node)
}
return nil
}
// renderingFor is everything a node's declaration is composed with, and the node's record.
@@ -542,7 +471,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
plan catalogue.Resolution, settings catalogue.SettingsBy,
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
inv := open.inventory
grants, withheld, unbound, err := grantsFor(ctx, open, node)
grants, err := grantsFor(ctx, open, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
@@ -554,7 +483,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
// consumer is told are all derived from it.
// What this machine was already given, for a composition that may not allocate.
already := map[string]map[int]int{}
if choosing != Allocating {
if choosing == Reading {
held, err := inv.PortsFor(ctx, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
@@ -640,7 +569,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
// And each module's own secrets — a superuser password, an administrator, an account. Made
// per node, so a module running on three machines has three.
needed := map[string]map[string]string{}
foreseen := map[string]map[string]bool{}
for _, m := range plan.Modules {
for name := range m.OwnSecrets {
// Minted on the send path and only read on every other. Making one is an insert, and
@@ -648,29 +576,27 @@ func renderingFor(ctx context.Context, open *stores, node string,
var sealed string
var err error
if choosing == Allocating {
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
// **The broker credential is never invented here** (novox/hq issue 203). Every other
// own secret is the mesh's to make — a password nobody else knows — but this one
// is an account on the bus, minted by `module issue` and sealed by it; a push that
// made a random one would deliver a file the process cannot read and report the
// machine applied. Refused by name, with the verb.
if name == "broker" {
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
} else if !minted {
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
"`module issue %s --node %s`, then push again",
m.Module, node, user, m.Module, node)
}
}
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
} else {
var held bool
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
if err == nil && !held && choosing == Foreseeing {
// Asked ahead of the send: what the send would do about it, by the send's own
// rules. Made by it — a stand-in, named; refused by it — refused here, the same
// words (novox/hq issue 275).
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if foreseen[m.Module] == nil {
foreseen[m.Module] = map[string]bool{}
}
foreseen[m.Module][name] = true
continue
}
if err == nil && !held {
// Never issued, so this machine cannot be running it. Left out rather than
// invented: an empty string here would compose a declaration that differs
@@ -781,13 +707,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
return catalogue.Rendering{}, inventory.Node{}, err
}
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
// lists every holder of the mesh's resolver.
replicas, err := replicatedHolders(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -862,13 +781,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
}
return catalogue.Rendering{
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines, Zones: zones, Holders: replicas,
Machines: machines, Zones: zones,
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
BusUsers: busUsers, Withheld: withheld, Unbound: unbound,
BusUsers: busUsers,
}, record, nil
}
@@ -1004,45 +923,28 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
// The mirror of what a consumer is given, and the half that makes the credential real: a password
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
// the mesh hands over something it cannot itself use.
//
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
//
// **A provider is granted exactly the consumers whose own resolution binds them to it** (novox/hq
// issue 274). The pair credentials on record say only whom this node was ever asked by: after a
// consumer of a provision that keeps its data was moved and pinned back (issue 273, ADR 0232), the
// credential from the provider it left was still on record, so that provider went on being asked for
// five databases nobody used and never retired them. A credential whose consumer is bound elsewhere
// is withdrawn here like one nobody asks for — the provider retires it and keeps its data (ADR 0230)
// — and returned beside the grants, for plan and push to say. It stays on record: it is the key to
// the login the provider keeps until `cleanup delete`, and to that data should a person pin it back.
func grantsFor(ctx context.Context, open *stores, node string) (
[]catalogue.Grant, []catalogue.Overflow, []catalogue.Unbound, error) {
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
inv := open.inventory
issued, err := inv.SecretsFrom(ctx, node)
if err != nil {
return nil, nil, nil, err
return nil, err
}
// Where each consumer is, so a provider that must reach back to one does not have to know how
// the mesh names machines.
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, nil, nil, err
return nil, err
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return nil, nil, nil, err
return nil, err
}
// What each consumer actually asked for, taken from that machine's own resolution rather than
// from a record beside it. A provider told to create a password and not what to create it for
// can do nothing with it, and the name a consumer wants is the consumer's to say.
out := make([]catalogue.Grant, 0, len(issued))
var withheld []catalogue.Overflow
var unbound []catalogue.Unbound
for _, s := range issued {
plan, settings, err := planFor(ctx, open, s.Consumer)
switch {
@@ -1055,11 +957,11 @@ func grantsFor(ctx context.Context, open *stores, node string) (
// The mesh could not be asked what they wanted, which is not the same as their wanting
// nothing — and withholding a grant on that reading takes a consumer's access away
// (novox/hq 04-ISSUES/152).
return nil, nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
}
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
if err != nil {
return nil, nil, nil, err
return nil, err
}
// A port in there is the consumer's software port until this. The consumer is on another
// machine, so the assignment that moved it is that machine's — fetched here rather than
@@ -1067,7 +969,7 @@ func grantsFor(ctx context.Context, open *stores, node string) (
// this case (novox/hq 04-ISSUES/038, the cross-node half).
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
if err != nil {
return nil, nil, nil, err
return nil, err
}
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
from := s.ConsumerModule
@@ -1077,20 +979,10 @@ func grantsFor(ctx context.Context, open *stores, node string) (
// working for ever after its consumer went away.
from = ""
}
if bound := plan.BindsFrom(s.Name, s.ConsumerModule, s.Local); from != "" && !slices.Contains(bound, node) {
// It still asks, and not of this node: its resolution — the same one read above, so an
// unreadable one is the error above and never an empty answer here (issue 152) — binds
// this credential to another provider, or under this local name to none. Withdrawn
// exactly as a credential nobody asks for, and said.
from = ""
unbound = append(unbound, catalogue.Unbound{Provision: s.Name, Provider: node,
Consumer: s.Consumer, Module: s.ConsumerModule, Local: s.Local, BoundTo: bound})
}
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
// this provision, as the consumer's resolution states it from the provider's offer (ADR
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
// provider silently shortened — and rather than this whole machine refused for it.
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
// remedy a short slug rather than a login a provider silently shortened.
slug := ""
for _, mm := range plan.Modules {
if mm.Module == s.ConsumerModule {
@@ -1099,32 +991,15 @@ func grantsFor(ctx context.Context, open *stores, node string) (
}
}
if from != "" {
bound := boundOfGrant(plan, s, node)
source := catalogue.IdentitySource(slug, s.ConsumerModule)
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
Consumer: s.Consumer, Module: s.ConsumerModule,
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
continue
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
return nil, err
}
}
out = append(out, catalogue.Grant{
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
}
return out, withheld, unbound, nil
}
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
// than to none: what the provider keeps of it is not known here.
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
for _, n := range consumer.Needs {
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
return n.Identity
}
}
return catalogue.DefaultIdentityBound
return out, nil
}
// listensLines is what a person is told about what this module would open, and why — the same
@@ -1200,11 +1075,6 @@ func planCommand(ctx context.Context, args []string) error {
left := plan.LeftOut(settings, record.Adopted)
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
}
// And which of its modules no provider will grant, because the identity overflows the bound of
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
for _, o := range plan.Overflowing() {
fmt.Printf("%s: %s\n", args[0], o)
}
// And a setting that reaches nothing — refused where it is stored, and said here for one
// stored before its definition moved from under it.
for _, m := range plan.Modules {
cmd/mesh-controller/plan_retry.go
+11 -116
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"os"
"slices"
"sort"
"strings"
"time"
@@ -179,15 +178,6 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
p.Tier, p.ID, p.Note)
}
// **A build that failed its gate is not sent again** (novox/hq ADR 0236): it was put back on its first
// machine, and retrying would judge the build the mesh put back, or send the failed one by hand.
for _, m := range stopped {
if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed && !noVerdictOnItsBuild(g) {
return fmt.Errorf("%s failed its gate on %s (%s) and was put back: a build that failed its gate is not "+
"sent again — a newer merge, or `rebuild %s`, makes a new build, judged at the gate again",
m, strings.Join(g.Machines, ", "), g.Why, m)
}
}
// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
// sent — a newer build, and sending this one again would put the older build back on its machines.
for _, m := range stopped {
@@ -270,9 +260,6 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
if err := retryRefusal(p, plans); err != nil {
return "", err
}
if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
return retryTierWhole(ctx, open, &p, again)
}
if len(failedIn(p)) == 0 {
return retryRollouts(ctx, open, &p)
}
@@ -293,7 +280,7 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
}
}
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
if err := inv.SavePlan(ctx, &p); err != nil {
if err := inv.SavePlan(ctx, p); err != nil {
return "", err
}
if p.State != inventory.PlanBuilding {
@@ -336,7 +323,7 @@ func joinAPlan(ctx context.Context, open *stores, module string) (bool, string,
askModule(ctx, &p, module, byName)
s := p.Modules[module]
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
if err := inv.SavePlan(ctx, &p); err != nil {
if err := inv.SavePlan(ctx, p); err != nil {
return false, "", err
}
if s.State != "asked" {
@@ -392,115 +379,23 @@ func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.P
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
// Every machine once, for all the modules stopped there (novox/hq issue 281).
stopped := stoppedRollouts(*p)
var machines []string
for _, m := range stopped {
for _, n := range p.Modules[m].First {
if !slices.Contains(machines, n) {
machines = append(machines, n)
}
}
}
sort.Strings(machines)
sent, err := sendRollout(ctx, open, machines)
if err != nil {
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
strings.Join(stopped, ", "), strings.Join(machines, ", "), p.ID, err)
}
now := time.Now().UTC()
var said []string
for _, m := range stopped {
for _, m := range stoppedRollouts(*p) {
s := p.Modules[m]
var again []string
for _, n := range sent {
if slices.Contains(s.First, n) {
again = append(again, n)
}
sent, err := sendRollout(ctx, open, s.First)
if err != nil {
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
m, strings.Join(s.First, ", "), p.ID, err)
}
s.First, s.FirstAt, s.Why = again, &now, ""
said = append(said, m+" to "+strings.Join(again, ", "))
now := time.Now().UTC()
s.First, s.FirstAt, s.Why = sent, &now, ""
said = append(said, m+" to "+strings.Join(sent, ", "))
}
p.State = inventory.PlanRolling
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
if err := open.inventory.SavePlan(ctx, p); err != nil {
if err := open.inventory.SavePlan(ctx, *p); err != nil {
return "", err
}
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
}
// noVerdictOnItsBuild says a failed gate said nothing about the module's build (novox/hq issue 281): its
// send changed nothing of the module there — the machine already ran that build, carried there by an
// earlier send of the same tier, or one identical to it. Such a module was blamed for its machine.
func noVerdictOnItsBuild(g *inventory.PlanGate) bool {
return g.Rollback == gateUnchanged || (g.From != "" && sameCommit(g.From, g.To))
}
// unjudgedAtGate is the modules of a plan's current tier stopped at a gate that was no verdict on their
// build, sorted.
func unjudgedAtGate(p inventory.Plan) []string {
if p.Tier >= len(p.Tiers) {
return nil
}
var out []string
for _, m := range p.Tiers[p.Tier] {
if s := p.Modules[m]; s != nil && s.Gate != nil && s.Gate.Verdict == inventory.GateFailed && noVerdictOnItsBuild(s.Gate) {
out = append(out, m)
}
}
sort.Strings(out)
return out
}
// retryTierWhole retries a plan stopped at a gate that judged no build of the module it stopped on
// (issue 281): that module is asked again under a new id — its old build may be marked failed, and a new
// verdict is what takes the gate's condition away — and every module of the tier sent first and never
// passed is sent again, the tier whole, one send per machine, judged again. What passed stays passed.
func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again []string) (string, error) {
inv := open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
return "", err
}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
var resent []string
for _, m := range p.Tiers[p.Tier] {
s := p.Modules[m]
if s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(again, m) ||
(s.Gate != nil && s.Gate.Verdict == inventory.GatePassed) {
continue
}
sendAgain(s)
resent = append(resent, m)
}
var asked []string
for _, m := range again {
sendAgain(p.Modules[m])
askModule(ctx, p, m, byName)
if s := p.Modules[m]; s.State == "asked" {
asked = append(asked, m+" as "+s.Build)
}
}
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
p.State = inventory.PlanBuilding
p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
}
if err := inv.SavePlan(ctx, p); err != nil {
return "", err
}
if p.State != inventory.PlanBuilding {
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
}
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; %s sent again with the tier, one send per machine, "+
"judged again", p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", "), orNone(strings.Join(resent, ", "))), nil
}
// sendAgain forgets a module's first send, so its plan sends it again.
func sendAgain(s *inventory.PlanModule) {
s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
}
cmd/mesh-controller/plans_one_at_a_time_test.go
+1 -1
View File
@@ -18,7 +18,7 @@ func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
if err := open.inventory.SavePlan(ctx, plan); err != nil {
t.Fatal(err)
}
cmd/mesh-controller/probes.go
-1245
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/push.go
+59 -293
View File
@@ -8,11 +8,9 @@ import (
"errors"
"flag"
"fmt"
"github.com/novox/mesh-controller/internal/conditions"
"io"
"log"
"os"
"slices"
"sort"
"strings"
"time"
@@ -64,7 +62,7 @@ func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.En
return link.ConnectNats(js, enroller, listener), nil
}
func serve(ctx context.Context) (err error) {
func serve(ctx context.Context) error {
// The one process whose log is read over time, so the one that says each change to a node's
// unmet seat dependencies once (novox/hq ADR 0207).
logUnheldChanges = true
@@ -105,47 +103,9 @@ func serve(ctx context.Context) (err error) {
// being live is refused, because a mesh half on each is one where a declaration goes out on one
// and the report comes back on the other, and every component logs success while it happens.
// **The lease, before anything that acts** (novox/hq to-be 45 §6): asserting the bus's objects is the
// controller's to do, and so is everything after. A controller starting while another holds it waits
// here, said; one that loses it stops: every act's gate closes at once, and ctx ends so the process
// exits and is started again as a candidate.
busAddress, err := broker.BusAddress()
if err != nil {
return err
}
lost, err := theLease.serveUnderTheLease(ctx, inv, busAddress)
if err != nil {
return err
}
// Given back before the store closes, so the epoch is recorded as given back rather than found
// expired by the next holder.
defer theLease.release()
ctx, stopActing := context.WithCancel(ctx)
defer stopActing()
go func() {
select {
case <-lost:
stopActing()
case <-ctx.Done():
}
}()
defer func() {
select {
case <-lost:
if err == nil {
err = errors.New("the controller lease was lost; this controller stopped acting and exits, to " +
"be started again as a candidate")
}
default:
}
}()
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
OnNATS: true}
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
// something new is one thing that moves it, so the listener nudges it.
statusFrom = newStatusSummary(composeStatus(open))
server, err := connectLink(ctx, inv, work, nudgingListener{Enrolment: work, summary: statusFrom, open: open})
server, err := connectLink(ctx, inv, work, work)
if err != nil {
return err
}
@@ -161,33 +121,17 @@ func serve(ctx context.Context) (err error) {
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
// machines to report moves when they have, and a plan left by a replaced controller resumes.
go planTicker(ctx, open)
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
go forgettingOldDurations(ctx, inv)
// And what the catalogue decided a build meant. The builder's own result is already handled
// above; this is the other half — the control plane is the only one of the three that knows
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
if err := server.Follows(following{open}); err != nil {
return err
}
// And the merges the bus announced and never handed over, read back on a timer and acted on late
// rather than never (novox/hq issue 266).
go catchingUpOnMerges(ctx, open, server)
// And a catalogue that has just started, asking for what it missed. The same type answers
// both: what a build meant and what the builds were are two questions about one record.
if err := server.Answers(following{open}); err != nil {
return err
}
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
// 0224): a provider's journal must not be the only place that says so.
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
return err
}
// And what they say about consumers the mesh stopped asking for: one waiting for a person is an
// urgent condition, and an act asked of a provider some other way is recorded by hand (ADR 0230).
if err := server.KeepsRetirements(retirements{keeper: func() *conditions.Keeper { return conditionsFrom },
record: recordHandActOnTheBus}); err != nil {
return err
}
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
// from the store's row, so what the seat declares is what is answered.
@@ -206,44 +150,6 @@ func serve(ctx context.Context) (err error) {
if !isNATS {
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
}
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
handActConn = bus.Conn
// And says when it replaced a value given by hand (novox/hq ADR 0228).
givenEvents = bus
// And a pull request's merge check, asked when the forge announces its head and said when judged
// (novox/hq to-be 45 §9).
checkEvents = bus
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
inventory.PlanSaved = func(p inventory.Plan) { sayPlanMoved(ctx, bus, p) }
if err := server.Checks(following{open}); err != nil {
return err
}
// Composed now and kept current, before the verb that answers from it is served.
go statusFrom.keep(ctx)
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this
// controller holds the lease, read by the build seat from the artifact store.
go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion)
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
// to-be 45 §6).
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
// And every call is kept on the bus, so a restart of this process keeps what came of each
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
// answering no calls at all would be worse than answering them without the record.
said := log.New(os.Stdout, "", log.LstdFlags)
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
} else if err := link.Calls.Durably(ctx, keeper, instance, said); err != nil {
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
}
// What is wrong, kept and said (novox/hq to-be 45 §2): the condition store, the watchdogs of the
// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
// is said and the controller serves on: status then says the conditions cannot be read, and is
// not well — louder than not serving at all, and the push that repairs the bus still runs.
if stopWatching := watchTheMesh(ctx, open, server, bus); stopWatching != nil {
defer stopWatching()
}
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
if err != nil {
return err
@@ -313,12 +219,7 @@ func declare(ctx context.Context, args []string) error {
// is still what the machine was last told, and status must not read it as current for the one
// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
// the mesh did not compose it, so a push that does not name this machine treats it as held.
// The epoch it carried, if a person wrote one in, is what the machine heard.
var carried struct {
Epoch uint64 `json:"epoch"`
}
_ = json.Unmarshal(raw, &carried)
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
return err
}
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
@@ -349,9 +250,6 @@ func pushCommand(ctx context.Context, args []string) error {
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
wait := set.Duration("wait", 0,
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
// recorded when given at a shell — see handacts.go for why a shell is not refused.
why := addHandActFlags(set)
positionals, err := parseAround(set, args)
if err != nil {
return err
@@ -366,11 +264,6 @@ func pushCommand(ctx context.Context, args []string) error {
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
"other asks which machines need one")
}
recorded := append([]string(nil), args...)
if *behind {
recorded = append(recorded, "--behind")
}
why.record(ctx, "push", recorded)
open, err := openStores(ctx)
if err != nil {
return err
@@ -424,7 +317,7 @@ func pushCommand(ctx context.Context, args []string) error {
if len(needsOne) == 0 {
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
// must never look the same.
fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent")
fmt.Println("every machine is doing what it was told")
return nil
}
}
@@ -465,61 +358,6 @@ func pushCommand(ctx context.Context, args []string) error {
}
asked = append(asked, n.Name)
}
// **A push that named no machine says so, first.** Through the console a machine the caller
// meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as
// `push --behind`, and every machine behind was pushed by someone who thought they had pushed one.
// Its whole-mesh reach is the first line of the answer, with the machines it is about to send.
if len(args) == 0 {
which := "every machine"
if *behind {
which = "every machine that is behind"
}
fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n",
which, len(asked), strings.Join(asked, ", "))
}
// **A whole-mesh push sends no build a gate has not seen** (novox/hq ADR 0236): a machine where one
// waits is left, named, for the release plan — `push <node>` still sends one machine by a person's word.
if len(args) == 0 {
f, err := readMoveFacts(ctx, inv)
if err != nil {
return err
}
var kept []string
for _, n := range asked {
moves, err := machineMoves(ctx, open, f, n, false)
if err != nil {
return err
}
if len(moves) > 0 {
fmt.Printf(" %s is left: %d build(s) wait there for a gate, which a release plan sends one machine "+
"at a time (`upgrade backlog` lists them; `push %s` sends it by name)\n", n, len(moves), n)
continue
}
kept = append(kept, n)
}
asked = kept
}
// **The bus is replaced only as a planned step** (novox/hq ADR 0236, to-be 45 §8): a machine whose bus
// would move is not sent by a push — named, it is refused; otherwise it is left and said.
busKept, err := busHeld(ctx, inv, asked)
if err != nil {
return err
}
if len(busKept) > 0 {
if len(args) == 1 {
return fmt.Errorf("%s. Nothing was sent", busKept[args[0]])
}
var kept []string
for _, n := range asked {
if why, h := busKept[n]; h {
fmt.Printf(" %s is left: %s\n", n, why)
continue
}
kept = append(kept, n)
}
asked = kept
}
// **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's
// user list, and a module's new grants are refused by the bus until that list says them. Among
@@ -567,7 +405,6 @@ func pushCommand(ctx context.Context, args []string) error {
// healthy modules beside it are still resolved and sent. Reported so it is not silently
// dropped — the remedy is to move it, and until then the rest of the node converges.
reportUnhostable(node, plan)
reportKept(held, plan)
unheld[node] = plan.Unheld
// The private network is in here with everything else. It used to be composed separately
// and prepended, which meant every machine with an address was on it and no machine could
@@ -588,11 +425,7 @@ func pushCommand(ctx context.Context, args []string) error {
return err
}
release()
told := make([]string, 0, len(sending))
for _, r := range sending {
told = append(told, r.node)
}
fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", "))
fmt.Printf("\n%d node(s) told\n", len(sending))
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
@@ -691,7 +524,7 @@ type readyNode struct {
//
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
// across two machines that must agree — and dropped here, where it never did.
func composeEach(names []string, allot func(node string) (order, error),
func composeEach(names []string, allot func(node string) (int64, error),
compose func(node string) (sendable, error)) ([]readyNode, []string) {
var sending []readyNode
@@ -705,7 +538,7 @@ func composeEach(names []string, allot func(node string) (order, error),
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
// two machines was undone two seconds later by exactly that. Taken here, before the first
// read, what was composed earlier is numbered lower whatever order the sends happen in.
numbered, err := allot(name)
seq, err := allot(name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
@@ -715,7 +548,7 @@ func composeEach(names []string, allot func(node string) (order, error),
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
declared.Sequence = seq
if len(declared.Resources) == 0 {
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
// nothing may have HELD something before — the broker opening a placement gave it,
@@ -883,13 +716,6 @@ type overTheBus struct {
}
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
// when they cannot be, and never what holds the send back (assertOnSend).
if bus, ok := b.server.Bus().(link.OverNATS); ok {
js := broker.OnConn(bus.Conn)
js.Note = func(format string, args ...any) { fmt.Printf(b.indent+" "+format+"\n", args...) }
_ = assertOnSend(ctx, b.open.inventory, js, b.indent)
}
return issueMemberships(ctx, b.open, b.server, sending)
}
@@ -899,21 +725,10 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
}
// After it is away, not before. A digest recorded for something that failed to send would make
// the machine look current for a declaration it never received.
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
if err != nil {
return "", err
}
if len(s.declared.Bindings) > 0 {
// And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232):
// what the next resolution keeps it at. On the same outliving context as the send's record.
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
err := b.open.inventory.RecordBindings(kept, s.node, s.declared.Bindings)
cancel()
if err != nil {
return "", fmt.Errorf("%s was sent its declaration, and where its consumers are bound to their "+
"data could not be recorded: %w", s.node, err)
}
}
if s.declared.BusUsers != "" {
// And the user list it carried, so the next send reads whether it must go first from the
// list alone (novox/hq issue 249). On the same outliving context as the send's record.
@@ -1035,21 +850,6 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
// the machines it sent waits for that one too.
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
inv := open.inventory
// **No send replaces the bus but its planned step** (novox/hq ADR 0236). A plan's send to the bus's
// machine for some other module carried the bus's new build with it on 2026-10-06, and the bus
// restarted under every machine with nobody having asked. Refused whole — the send cannot leave the
// bus behind and carry the rest (ADR 0221 option 2) — and said with the remedy; the plan tries again.
if !busStepSending(ctx) {
held, err := busHeld(ctx, inv, names)
if err != nil {
return nil, err
}
for _, n := range names {
if why, h := held[n]; h {
return nil, fmt.Errorf("%w: %s", errBusWaits, why)
}
}
}
ident, err := openIdentity(ctx)
if err != nil {
return nil, err
@@ -1064,16 +864,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
if err != nil {
return nil, err
}
added := ""
if behind && !slices.Contains(names, holder) {
added = holder
}
names = brokerFirst(names, holder, behind)
// **No build moves on a machine without a gate** (novox/hq ADR 0236): a send outside its scope that
// would carry one is refused, said with what waits; the bus's machine added for its user list is left.
if names, err = ungatedIn(ctx, open, names, added); err != nil {
return nil, err
}
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
// converge, which flips the node and then sends it — is not made to wait on itself.
@@ -1087,7 +878,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
var refusals []string
for _, name := range names {
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
numbered, err := allot(ctx, inv, name)
seq, err := allot(ctx, inv, name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
@@ -1098,13 +889,12 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
continue
}
reportUnhostable(name, plan)
reportKept(ctx, plan)
declared, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
declared.Sequence = seq
reportLeftOut(name, declared)
sending = append(sending, readyNode{name, declared})
}
@@ -1223,20 +1013,6 @@ func digestOf(body []byte) string {
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
func wouldSend(ctx context.Context, open *stores,
nodes []inventory.Node) (map[string]string, error) {
return wouldSendFrom(ctx, open, nodes, nil)
}
// planned is one machine's plan as planFor answered it, for a caller that already asked.
type planned struct {
plan catalogue.Resolution
settings catalogue.SettingsBy
}
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
// to-be 45 Phase 0). A machine absent from plans is worked out here.
func wouldSendFrom(ctx context.Context, open *stores,
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
gens, err := generators(ctx, open)
if err != nil {
@@ -1244,12 +1020,9 @@ func wouldSendFrom(ctx context.Context, open *stores,
}
out := map[string]string{}
for _, n := range nodes {
known, have := plans[n.Name]
plan, settings := known.plan, known.settings
if !have {
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
continue
}
plan, settings, err := planFor(ctx, open, n.Name)
if err != nil {
continue
}
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
if err != nil {
@@ -1261,11 +1034,6 @@ func wouldSendFrom(ctx context.Context, open *stores,
if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil {
return nil, err
}
// And the epoch it was last sent under, for the same reason: a new holder of the lease is not a
// change of the machine (novox/hq to-be 45 §6).
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
return nil, err
}
body, err := declared.Body()
if err != nil {
return nil, err
@@ -1313,22 +1081,35 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
}
}
// The mesh's own streams and consumers, the seats' work queues and their workers, and how every
// machine hears its declaration — one derivation, which the self-check reads as well (D6, D7).
names, err := assertBusObjects(ctx, inv, js)
nodes, err := inv.Nodes(ctx)
if err != nil {
return err
}
names := make([]string, 0, len(nodes))
for _, n := range nodes {
names = append(names, n.Name)
}
if err := broker.Raise(js, names); err != nil {
return err
}
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
// after something started asking for builds flushes the backlog instead of having lost it.
// With the seats' holders, so each role's work queue gets the consumer its holder takes
// work from. Passed as nil until the first live raise, which left the build machine bound to a
// consumer nothing had created (2026-09-28).
holders, err := seatHolders(ctx, inv)
if err != nil {
return err
}
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
return err
}
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
// whether it was cancelled the moment it took it.
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
return err
}
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
// by hand, kept where a restart of this process does not take them.
if err := js.EnsureControllerBuckets(); err != nil {
return err
}
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
// registration, so a module reading one may watch it before its owner runs anywhere. One that
// nothing declares any more is said and kept — what it holds is data.
@@ -1344,11 +1125,25 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
"removing it is a person's act\n", strings.Join(undeclared, ", "))
}
// And how every module hears what it consumes: asserted with the rest above, counted here.
hearing, err := moduleConsumerCount(ctx, inv)
// And how every module hears what it consumes. Derived from the same records the user list is
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
// Done on every raise, not only when a credential is issued: every module moved onto this bus
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
records, err := inv.BusRecords(ctx)
if err != nil {
return err
}
users, err := broker.Users(records)
if err != nil {
return err
}
hearing := 0
for _, c := range broker.ConsumersOf(users) {
if err := js.EnsureConsumer(c.Consumer); err != nil {
return fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
}
hearing++
}
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
return nil
@@ -1391,46 +1186,17 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
// allotting is allot over one inventory, in the shape composeEach takes.
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (order, error) {
return func(node string) (order, error) { return allot(ctx, inv, node) }
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (int64, error) {
return func(node string) (int64, error) { return allot(ctx, inv, node) }
}
// epochForActs is the lease's gate as a composition asks it; a variable so a test can act under an epoch
// without a bus.
var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }
// order is what a declaration carries of its writer's order (link/order.go): its sequence, and the
// epoch of the lease it is composed under — zero for a machine that has not said it reads one.
type order struct {
sequence int64
epoch uint64
}
// allot takes the next sequence for a machine — the number its next declaration carries — under the
// lease: a process that may not act takes none, and composes nothing (novox/hq to-be 45 §6).
func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, error) {
epoch, err := epochForActs(ctx)
if err != nil {
return order{}, fmt.Errorf("nothing was composed for %s: %w", node, err)
}
// allot takes the next sequence for a machine — the number its next declaration carries.
func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, error) {
record, err := inv.NodeByName(ctx, node)
if err != nil {
return order{}, err
return 0, err
}
if epoch > 0 {
reads, err := inv.ReadsEpoch(ctx, record.ID)
if err != nil {
return order{}, err
}
if !reads {
epoch = 0
}
}
seq, err := inv.NextSequence(ctx, record.ID)
if err != nil {
return order{}, err
}
return order{sequence: seq, epoch: epoch}, nil
return inv.NextSequence(ctx, record.ID)
}
// recordSent writes down what a machine was just sent, and returns the digest.
@@ -1445,7 +1211,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
// what tells a machine held back by a policy or a plan from one a push left behind.
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
builds map[string]string, epoch uint64) (string, error) {
builds map[string]string) (string, error) {
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
record, err := inv.NodeByName(kept, node)
@@ -1453,7 +1219,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
return "", err
}
digest := digestOf(body)
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
return "", err
}
return digest, nil
cmd/mesh-controller/push_test.go
+2 -2
View File
@@ -76,7 +76,7 @@ func TestASkippedMachineIsStillAnError(t *testing.T) {
}
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
func numbered() func(string) (order, error) {
func numbered() func(string) (int64, error) {
var n int64
return func(string) (order, error) { n++; return order{sequence: n}, nil }
return func(string) (int64, error) { n++; return n, nil }
}
cmd/mesh-controller/queue_test.go
+15 -15
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
"testing"
@@ -15,15 +16,15 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
//
// docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js
// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
// MESH_TEST_NATS=nats://127.0.0.1:14294 \
// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused' (each test on a bus of its own)
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused'
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
func asksRecorded(t *testing.T) *[]string {
@@ -63,7 +64,7 @@ func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
Note: "a failed to build in tier 0",
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
Why: link.KilledByHand}}}
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
if err := open.inventory.SavePlan(ctx, failed); err != nil {
t.Fatal(err)
}
@@ -155,7 +156,7 @@ func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
Note: "a failed to build in tier 0",
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
if err := open.inventory.SavePlan(ctx, failed); err != nil {
t.Fatal(err)
}
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
@@ -356,7 +357,10 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
// pointed at it, and a function that asks the seat one build.
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
t.Helper()
url := testbus.URL(t)
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
t.Setenv(broker.NATSVar, url)
js, err := broker.Dial(url)
if err != nil {
@@ -429,7 +433,7 @@ func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
if err := open.inventory.SavePlan(ctx, plan); err != nil {
t.Fatal(err)
}
@@ -621,31 +625,27 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
}
t.Cleanup(func() { sendRollout = was })
// a records: a person's choice, since the default rolls out (novox/hq ADR 0236).
if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil {
t.Fatal(err)
}
long := time.Now().UTC().Add(-2 * time.Hour)
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
if err := open.inventory.SavePlan(ctx, &stopped); err != nil {
if err := open.inventory.SavePlan(ctx, stopped); err != nil {
t.Fatal(err)
}
// A newer plan holding a refuses it: sending the older build would put it back.
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
if err := open.inventory.SavePlan(ctx, newer); err != nil {
t.Fatal(err)
}
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
t.Fatalf("retried under a newer plan: %v", err)
}
newer.State = inventory.PlanSuperseded
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
if err := open.inventory.SavePlan(ctx, newer); err != nil {
t.Fatal(err)
}
@@ -683,7 +683,7 @@ func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
if err := open.inventory.SavePlan(ctx, plan); err != nil {
t.Fatal(err)
}
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
cmd/mesh-controller/readable.go
-31
View File
@@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"sort"
"time"
@@ -79,28 +78,6 @@ type meshStatus struct {
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
// dependency is met. Reported, not refused, until the switch.
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
// HandActsUnread says why when it could not be read, rather than reading as none.
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
HandActsUnread string `json:"handActsUnread,omitempty"`
// HealsThisWeek is what the healers did in the last seven days (novox/hq to-be 45 §7); HealsUnread
// why it could not be read.
HealsThisWeek *healsCount `json:"healsThisWeek,omitempty"`
HealsUnread string `json:"healsUnread,omitempty"`
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
Conditions []conditions.Condition `json:"conditions"`
ConditionsUnread string `json:"conditionsUnread,omitempty"`
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
// provider says so. A document without it called the mesh well while the identity provider
// refused every consumer for a day (04-ISSUES/179).
Failing []conditions.Condition `json:"failing,omitempty"`
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
}
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
@@ -233,14 +210,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
}
}
out.Unheld = asked.unheld
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
if out.Conditions == nil {
out.Conditions = []conditions.Condition{}
}
out.Failing = providerStandings(asked.conditions)
out.Overflowing = asked.overflowing
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
cmd/mesh-controller/release.go
-667
View File
@@ -1,667 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// No build reaches a machine without a gate (novox/hq ADR 0236).
//
// **A send carries the machine's whole declaration.** A plan sending one module to its first machine
// sends every other module whose build moved there too; a cascade, a healer's resend and a whole-mesh
// push do the same. Under the old default (`record`) builds were registered and sent nowhere, so on the
// day the default became `roll` every machine was behind on dozens of builds no gate had seen — and the
// next send of anything would have restarted all of them at once, on every machine.
//
// So **a build moves on a machine only through a send that judges it there**, or a person's:
//
// - a gated send — a plan's first machine, a release plan's machine — carries every move waiting on that
// machine, and its gate judges each of them; a pass is each build's verdict, a failure puts back what
// failed;
// - a module a send exists for may move anywhere it is sent: a policy of *together*, a rollback, a build
// whose gate already passed;
// - a send naming a machine by a person (`push <node>`, the bus step) carries what it carries;
// - every other send — a plan's "rest", a cascade, a healer's, the bus's user list carried — is refused,
// or leaves the machine, while a move there waits for a gate. A rebuild that made the same artifacts
// from the same manifest is no move.
//
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
// sendScope is what a send may carry that no gate has seen.
type sendScope struct {
// judged are the machines whose every move this send's gate judges.
judged map[string]bool
// modules are those a send exists for, which may move wherever it goes.
modules map[string]bool
// person is a person's act: it carries what it carries.
person bool
}
type sendScopeKey struct{}
func withScope(ctx context.Context, s sendScope) context.Context {
return context.WithValue(ctx, sendScopeKey{}, s)
}
func scopeOf(ctx context.Context) sendScope {
s, _ := ctx.Value(sendScopeKey{}).(sendScope)
return s
}
// errUngated is a send refused because it would carry a build no gate has seen.
var errUngated = errors.New("a build waits there for its gate")
// errWalkedElsewhere is a gated send refused because a plan that has started walks a move there.
var errWalkedElsewhere = errors.New("a plan already walking a build there sends it")
// moveFacts is what tells a move from a rebuild, and a gated build from one no gate has seen.
type moveFacts struct {
current map[string]inventory.CurrentBuild
fps map[string]map[string]string
// srcs is, per module, per commit, its build's source fingerprint (novox/hq issue 280).
srcs map[string]map[string]string
passed map[string]map[string]bool
plans []inventory.Plan
}
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
var f moveFacts
var err error
if f.current, err = inv.CurrentBuilds(ctx); err != nil {
return f, err
}
if f.fps, err = inv.Fingerprints(ctx); err != nil {
return f, err
}
if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
return f, err
}
if f.passed, err = inv.PassedCommits(ctx); err != nil {
return f, err
}
f.plans, err = inv.OpenPlans(ctx)
return f, err
}
// identical is whether two builds of a module put the same thing on a machine: the same commit,
// builds made from the same source (novox/hq issue 280) — the module's tree, the contexts it read, its
// bases and toolchains, whatever digests an image rebuild made of them — or builds that made the same
// artifacts from the same manifest.
func (f moveFacts) identical(module, a, b string) bool {
if a == b || sameCommit(a, b) {
return true
}
if sa := f.srcs[module][a]; sa != "" && sa == f.srcs[module][b] {
return true
}
fa := f.fps[module][a]
return fa != "" && fa == f.fps[module][b]
}
// gated is whether a build of a module from this commit — or one identical to it — passed a gate.
func (f moveFacts) gated(module, commit string) bool {
for c := range f.passed[module] {
if f.identical(module, c, commit) {
return true
}
}
return false
}
// unchangedOnEvery is whether a plan's build of a module was made from the source of the build every
// machine running it was last sent (novox/hq issue 280): no move on any of them. False when no machine
// runs it, when what one was sent is not known, or when the build stands for itself.
func unchangedOnEvery(ctx context.Context, inv *inventory.Inventory, module, build string, running []string) (bool, error) {
if build == "" || len(running) == 0 {
return false, nil
}
same, err := inv.SameSourceCommits(ctx, module, build)
if err != nil || len(same) == 0 {
return false, err
}
for _, n := range running {
sent, known, err := inv.SentBuilds(ctx, n)
if err != nil {
return false, err
}
was, carried := sent[module]
if !known || !carried || !inRun(same, was) {
return false, nil
}
}
return true, nil
}
// inRun is whether a commit is one of a run's, however either is abbreviated.
func inRun(run map[string]bool, commit string) bool {
for c := range run {
if sameCommit(c, commit) {
return true
}
}
return false
}
// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out
// (a recorded one is a person's push), that the machine was sent before, moving to a build not identical
// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names
// none: the cascade holds it whole (ADR 0221).
//
// With all, a move to a build that passed a gate elsewhere counts too: what a gated send carries.
func (f moveFacts) moves(node string, modules []string, sent map[string]string, known, all bool) []inventory.CarriedMove {
if !known {
return nil
}
var out []inventory.CarriedMove
for _, m := range modules {
now := f.current[m]
was, carried := sent[m]
if !now.RollOut || !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) {
continue
}
out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit})
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// walkedBy is the open plan that has started walking a module's build — sent it to a first machine,
// not yet passed — other than to this machine; empty when none does.
func (f moveFacts) walkedBy(module, node string) string {
for _, p := range f.plans {
s, holds := p.Modules[module]
if !p.Open() || !holds || s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(s.First, node) {
continue
}
if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed {
continue
}
return p.ID
}
return ""
}
// machineMoves is the moves no gate has seen on one machine, read from what it would be sent now.
var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
plan, _, err := planFor(ctx, open, node)
if err != nil {
return nil, nil // it cannot be worked out: the send says why
}
modules := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
modules = append(modules, m.Module)
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the
// machines named, and why; the machine holding the bus, added only for its user list, is left instead.
func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) {
scope := scopeOf(ctx)
if scope.person {
return names, nil
}
f, err := readMoveFacts(ctx, open.inventory)
if err != nil {
return nil, err
}
var kept, refused []string
for _, n := range names {
moves, err := machineMoves(ctx, open, f, n, false)
if err != nil {
return nil, err
}
var waiting []string
for _, mv := range moves {
if !scope.judged[n] && !scope.modules[mv.Module] {
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
}
}
switch {
case len(waiting) == 0:
kept = append(kept, n)
case n == addedHolder:
fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s wait there for a gate "+
"— the bus may refuse what was newly granted until it is\n", n, strings.Join(waiting, ", "))
default:
refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", ")))
}
}
if len(refused) > 0 {
return nil, fmt.Errorf("%w: %s — a release plan sends them, one machine at a time, each judged (novox/hq ADR "+
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
}
return kept, nil
}
// gatedSend sends one machine everything waiting there, under a gate that judges it all: what moved is
// answered, with the build each moved to, for the gate to judge and to put back. Refused while a plan that
// has started walks one of those builds elsewhere: that plan sends it here once its gate passed.
//
// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in
// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per
// module was the same declaration sent again and again, each one setting aside the one before.
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) {
inv := open.inventory
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, nil, err
}
moves, err := machineMoves(ctx, open, f, node, true)
if err != nil {
return nil, nil, err
}
own := func(module string) bool {
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
}
for _, mv := range moves {
if own(mv.Module) {
continue
}
if id := f.walkedBy(mv.Module, node); id != "" {
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
mv.Module, short(mv.To), node, id)
}
}
for _, o := range owns {
i := slices.IndexFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == o.Module })
switch {
case i < 0:
moves = append(moves, o)
case moves[i].Build == "":
moves[i].Build = o.Build
}
}
if len(moves) == 0 && len(owns) == 0 {
return nil, nil, nil
}
for i := range moves {
if moves[i].Build == "" {
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
return nil, nil, err
}
}
}
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
if err != nil {
return nil, nil, err
}
return moves, sent, nil
}
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
seen := map[string]bool{}
for _, c := range g.Carried {
if c.Module == except || c.Build == "" || seen[c.Build] {
continue
}
seen[c.Build] = true
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why,
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err)
}
}
}
// failCarried puts back every build the gate carried that it found wanting, on the machines that were
// sent it, once each.
func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
var notes []string
if p.Note != "" {
notes = append(notes, p.Note)
}
done := map[string]bool{except: true}
for _, c := range g.Carried {
if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) {
continue
}
done[c.Module] = true
machines, err := sentTheBuild(ctx, open, c.Module, c.To)
if err != nil || len(machines) == 0 {
machines = []string{c.Node}
}
state := &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To}
// A module of the plan's tier sent in the same send (issue 281) keeps its own record of it.
if s := p.Modules[c.Module]; s != nil && except != "" && s.GatedBy == except && s.Build == c.Build {
state = s
}
p.Note = ""
gateFailed(ctx, open, p, c.Module, state, machines, g.Why)
notes = append(notes, p.Note)
}
p.State = inventory.PlanFailed
p.Note = strings.Join(notes, "; ")
}
// sentTheBuild is every machine running a module that was last sent this build of it.
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
running, err := open.inventory.Running(ctx, module)
if err != nil {
return nil, err
}
var out []string
for _, n := range running {
sent, known, err := open.inventory.SentBuilds(ctx, n)
if err != nil {
return nil, err
}
if known && sameCommit(sent[module], commit) {
out = append(out, n)
}
}
return out, nil
}
// releaseRepository is what a release plan says it is for, where a merge's says its repository.
const releaseRepository = "the builds waiting for a gate"
// releaseHeard is the machines a release plan may send: heard within their heartbeat's bound. A machine
// away is left, not judged against a bound it cannot meet. A variable so a test says who is heard.
var releaseHeard = func(ctx context.Context, open *stores) (map[string]bool, error) {
if d := doctorFrom; d != nil && d.watchdogs != nil {
return heardMachines(d), nil
}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return nil, err
}
out := map[string]bool{}
for _, r := range reports {
if r.At != nil && time.Since(*r.At) < 15*time.Minute {
out[r.Node] = true
}
}
return out, nil
}
// backlogNow is what the newest look found waiting, for `upgrade backlog` and the gate's probe.
var backlogNow struct {
held string
waiting map[string][]inventory.CarriedMove
}
// waitingMoves is every machine's moves no gate has seen and no started plan walks.
func waitingMoves(ctx context.Context, open *stores, all bool) (map[string][]inventory.CarriedMove, error) {
f, err := readMoveFacts(ctx, open.inventory)
if err != nil {
return nil, err
}
nodes, err := open.inventory.Nodes(ctx)
if err != nil {
return nil, err
}
out := map[string][]inventory.CarriedMove{}
for _, n := range nodes {
moves, err := machineMoves(ctx, open, f, n.Name, all)
if err != nil {
return nil, err
}
for _, mv := range moves {
if f.walkedBy(mv.Module, n.Name) == "" {
out[n.Name] = append(out[n.Name], mv)
}
}
}
return out, nil
}
// releaseBacklog opens a release plan when builds wait for a gate and none is open; not after a release
// plan failed, until a person releases one (by). Called with the plans held.
func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Plan, error) {
inv := open.inventory
plans, err := inv.OpenPlans(ctx)
if err != nil {
return nil, err
}
for _, p := range plans {
if p.Release != nil {
if by != "" {
return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID)
}
return nil, nil
}
}
waiting, err := waitingMoves(ctx, open, false)
if err != nil {
return nil, err
}
backlogNow.waiting, backlogNow.held = waiting, ""
if len(waiting) == 0 {
return nil, nil
}
// Opened by what no gate has seen; it walks every machine where anything of the release waits — a
// build that passed on the first machine still goes to the next one by this plan, judged there too.
if waiting, err = waitingMoves(ctx, open, true); err != nil {
return nil, err
}
if by == "" {
recent, err := inv.RecentPlans(ctx, 50)
if err != nil {
return nil, err
}
for _, p := range recent {
if p.Release == nil {
continue
}
if p.State == inventory.PlanFailed {
backlogNow.held = fmt.Sprintf("%s failed (%s); what waits is released again by a person", p.ID, p.Note)
return nil, nil
}
break
}
}
heard, err := releaseHeard(ctx, open)
if err != nil {
return nil, err
}
controllers, err := inv.Running(ctx, catalogue.ControllerSeatName)
if err != nil {
return nil, err
}
var order, last []string
modules := map[string]bool{}
for node, moves := range waiting {
if !heard[node] {
continue
}
for _, mv := range moves {
modules[mv.Module] = true
}
if slices.Contains(controllers, node) {
last = append(last, node)
} else {
order = append(order, node)
}
}
if len(order)+len(last) == 0 {
return nil, nil
}
sort.Strings(order)
sort.Strings(last)
order = append(order, last...)
names := make([]string, 0, len(modules))
for m := range modules {
names = append(names, m)
}
sort.Strings(names)
now := time.Now().UTC()
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
Release: &inventory.PlanRelease{Order: order, By: by},
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
strings.Join(order, ", "))}
if err := inv.SavePlan(ctx, &p); err != nil {
return nil, err
}
fmt.Printf("%s: %s\n", p.ID, p.Note)
return &p, nil
}
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
// gate, or the machine being judged judged once more; the plan done when every machine is.
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
r := p.Release
now := time.Now().UTC()
if r.Gate == nil {
heard, err := releaseHeard(ctx, open)
if err != nil {
return false, err
}
for r.Next < len(r.Order) {
node := r.Order[r.Next]
if !heard[node] {
r.Skipped = append(r.Skipped, node)
r.Next++
continue
}
moves, sent, err := gatedSend(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) {
note := fmt.Sprintf("waiting before %s: %v", node, err)
changed := p.Note != note
p.Note = note
return changed, nil
}
if err != nil {
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
}
if len(moves) == 0 {
r.Done = append(r.Done, node)
r.Next++
continue
}
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves}
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
if len(r.Skipped) > 0 {
p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
}
return true, nil
}
g := r.Gate
verdict, err := judgeMoves(ctx, open, g, nil, now)
if err != nil {
return false, err
}
switch verdict {
case "":
note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g))
changed := p.Note != note
p.Note = note
return changed, nil
case inventory.GatePassed:
passCarried(ctx, open, p, g, "")
r.Done = append(r.Done, firstOf(g.Machines))
r.Next++
r.Gate = nil
return true, nil
}
p.Note = ""
batched, back := batchingRollbacks(ctx)
failCarried(batched, open, p, g, "")
sendRollbacks(ctx, open, p, back)
p.Note = fmt.Sprintf("failed its gate on %s: %s — %s", strings.Join(g.Machines, ", "), g.Why, p.Note)
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
// backlogObservation is what the gate's probe says of a release held after a failure.
func backlogObservation() []conditions.Observation {
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
return nil
}
n := 0
var machines []string
for node, moves := range backlogNow.waiting {
n += len(moves)
machines = append(machines, node)
}
sort.Strings(machines)
return []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held",
Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%d build move(s) on %s wait for a gate and are not released: %s — `upgrade backlog` lists "+
"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}}
}
// backlogCommand is `upgrade backlog`, read-only, and `upgrade release-backlog --why`.
func backlogCommand(ctx context.Context, sub string, args []string) error {
set := flag.NewFlagSet("upgrade "+sub, flag.ContinueOnError)
why := addHandActFlags(set)
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("upgrade backlog | upgrade release-backlog --why <text>")
}
if sub == "release-backlog" {
if err := why.require("upgrade release-backlog"); err != nil {
return err
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if sub == "release-backlog" {
release, err := open.inventory.HoldPlans(ctx, true)
if err != nil {
return err
}
defer release()
why.record(ctx, "upgrade release-backlog", nil)
p, err := releaseBacklog(ctx, open, link.Caller())
if err != nil {
return err
}
if p == nil {
fmt.Println("nothing waits for a gate on any machine heard from: nothing to release")
return nil
}
fmt.Printf("%s releases it; `plans %s` says where it is\n", p.ID, p.ID)
return nil
}
waiting, err := waitingMoves(ctx, open, false)
if err != nil {
return err
}
if len(waiting) == 0 {
fmt.Println("no build waits for a gate on any machine")
return nil
}
nodes := make([]string, 0, len(waiting))
for n := range waiting {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
fmt.Printf("%s: %d build(s) wait for a gate\n", n, len(waiting[n]))
for _, mv := range waiting[n] {
fmt.Printf(" %-28s %s → %s\n", mv.Module, short(mv.From), short(mv.To))
}
}
return nil
}
cmd/mesh-controller/release_plan.go
+43 -365
View File
@@ -5,7 +5,6 @@ import (
"errors"
"flag"
"fmt"
"slices"
"sort"
"strings"
"time"
@@ -224,9 +223,6 @@ func supersededBy(newer inventory.Plan, open []inventory.Plan, rollsOut func(str
}
var took []string
for name, s := range old.Modules {
if s != nil && s.State == planDeleted {
continue // deleted at its source: nothing to plan again
}
if s == nil || s.State != "built" || (s.SentAt == nil && rollsOut(name)) {
folded[name] = true
took = append(took, name)
@@ -431,11 +427,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
} else if askedBefore(asked, state.AskedAt) {
continue
}
if failed != "" && deletedAtSource(failed) {
// Deleted at its source by the merge, not broken (novox/hq ADR 0236): the plan goes on.
state.State, state.Why = planDeleted, "deleted at its source: "+firstLine(failed)
forgetDeleted(ctx, inv, module, p)
} else if failed != "" {
if failed != "" {
state.State = "failed"
state.Why = failed
p.State = inventory.PlanFailed
@@ -449,7 +441,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
state.BuiltAt = &now
state.Commit = commit
}
if err := inv.SavePlan(ctx, p); err != nil {
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
continue
}
@@ -476,10 +468,6 @@ func advancePlans(ctx context.Context, open *stores) {
return
}
defer release()
// What waits for a gate, released one machine at a time (ADR 0236).
if _, err := releaseBacklog(ctx, open, ""); err != nil {
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
}
advanceHeld(ctx, open)
}
@@ -512,7 +500,7 @@ func advanceHeld(ctx context.Context, open *stores) {
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
// the state is left as it was and the step is tried again on the next tick.
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
if err := inv.SavePlan(ctx, p); err != nil {
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
}
break
@@ -520,7 +508,7 @@ func advanceHeld(ctx context.Context, open *stores) {
if p.State == inventory.PlanFailed {
sayUnsent(p, rollsOut)
}
if err := inv.SavePlan(ctx, p); err != nil {
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
break
}
@@ -535,17 +523,6 @@ func advanceHeld(ctx context.Context, open *stores) {
func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
edges []inventory.Edge, rollsOut func(string) bool) (bool, error) {
inv := open.inventory
if p.Release != nil {
return advanceRelease(ctx, open, p)
}
// **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239): nothing of it is
// registered before its turn, so no other send carries it to a machine.
if p.Waiting() {
note := waitingNote(*p)
changed := p.Note != note
p.Note = note
return changed, nil
}
if p.Tier >= len(p.Tiers) {
p.State = inventory.PlanDone
fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers))
@@ -598,7 +575,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
var latest time.Time
for _, m := range tier {
s := p.Modules[m]
if s == nil || (s.State != "built" && s.State != planDeleted) {
if s == nil || s.State != "built" {
return false, nil
}
if s.BuiltAt != nil && s.BuiltAt.After(latest) {
@@ -618,44 +595,16 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
// minute. Now the first machine is sent, the plan records it and waits for that machine's report
// after the send to say it applied what it was sent; only then are the rest sent. A first machine
// that fails or refuses stops the module's rollout and the plan with it, the rest untouched.
//
// **One send per machine for the whole tier** (novox/hq issue 281). A send carries the machine's
// whole declaration (ADR 0221), and the plan sent the first machine once per module: on 2026-10-06 a
// tier of seventy modules sent one laptop a new declaration every few seconds, the node-engine set
// aside one for the next, the node tools never settled, and the gate blamed a module for the machine
// being behind. Now every module of the tier whose first machine is the same is sent there in one
// send, judged there by one gate — kept on the first of them, the others gated by it — and the rest
// of the tier's machines are sent once, together, when the gates have passed.
var pending []string
firstOn := map[string][]string{}
runningOf := map[string][]string{}
var rest []string
restTo := map[string][]string{}
together := map[string]bool{}
for _, m := range tier {
state := p.Modules[m]
if state == nil || state.SentAt != nil || state.State == planDeleted || !rollsOut(m) {
if state == nil || state.SentAt != nil || !rollsOut(m) {
continue
}
running, err := inv.Running(ctx, m)
if err != nil {
return false, err
}
runningOf[m] = running
// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
// of the build every machine running it was sent, it is registered with that build's artifacts —
// nothing to send, and nothing for a gate to judge.
if state.FirstAt == nil {
if same, err := unchangedOnEvery(ctx, inv, m, state.Build, running); err != nil {
return false, err
} else if same {
now := time.Now().UTC()
state.SentAt = &now
state.Why = "no move: made from the source every machine running it runs"
fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
continue
}
}
policy, err := inv.UpgradeOf(ctx, m)
if err != nil {
return false, err
@@ -669,121 +618,41 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
}
now := time.Now().UTC()
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
// **Sent with others, judged with them** (issue 281): the gate of the send that carried it is its
// verdict on its first machine. A failure there stopped the plan already.
if state.GatedBy != "" {
lead := p.Modules[state.GatedBy]
if lead == nil || lead.Gate == nil || lead.Gate.Verdict == "" {
pending = append(pending, fmt.Sprintf("%s judged with %s on %s", m, state.GatedBy, strings.Join(state.First, ", ")))
continue
}
if lead.Gate.Verdict != inventory.GatePassed {
continue
}
passedWith(state, state.GatedBy, lead.Gate)
}
switch {
case step.failed != "":
// The first machine refused or failed what it was sent, or never said: the gate failed, and
// what it carried is put back there (novox/hq ADR 0236); the rest are left as they were.
failFirstSend(ctx, open, p, m, state, firstRunning(state.First, running), step.failed, step.rest)
state.Why = step.failed
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s stopped at its first machine in tier %d: %s; %s left as it was",
m, p.Tier, step.failed, orNone(strings.Join(step.rest, ", ")))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
case step.waiting != "":
at := ""
if state.FirstAt != nil {
at = state.FirstAt.Local().Format("15:04")
}
pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, at))
pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Local().Format("15:04")))
continue
}
// **The gate** (novox/hq ADR 0236, to-be 45 §8): the first machine reported the build applied;
// it is judged by its health before anything else is sent — the rest, or, where it is the only
// machine, the plan's next step.
if state.GatedBy == "" && !policy.Together && state.FirstAt != nil && len(state.First) > 0 {
verdict, err := judgeGate(ctx, open, p, m, state, running, now)
if err != nil {
return false, err
}
switch verdict {
case "":
pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m,
strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate)))
continue
case inventory.GateFailed:
failFirstSend(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why, step.rest)
return true, nil
case inventory.GatePassed:
if !state.Gate.Kept {
gatePassed(ctx, open, p, m, state)
state.Gate.Kept = true
}
}
}
switch {
case len(step.send) == 0:
// No machine runs it: nothing to send, and nothing to wait for.
state.SentAt = &now
continue
case step.first:
firstOn[step.send[0]] = append(firstOn[step.send[0]], m)
continue
}
rest = append(rest, m)
restTo[m] = step.send
together[m] = policy.Together
}
// The first machines: one send each for everything of the tier that goes there first, one machine
// per step — the plan is kept between them, so the next machine's send sees what this one walks.
if len(firstOn) > 0 {
machines := make([]string, 0, len(firstOn))
for n := range firstOn {
machines = append(machines, n)
// What sendToEach answers, not what was asked: the machine holding the bus is sent before
// the first when its user list must change (issue 249), and the plan waits for it too.
sent, err := sendToEach(ctx, open, step.send)
if err != nil {
// Not marked sent, so the next step tries again (issue 249): a grant that could not be
// issued is a send that did not happen.
return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(step.send, ", "), p.Tier, err)
}
sort.Strings(machines)
for _, node := range machines {
err := firstSend(ctx, open, p, node, firstOn[node], runningOf)
if errors.Is(err, errWalkedElsewhere) {
// A build this machine would carry is being judged elsewhere: it goes once that gate passed.
pending = append(pending, fmt.Sprintf("%s for %s: %v", node, strings.Join(firstOn[node], ", "), err))
continue
}
if err != nil {
// Not marked sent, so the next step tries again (issue 249): a grant that could not be
// issued is a send that did not happen.
return false, fmt.Errorf("sending %s to %s first after tier %d: %w", strings.Join(firstOn[node], ", "),
node, p.Tier, err)
}
if step.first {
state.First = sent
state.FirstAt = &now
p.State = inventory.PlanRolling
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first", p.Tier, m, strings.Join(sent, ", "))
fmt.Printf("%s: tier %d built; sent %s to %s first, the rest once it reports it applied\n",
p.ID, p.Tier, m, strings.Join(sent, ", "))
return true, nil
}
}
// The rest, after their gates passed, and what rolls out together: every machine once.
if len(rest) > 0 {
var machines []string
scope := sendScope{modules: map[string]bool{}}
for _, m := range rest {
for _, n := range restTo[m] {
if !slices.Contains(machines, n) {
machines = append(machines, n)
}
}
if together[m] {
// What its policy rolls out together may move wherever it goes; anything else no gate has
// seen still stops the send.
scope.modules[m] = true
}
}
sort.Strings(machines)
sent, err := sendRollout(withScope(ctx, scope), open, machines)
if err != nil {
return false, fmt.Errorf("sending %s to %s after tier %d: %w", strings.Join(rest, ", "),
strings.Join(machines, ", "), p.Tier, err)
}
now := time.Now().UTC()
for _, m := range rest {
p.Modules[m].SentAt = &now
}
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
strings.Join(sent, ", "))
state.SentAt = &now
fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(sent, ", "))
return true, nil
}
if len(pending) > 0 {
@@ -812,9 +681,6 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
state = &inventory.PlanModule{}
p.Modules[m] = state
}
if state.State == planDeleted {
continue
}
// The plan sends what it waits for. A rebuild from the same source commit is not a
// move the catalogue announces — the build machine rebuilt for a controller change
// is one — so the roll-out that opens this gate is the plan's to make, once, and
@@ -861,91 +727,6 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
return true, nil
}
// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
// before the send, so a module the same send carries is never read as already moved — the machines it
// went to, and the gate. The gate is kept on the first of them and judges everything the send moved;
// the others are gated by it.
func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
runningOf map[string][]string) error {
inv := open.inventory
before, beforeKnown, _ := inv.SentBuilds(ctx, node)
owns := make([]inventory.CarriedMove, 0, len(modules))
for _, m := range modules {
s := p.Modules[m]
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
}
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
// all of it there.
carried, sent, err := gatedSend(ctx, open, node, owns)
if err != nil {
return err
}
now := time.Now().UTC()
lead := modules[0]
for _, m := range modules {
s := p.Modules[m]
// What the first machine ran before: what a failed gate puts back (ADR 0236).
if beforeKnown {
s.Previous = before[m]
}
s.First, s.FirstAt = sent, &now
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
From: s.Previous, To: s.Commit, Since: &now}
s.GatedBy = ""
if m == lead {
s.Gate.Carried = carried
} else {
s.GatedBy = lead
}
}
p.State = inventory.PlanRolling
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
strings.Join(sent, ", "))
fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
return nil
}
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
func passedWith(s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
if s.Gate == nil {
s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since}
}
if s.Gate.Verdict != "" {
return
}
s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+g.Why, g.JudgedAt, g.Took
s.Gate.Passes, s.Gate.Kept = g.Passes, true
}
// failFirstSend stops the plan at a first send whose gate failed, or whose machine refused or failed
// what it was sent: what the gate found wanting is put back there, each once — the module the gate is
// kept on only when it is among them (issue 281: the module whose gate it is was blamed for whatever
// the send carried) — and the machines after it are left as they were.
func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
machines []string, why string, rest []string) {
batched, back := batchingRollbacks(ctx)
defer func() {
sendRollbacks(ctx, open, p, back)
p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(rest, ", ")))
fmt.Printf("%s: %s\n", p.ID, p.Note)
}()
g := state.Gate
if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
gateFailed(batched, open, p, module, state, machines, why)
} else {
state.Why = "not found wanting; stopped with the send that carried it: " + g.Why
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was not found wanting and is left as it is",
strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
}
if state.Gate != nil && len(state.Gate.Carried) > 0 {
failCarried(batched, open, p, state.Gate, module)
}
}
// rolloutStep is what a plan does next with one built module's machines (novox/hq issue 249).
type rolloutStep struct {
// send is the machines to send now; first, whether they are the first machine's send.
@@ -1081,10 +862,6 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
}
since := now.Sub(p.Updated).Round(time.Second)
if p.Waiting() {
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
return fmt.Sprintf("%s %s %s, %s, for %s", p.Repository, short(p.Commit), where, waitingNote(p), since)
}
if waiting, paused := pausedWaiting(p, pause, now); paused {
return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
}
@@ -1209,20 +986,10 @@ func plansCommand(ctx context.Context, args []string) error {
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
modules := set.String("modules", "", "or the modules it would change, comma-separated")
moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+
"comma-separated, as the forge's announcer says them (novox/hq issue 278)")
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
why := addHandActFlags(set)
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close" || positionals[0] == "go") {
// Refused before anything is opened: a repair by hand says why.
if err := why.require("plans " + positionals[0]); err != nil {
return err
}
}
open, err := openStores(ctx)
if err != nil {
return err
@@ -1236,18 +1003,6 @@ func plansCommand(ctx context.Context, args []string) error {
return err
}
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
if r := p.Release; r != nil {
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
if r.Gate != nil {
fmt.Printf(" %s\n", gateLine(r.Gate))
for _, c := range r.Gate.Carried {
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
}
}
return nil
}
for i, tier := range p.Tiers {
marker := " "
if i == p.Tier && p.Open() {
@@ -1270,36 +1025,12 @@ func plansCommand(ctx context.Context, args []string) error {
}
}
fmt.Printf(" %-22s %s\n", m, state)
// The rollout's record at its gate (novox/hq to-be 45 §8): first machine, from and to,
// verdict, time to it, rolled back or not.
switch {
case s != nil && s.GatedBy != "" && (s.Gate == nil || s.Gate.Verdict == ""):
// Sent with others, judged by the gate of that send (issue 281).
fmt.Printf(" %-22s judged with %s, in the send to %s\n", "", s.GatedBy, strings.Join(s.First, ", "))
case s != nil && s.Gate != nil:
fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate))
}
}
}
return nil
}
if *whatIf != "" {
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules), moduleDirsOf(*moduleDirs))
}
// `go` (novox/hq ADR 0239): a person's word in place of the delivery's owner, for a walk that waits for it
// — mesh-delivery down, or a person who will not wait. Recorded in the hand-act log with why.
if len(positionals) == 2 && positionals[0] == "go" {
why.record(ctx, "plans go", positionals[1:])
var p inventory.Plan
if err := sayingOnTheBus(ctx, func() (err error) {
p, err = letGo(ctx, inv, positionals[1], "a person ("+whoAsked()+")", strings.TrimSpace(*why.why))
return err
}); err != nil {
return err
}
fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked at the next pass\n", p.ID, p.Repository,
short(p.Commit))
return nil
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
}
// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
if len(positionals) == 2 && positionals[0] == "retry" {
@@ -1330,14 +1061,13 @@ func plansCommand(ctx context.Context, args []string) error {
if !p.Open() {
return fmt.Errorf("%s is already %s", p.ID, p.State)
}
why.record(ctx, "plans "+positionals[0], positionals[1:])
p.State = inventory.PlanFailed
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
sayUnsent(&p, func(m string) bool {
u, err := inv.UpgradeOf(ctx, m)
return err == nil && u.RollOut
})
if err := inv.SavePlan(ctx, &p); err != nil {
if err := inv.SavePlan(ctx, p); err != nil {
return err
}
fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
@@ -1362,15 +1092,12 @@ func plansCommand(ctx context.Context, args []string) error {
// planWhatIf is the plan a merge would produce, computed the way the merge handler computes one
// and saved nowhere: the modules the repository's changed files touch (or the modules named), what
// packages their source, everything reachable from them, in tiers. For reading before merging.
func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string, paths, modules []string, moduleDirs *[]string) error {
func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string, paths, modules []string) error {
owner, repo, found := strings.Cut(repository, "/")
if !found {
return fmt.Errorf("--what-if takes owner/repository, not %q", repository)
}
m := link.SourceMoved{Owner: owner, Repo: repo, Base: "main", Commit: "what-if", Paths: paths}
if moduleDirs != nil {
m.ModuleDirs, m.ModuleDirsSaid = *moduleDirs, true
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
@@ -1384,21 +1111,19 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
for _, name := range modules {
named[name] = true
}
if len(named) == 0 {
// The changed files: the planner's own answer, as the merge handler, the merge gate and a pull
// request's check ask it (novox/hq ADR 0238).
r := reachOfMerge(m, entries, read, nil)
from, packaging = append(append([]inventory.Entry{}, r.Touched...), r.Deleted...), r.Packaging
} else {
for _, e := range entries {
switch {
case named[e.Manifest.Module]:
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
for _, e := range entries {
switch {
case named[e.Manifest.Module]:
from = append(from, e)
case len(named) == 0 && sourceIs(e.Source, m):
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
}
if len(named) == 0 {
from = whatTheMergeTouched(from, entries, m)
}
moved := append(append([]inventory.Entry{}, from...), packaging...)
if len(moved) == 0 {
fmt.Printf("a merge of %s changing %s would build nothing the mesh holds\n", repository,
@@ -1448,16 +1173,6 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
return nil
}
// moduleDirsOf is --module-dirs as an announcer would say it: nothing when not given, so the what-if
// reads as an announcement from an announcer that does not say.
func moduleDirsOf(s string) *[]string {
if strings.TrimSpace(s) == "" {
return nil
}
dirs := splitList(s)
return &dirs
}
func splitList(s string) []string {
var out []string
for _, part := range strings.Split(s, ",") {
@@ -1507,12 +1222,6 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
continue
}
at := outcome.At
if !outcome.Worked() && deletedAtSource(outcome.Failed) {
s.State, s.Why = planDeleted, "deleted at its source: "+firstLine(outcome.Failed)
fmt.Printf("%s: %s was deleted at its source (%s): not built, and the plan goes on\n", p.ID, m, outcome.ID)
changed = true
continue
}
if outcome.Worked() {
s.State = "built"
s.BuiltAt = &at
@@ -1534,34 +1243,3 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
func askedBefore(asked time.Time, planAsked *time.Time) bool {
return !asked.IsZero() && planAsked != nil && asked.Before(*planAsked)
}
// firstRunning is the machines sent first that run the module — not the machine holding the bus, sent
// with them only for its user list.
func firstRunning(first, running []string) []string {
var out []string
for _, n := range first {
if slices.Contains(running, n) {
out = append(out, n)
}
}
if len(out) == 0 {
return first
}
return out
}
// planDeleted is a plan's module that the merge deleted at its source: not built, not sent, and no
// failure of the plan (novox/hq ADR 0236).
const planDeleted = "deleted"
// forgetDeleted forgets a module a plan found deleted at its source, where nothing holds it, and says
// it otherwise.
func forgetDeleted(ctx context.Context, inv *inventory.Inventory, module string, p *inventory.Plan) {
switch err := inv.ForgetModule(ctx, module); {
case err == nil:
fmt.Printf("%s: %s was deleted at its source: forgotten, and the plan goes on\n", p.ID, module)
default:
fmt.Printf("%s: %s was deleted at its source and is not built; it is kept until a person unassigns it and "+
"`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error()))
}
}
cmd/mesh-controller/release_plan_test.go
-42
View File
@@ -212,45 +212,3 @@ func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
t.Fatal("a report from before the machine was sent opened the gate")
}
}
// novox/hq issue 278 and ADR 0236's open question: a change to the build agent — its manifest alone, as
// the catalogue's data sections were, or its program — rebuilds the build agent and nothing it builds.
// What it builds is ordered after it in a plan that holds both, never added to one for its sake. The
// merge that rebuilt 103 modules with the agent in tier 0 rebuilt them for a file read as shared code;
// the agent stood first only because everything else is built by it.
func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) {
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
agent := fromRepo("build-agent", repo, "modules/build-agent")
redis := fromRepo("redis", repo, "modules/redis")
postgres := fromRepo("postgres", repo, "modules/postgres")
entries := []inventory.Entry{agent, redis, postgres}
edges := []inventory.Edge{
{From: "redis", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
{From: "redis", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "postgres", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "mesh-tools", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
}
for _, paths := range [][]string{
{"modules/build-agent/module.json"}, // its manifest alone
{"modules/build-agent/cmd/agent/main.go", "modules/build-agent/module.json"}, // its program too
} {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", Paths: paths,
ModuleDirs: []string{"modules/build-agent"}, ModuleDirsSaid: true}
touched := whatTheMergeTouched(entries, entries, m)
if len(touched) != 1 || touched[0].Manifest.Module != "build-agent" {
t.Fatalf("%v touched %v", paths, touched)
}
p := planOfMerge(m, []string{"build-agent"}, edges)
if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "build-agent" || len(p.Modules) != 1 {
t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers)
}
}
// With what it builds moved beside it, the agent comes first and they follow: an order, not a widening.
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"},
[]string{"build-agent", "redis"}, edges)
if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 {
t.Fatalf("the agent, then what moved beside it: %v", p.Tiers)
}
}
cmd/mesh-controller/release_test.go
-257
View File
@@ -1,257 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The backlog the old default left — builds registered and sent nowhere — is released by a plan, one
// machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236).
type backlogMesh struct {
open *stores
sent [][]string
broken map[string]bool
}
// aBacklog is a mesh where `app` moved c1 → c2 on anchor and laptop, `late` moved on laptop only, and
// `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once.
func aBacklog(t *testing.T) *backlogMesh {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
b := &backlogMesh{open: open, broken: map[string]bool{}}
withConditionsInMemory(t)
was := doctorFrom
doctorFrom = nil
t.Cleanup(func() { doctorFrom = was })
build := func(module, commit, made string, asked time.Time) {
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}}
for _, m := range []string{"app", "late", "same"} {
build(m, "c1", "sha256:"+m+"-1", old)
for _, n := range on[m] {
if _, err := inv.Assign(ctx, n, m); err != nil {
t.Fatal(err)
}
}
}
for _, n := range []string{"anchor", "laptop"} {
sent := map[string]string{}
for m, nodes := range on {
for _, x := range nodes {
if x == n {
sent[m] = "c1"
}
}
}
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
t.Fatal(err)
}
}
build("app", "c2", "sha256:app-2", now)
build("late", "c2", "sha256:late-2", now)
build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes
assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
machineMoves = assigned
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
return map[string]bool{"anchor": true, "laptop": true}, nil
}
n := 0
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
b.sent = append(b.sent, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
modules, _ := open.inventory.Assigned(ctx, node)
carried := map[string]string{}
for _, m := range modules {
carried[m] = current[m].Commit
}
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
if b.broken[r.Node] {
r.Outcome = inventory.OutcomeFailed
}
f.reports[r.Node] = r
}
return f, nil
}
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
})
return b
}
func (b *backlogMesh) release(t *testing.T) inventory.Plan {
t.Helper()
plans, err := b.open.inventory.RecentPlans(t.Context(), 10)
if err != nil {
t.Fatal(err)
}
for _, p := range plans {
if p.Release != nil {
return p
}
}
t.Fatal("no release plan")
return inventory.Plan{}
}
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
f, err := readMoveFacts(ctx, inv)
if err != nil {
t.Fatal(err)
}
moves, _ := machineMoves(ctx, b.open, f, "laptop", false)
var names []string
for _, mv := range moves {
names = append(names, mv.Module)
}
if !reflect.DeepEqual(names, []string{"app", "late"}) {
t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names)
}
// Nothing else may carry them: a send that judges nothing is refused.
if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) {
t.Fatalf("a send that judges nothing would carry them: %v", err)
}
advancePlans(ctx, b.open)
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent)
}
p := b.release(t)
if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil {
t.Fatalf("the release plan is %+v", p.Release)
}
gateEvery = 0
for i := 0; i < 4; i++ {
advancePlans(ctx, b.open)
}
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("sent %v", b.sent)
}
p = b.release(t)
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) {
t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release)
}
for _, build := range []string{"build-app-c2", "build-late-c2"} {
if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err)
}
}
// Nothing waits now, and nothing opens again.
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
t.Fatalf("a release opened with nothing waiting: %+v %v", p, err)
}
}
// A release that fails on its first machine puts back what it carried there, goes no further, and the
// next one waits for a person, said as a condition; a person releases it with why.
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
gateEvery = 0
b.broken["anchor"] = true
advancePlans(ctx, b.open)
p := b.release(t)
if p.State != inventory.PlanFailed {
t.Fatalf("the release is %s: %s", p.State, p.Note)
}
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) {
t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("app is at %s, not put back", current["app"].Commit)
}
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
t.Fatal("app's build is not marked failed at its gate")
}
// late still waits on laptop, and is not released on its own after a failure.
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
t.Fatalf("a release opened after a failed one: %+v %v", p, err)
}
if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") {
t.Fatalf("the held release is not said: %+v", obs)
}
b.broken["anchor"] = false
if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
advancePlans(ctx, b.open)
}
if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" {
t.Fatalf("the person's release is %s (%+v)", p.State, p.Release)
}
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
t.Fatalf("late was not released to laptop: %v", sent)
}
}
cmd/mesh-controller/replays_test.go
-113
View File
@@ -1,113 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
// happened, asserting the rule's outcome rather than the fix's mechanism, so it can be run against the
// commit before the fix and fail there, and against the fix and pass. **Written only with what the
// controller had before each fix** — the stores, register, assign, planFor, declarationFor — so the
// prover (mesh-lab replays/cmd/prove) can lay this file over the older commit and run it there.
//
// Registered in mesh-lab's replays/register.go with the fix each one proves; run by this repository's
// merge check on every pull request with the rest of the suite.
// **R263 — a consumer's identity never refuses its provider's machine.** On 2026-10-06 the network
// manager came to require the mesh's resolver; on a machine whose name made its identity 23 to 26
// characters against the one global bound of 20, the anchor — the resolver's holder, carrying every
// consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the
// provider's machine composes whatever its consumers are called, and a provision that mints no
// credential holds no consumer to a key's length.
func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
Requires: []string{"wildcard-resolution"}})
for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
// The consumer's machine composes first, as a push does: what it is sent is what its provider grants.
for _, node := range []string{"laptop", "anchor"} {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatalf("%s does not resolve: %v", node, err)
}
if _, err := declarationFor(ctx, open, node, plan, settings); err != nil {
t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+
"mesh_laptop_networkmanager (26 characters), refused it: %v", node, err)
}
}
}
// **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for
// the resolver re-bound every database consumer on the home server — which runs its own store, beside
// its applications' data — to the store seat's holder on the control node, which made each a new empty
// database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a
// machine running its own store stays bound to it while another machine holds the store's seat; the
// resolver, which every holder answers alike, follows its seat.
func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
for _, m := range []catalogue.Manifest{
{Module: "store", Version: "1",
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
} {
register(t, open, m)
}
assignAll := func(pairs ...[2]string) {
for _, a := range pairs {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
}
// The control node holds the mesh's store and resolver seats; the home server runs its own of each.
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
t.Fatal(err)
}
}
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
[2]string{"laptop", "board"})
plan, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
var board, network string
for _, n := range plan.Needs {
switch {
case n.For == "board" && n.Name == "postgres-database":
board = n.From
case n.For == "network" && n.Name == "wildcard-resolution":
network = n.From
}
}
if board != "laptop" {
t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+
"database there (issue 273)", board)
}
if network != "anchor" {
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
}
}
cmd/mesh-controller/retire_verbs.go
-479
View File
@@ -1,479 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The verbs a person answers a provider's retirement with, and cleans up with (novox/hq ADR 0230).
//
// **The controller asks; the provider acts.** Approving, rejecting and deleting are each a question to
// the provider on the machine it runs on — its `provisioner_*` tools — because the provider owns its
// backend and the controller touches none. Every one says why, is written in the hand-act log before
// it is asked, and the provider says what it did as its `provisioner.retirement` event.
const retireUsage = "retire [--json] | retire approve <node> <module> --why <text> | retire reject <node> <module> --why <text>"
const cleanupUsage = "cleanup [list] [--json] | cleanup delete <node> <module> <consumer> --why <text> | " +
"cleanup delete --older-than <days> --why <text> [--confirm]"
func isNothingServes(err error) bool { return errors.Is(err, link.ErrNothingServes) }
func unmarshalAnswer(a link.Answer, v any) error {
if len(a.Result) == 0 {
return errors.New("an empty answer")
}
return json.Unmarshal(a.Result, v)
}
// retireCommand is `retire`, `retire approve` and `retire reject`.
func retireCommand(ctx context.Context, args []string) error {
sub := "list"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
switch sub {
case "list":
set := flag.NewFlagSet("retire", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(retireUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error { return listRetiring(ctx, open.inventory, conn, *asJSON) })
case "approve", "reject":
set := flag.NewFlagSet("retire "+sub, flag.ContinueOnError)
f := addHandActFlags(set)
rest, err := parseAround(set, args)
if err != nil {
return err
}
if len(rest) != 2 {
return errors.New(retireUsage)
}
if err := f.require("retire " + sub); err != nil {
return err
}
return onTheBus(func(conn *nats.Conn) error {
return answerRetirement(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, sub == "approve", f)
})
}
return errors.New(retireUsage)
}
// retiringRow is one provider's waiting or rejected set, as `retire` lists it.
type retiringRow struct {
Node string `json:"node"`
Module string `json:"module"`
Waiting []link.RetiredConsumer `json:"waiting,omitempty"`
Since string `json:"since,omitempty"`
Held int `json:"held,omitempty"`
Bound string `json:"bound,omitempty"`
Rejected []link.RetiredConsumer `json:"rejected,omitempty"`
RejectWhy string `json:"rejected-why,omitempty"`
Unasked string `json:"unasked,omitempty"`
}
func listRetiring(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, asJSON bool) error {
instances, err := providerInstances(ctx, inv)
if err != nil {
return err
}
var rows []retiringRow
for _, p := range instances {
row := retiringRow{Node: p.Node, Module: p.Module}
state, err := askRetirement(ctx, conn, p)
switch {
case isNothingServes(err):
row.Unasked = "answers no retirement question: it predates ADR 0230"
case err != nil:
row.Unasked = err.Error()
default:
if state.Waiting != nil {
row.Waiting, row.Since, row.Held = state.Waiting.Consumers, state.Waiting.Since, state.Waiting.Held
}
if state.Rejected != nil {
row.Rejected, row.RejectWhy = state.Rejected.Consumers, orWhy(state.Rejected.By, state.Rejected.Why)
}
row.Bound = state.Bound
if row.Waiting == nil && row.Rejected == nil {
continue
}
}
rows = append(rows, row)
}
if asJSON {
if rows == nil {
rows = []retiringRow{}
}
return printJSON(map[string]any{"providers": rows})
}
said := false
for _, r := range rows {
switch {
case r.Unasked != "":
fmt.Printf("%s on %s: not asked — %s\n", r.Module, r.Node, r.Unasked)
default:
if r.Waiting != nil {
said = true
fmt.Printf("%s on %s WAITS since %s to retire %d of the %d it holds (%s): %s\n"+
" retire approve %s %s --why … | retire reject %s %s --why …\n",
r.Module, r.Node, r.Since, len(r.Waiting), r.Held, orBound(r.Bound), consumerList(r.Waiting),
r.Node, r.Module, r.Node, r.Module)
}
if r.Rejected != nil {
said = true
fmt.Printf("%s on %s keeps active, by a rejection (%s): %s\n", r.Module, r.Node, r.RejectWhy,
consumerList(r.Rejected))
}
}
}
if !said {
fmt.Println("no provider waits for a person to approve a retirement")
}
return nil
}
// answerRetirement approves or rejects what one provider waits with. **The set sent is the set the
// provider says it waits with, read now**, and the provider refuses any other: a person approves what
// they were shown, never a set that moved since.
func answerRetirement(ctx context.Context, conn *nats.Conn, p providerInstance, approve bool, f handActFlags) error {
state, err := askRetirement(ctx, conn, p)
if err != nil {
return err
}
verb, tool := "retire reject", link.ToolRetireReject
var set []link.RetiredConsumer
if state.Waiting != nil {
set = state.Waiting.Consumers
}
if approve {
verb, tool = "retire approve", link.ToolRetireApprove
if set == nil && state.Rejected != nil {
// A rejection can be taken back: the consumers it kept are retired after all.
set = state.Rejected.Consumers
}
}
if len(set) == 0 {
return fmt.Errorf("%s on %s waits for nobody to approve or reject a retirement. Nothing was done", p.Module, p.Node)
}
names := make([]string, 0, len(set))
for _, c := range set {
names = append(names, c.Consumer)
}
if strings.TrimSpace(*f.cause) == "" {
*f.cause = kindRetireWaiting
}
if strings.TrimSpace(*f.condition) == "" {
*f.condition = retireWaitingKey(p.Module, p.Node)
}
f.record(ctx, verb, append([]string{p.Node, p.Module}, names...))
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, tool, p.Node, map[string]any{
"consumers": names, "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController,
}, retirementAsk)
if err != nil {
return err
}
if answer.Error != "" {
return fmt.Errorf("%s on %s refused: %s", p.Module, p.Node, answer.Error)
}
if approve && state.RetiresBy == "mark-only" {
fmt.Printf("%s on %s marked %s retired, MARK ONLY: this provider cannot disable a consumer, so they keep "+
"their access until `cleanup delete`; `cleanup list` shows them\n", p.Module, p.Node, strings.Join(names, ", "))
} else if approve {
fmt.Printf("%s on %s retired %s: access disabled, data kept; `cleanup list` shows them\n", p.Module, p.Node,
strings.Join(names, ", "))
} else {
fmt.Printf("%s on %s keeps %s active; the warning stays open until the mesh asks for them again or the "+
"retirement is approved\n", p.Module, p.Node, strings.Join(names, ", "))
}
return nil
}
// cleanupCommand is `cleanup list` and `cleanup delete`.
func cleanupCommand(ctx context.Context, args []string) error {
sub := "list"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
switch sub {
case "list":
set := flag.NewFlagSet("cleanup", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(cleanupUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error {
listing, err := gatherRetired(ctx, open.inventory, conn, time.Now())
if err != nil {
return err
}
return printRetired(listing, *asJSON)
})
case "delete":
set := flag.NewFlagSet("cleanup delete", flag.ContinueOnError)
f := addHandActFlags(set)
olderThan := set.Int("older-than", 0, "every retired consumer older than this many days")
confirm := set.Bool("confirm", false, "with --older-than: delete what is listed, rather than only list it")
rest, err := parseAround(set, args)
if err != nil {
return err
}
if err := f.require("cleanup delete"); err != nil {
return err
}
if strings.TrimSpace(*f.cause) == "" {
*f.cause = kindCleanupWaiting
}
switch {
case *olderThan > 0 && len(rest) == 0:
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error {
return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now())
})
case *olderThan == 0 && len(rest) == 3 && !*confirm:
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error {
// A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a
// provider's retired consumer otherwise.
if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil &&
r.RetiredAt != nil {
return deleteRetiredData(ctx, conn, open, r, f)
}
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
})
}
return errors.New(cleanupUsage)
}
return errors.New(cleanupUsage)
}
// retiredRow is one retired consumer, as `cleanup list` shows it.
type retiredRow struct {
Node string `json:"node"`
Module string `json:"module"`
Consumer string `json:"consumer"`
// ConsumerNode is where the consumer was, when the provider knows.
ConsumerNode string `json:"consumer-node,omitempty"`
Kind string `json:"kind,omitempty"`
RetiredAt string `json:"retired-at,omitempty"`
// AgeDays is whole days since it was retired; -1 when the provider could not say when.
AgeDays int `json:"age-days"`
SizeBytes *int64 `json:"size-bytes,omitempty"`
Why string `json:"why,omitempty"`
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
Access string `json:"access,omitempty"`
// Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is
// kept on its machine, and its class. Consumer is then the item.
Path string `json:"path,omitempty"`
Class string `json:"class,omitempty"`
}
// retiredListing is every provider's retired consumers, and the providers that could not say.
type retiredListing struct {
Retired []retiredRow `json:"retired"`
// Unasked names each provider not asked, and why: one older than the question, or one that failed.
Unasked []string `json:"unasked,omitempty"`
}
func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, now time.Time) (retiredListing, error) {
instances, err := providerInstances(ctx, inv)
if err != nil {
return retiredListing{}, err
}
listing := retiredOf(ctx, conn, instances, now)
// And every module's own data retired on its machine (novox/hq ADR 0233).
records, err := inv.Data(ctx)
if err != nil {
return retiredListing{}, err
}
listing.Retired = append(listing.Retired, retiredData(records, now)...)
sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays })
return listing, nil
}
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
out := retiredListing{Retired: []retiredRow{}}
for _, p := range instances {
state, err := askRetirement(ctx, conn, p)
if err != nil {
if isNothingServes(err) {
out.Unasked = append(out.Unasked, fmt.Sprintf("%s on %s answers no retirement question: it predates ADR 0230", p.Module, p.Node))
} else {
out.Unasked = append(out.Unasked, err.Error())
}
continue
}
for _, c := range state.Retired {
row := retiredRow{Node: p.Node, Module: p.Module, Consumer: c.Consumer, ConsumerNode: c.Node, Kind: c.Kind,
RetiredAt: c.RetiredAt, AgeDays: -1, SizeBytes: c.SizeBytes, Why: c.Why, Access: c.Access}
if at := retiredAt(c); !at.IsZero() {
row.AgeDays = int(now.Sub(at).Hours() / 24)
}
out.Retired = append(out.Retired, row)
}
}
sort.SliceStable(out.Retired, func(i, j int) bool { return out.Retired[i].AgeDays > out.Retired[j].AgeDays })
return out
}
func printRetired(l retiredListing, asJSON bool) error {
if asJSON {
return printJSON(l)
}
if len(l.Retired) == 0 {
fmt.Println("no provider holds a retired consumer, and no machine holds retired data")
}
for _, r := range l.Retired {
age := "age unknown"
if r.AgeDays >= 0 {
age = strconv.Itoa(r.AgeDays) + " day(s)"
}
kind := ""
if r.Kind != "" && r.Kind != "consumer" {
kind = " [" + r.Kind + "]"
}
if r.Access == "kept" {
kind += " [MARK ONLY: access kept until deleted]"
}
if r.Kind == retiredDataKind {
fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer,
r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
continue
}
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
}
for _, u := range l.Unasked {
fmt.Printf("not asked: %s\n", u)
}
return nil
}
// deleteRetired asks one provider to delete one consumer it holds retired — never an active one: the
// provider refuses that, and this refuses it first, from what the provider says it holds.
func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, consumer string, f handActFlags) error {
state, err := askRetirement(ctx, conn, p)
if err != nil {
return err
}
found := false
for _, c := range state.Retired {
found = found || c.Consumer == consumer
}
if !found {
return fmt.Errorf("%s on %s holds no retired consumer %s — only a retired consumer is deleted. Nothing was done",
p.Module, p.Node, consumer)
}
f.record(ctx, "cleanup delete", []string{p.Node, p.Module, consumer})
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetiredDelete, p.Node, map[string]any{
"consumer": consumer, "confirm": consumer, "why": strings.TrimSpace(*f.why), "by": link.Caller(),
"via": link.ViaController,
}, retirementAsk)
if err != nil {
return err
}
if answer.Error != "" {
return fmt.Errorf("%s on %s refused to delete %s: %s", p.Module, p.Node, consumer, answer.Error)
}
var done struct {
FreedBytes *int64 `json:"freed_bytes"`
}
_ = unmarshalAnswer(answer, &done)
fmt.Printf("%s on %s deleted %s (%s freed)\n", p.Module, p.Node, consumer, sizeWords(done.FreedBytes))
return nil
}
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
// One whose age the provider cannot say is never in it.
func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool,
f handActFlags, now time.Time) error {
listing, err := gatherRetired(ctx, open.inventory, conn, now)
if err != nil {
return err
}
return deleteFrom(ctx, conn, open, listing, days, confirm, f)
}
func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error {
var due []retiredRow
unknown := 0
for _, r := range listing.Retired {
switch {
case r.AgeDays < 0:
unknown++
case r.AgeDays > days:
due = append(due, r)
}
}
for _, u := range listing.Unasked {
fmt.Printf("not asked: %s\n", u)
}
if unknown > 0 {
fmt.Printf("%d retired consumer(s) whose age their provider cannot say are left out\n", unknown)
}
if len(due) == 0 {
fmt.Printf("nothing has been retired more than %d day(s)\n", days)
return nil
}
fmt.Printf("retired more than %d day(s):\n", days)
for _, r := range due {
fmt.Printf(" %s on %s: %s — %d day(s), %s\n", r.Module, r.Node, r.Consumer, r.AgeDays, sizeWords(r.SizeBytes))
}
if !confirm {
fmt.Printf("nothing was deleted: add --confirm to delete these %d\n", len(due))
return nil
}
var failed []string
for _, r := range due {
var err error
if r.Kind == retiredDataKind {
var rec inventory.DataRecord
if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil {
err = deleteRetiredData(ctx, conn, open, rec, f)
}
} else {
err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f)
}
if err != nil {
failed = append(failed, err.Error())
}
}
if len(failed) > 0 {
return fmt.Errorf("%d of %d not deleted: %s", len(failed), len(due), strings.Join(failed, "; "))
}
return nil
}
cmd/mesh-controller/retirement.go
-343
View File
@@ -1,343 +0,0 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
// (novox/hq ADR 0230, the operator's decision of 2026-10-06; it replaces ADR 0229's withdrawal brake).
//
// A provider retires a consumer — disables its access, keeps its data, marks it with when and why —
// once it has seen the same consumers go unasked for in five passes. More than three at once, or more
// than half of those it holds where it holds more than one, is a person actively working on the mesh,
// so the provider retires nothing and waits: the controller keeps that as an urgent condition naming
// what would go and the two verbs that answer it. A retired consumer is deleted only by `cleanup
// delete`, which the provider executes on its own backend — the controller never touches one — and
// one retired more than thirty days is a warning that cleanup is waiting (D11).
// The kinds a provider's retirement word raises.
const (
kindRetireWaiting = "retire-waiting"
kindRetireRejected = "retire-rejected"
kindCleanupWaiting = "cleanup-waiting"
// sourceRetirement is what raised a retirement condition: the provider's own event.
sourceRetirement = "provisioner.retirement"
)
// cleanupAfter is how long a consumer may stay retired before cleanup is said to be waiting (D11).
const cleanupAfter = 30 * 24 * time.Hour
// retirementAsk is how long one provider is given to answer one question about its retired consumers.
var retirementAsk = 20 * time.Second
func retireWaitingKey(module, node string) string {
return conditions.Key(conditions.ScopeProvider, module+"."+node, "retire")
}
func retireRejectedKey(module, node string) string {
return conditions.Key(conditions.ScopeProvider, module+"."+node, "retire-rejected")
}
// retirements keeps what providers say about consumers they retire, as conditions.
type retirements struct {
keeper func() *conditions.Keeper
// record writes an act done by hand that reached a provider some other way than this controller's
// verbs; nil records nothing (a test).
record func(ctx context.Context, act link.HandAct) error
}
func consumerList(cs []link.RetiredConsumer) string {
parts := make([]string, 0, len(cs))
for _, c := range cs {
p := c.Consumer
if c.Node != "" {
p += " (" + c.Node + ")"
}
parts = append(parts, p)
}
return strings.Join(parts, ", ")
}
// waitingObservation is a provider waiting for a person to approve or reject a retirement.
func waitingObservation(r link.Retirement) conditions.Observation {
since := r.Since
if since.IsZero() {
since = r.At
}
summary := fmt.Sprintf("%s on %s would retire %d consumer(s) the mesh no longer asks for — %s — more than its "+
"bound (%s), so it retires nothing until a person answers: `retire approve %s %s --why …` or `retire reject "+
"%s %s --why …`", r.Module, r.ProviderNode, len(r.Consumers), consumerList(r.Consumers), orBound(r.Bound),
r.ProviderNode, r.Module, r.ProviderNode, r.Module)
said := fmt.Sprintf("waiting since %s, %d held: %s", since.UTC().Format("2006-01-02 15:04 MST"), r.Held,
consumerList(r.Consumers))
return conditions.Observation{Scope: conditions.ScopeProvider, ID: r.Module + "." + r.ProviderNode,
Token: "retire", Kind: kindRetireWaiting, Machine: r.ProviderNode, Also: consumerNodes(r),
Severity: conditions.Urgent, Summary: summary, Said: said, Source: sourceRetirement,
Resolver: conditions.ResolverOperator}
}
// rejectedObservation is consumers kept active by a person's rejection though the mesh asks for them no more.
func rejectedObservation(r link.Retirement) conditions.Observation {
summary := fmt.Sprintf("%s on %s keeps %s active although the mesh no longer asks for them: a person rejected "+
"their retirement (%s). Assign them again, or `retire approve %s %s --why …`", r.Module, r.ProviderNode,
consumerList(r.Consumers), orWhy(r.By, r.Why), r.ProviderNode, r.Module)
return conditions.Observation{Scope: conditions.ScopeProvider, ID: r.Module + "." + r.ProviderNode,
Token: "retire-rejected", Kind: kindRetireRejected, Machine: r.ProviderNode, Also: consumerNodes(r),
Severity: conditions.Warning, Summary: summary, Said: "rejected: " + orWhy(r.By, r.Why),
Source: sourceRetirement, Resolver: conditions.ResolverOperator}
}
func orBound(b string) string {
if b == "" {
return "more than 3, or more than half of those held"
}
return b
}
func orWhy(by, why string) string {
switch {
case by != "" && why != "":
return by + ": " + why
case why != "":
return why
case by != "":
return "by " + by
}
return "no reason given"
}
// consumerNodes are the other machines a retirement concerns: where its consumers are.
func consumerNodes(r link.Retirement) []string {
seen := map[string]bool{r.ProviderNode: true}
var out []string
for _, c := range r.Consumers {
if c.Node != "" && !seen[c.Node] {
seen[c.Node] = true
out = append(out, c.Node)
}
}
sort.Strings(out)
return out
}
// Retired keeps one word. Waiting raises the urgent condition; a rejection turns it into a warning; a
// retirement, an approval or the set settling clears both. An approval, a rejection or a deletion that
// did not come through this controller's verbs is recorded in the hand-act log here, so every one is.
func (s retirements) Retired(ctx context.Context, r link.Retirement) error {
k := s.keeper()
if k == nil {
return fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
}
waiting, rejected := retireWaitingKey(r.Module, r.ProviderNode), retireRejectedKey(r.Module, r.ProviderNode)
clear := func(keys ...string) error {
for _, key := range keys {
if _, err := k.Clear(ctx, key, fmt.Sprintf("%s on %s says %s", r.Module, r.ProviderNode, r.Change)); err != nil {
return storeAway(err)
}
}
return nil
}
var err error
switch r.Change {
case link.RetireWaiting:
if _, err = k.Observe(ctx, waitingObservation(r)); err != nil {
return storeAway(err)
}
case link.RetireRejected:
if err = clear(waiting); err != nil {
return err
}
if _, err = k.Observe(ctx, rejectedObservation(r)); err != nil {
return storeAway(err)
}
case link.RetireApproved, link.RetireRetired, link.RetireSettled:
if err = clear(waiting, rejected); err != nil {
return err
}
}
switch r.Change {
case link.RetireApproved, link.RetireRejected, link.RetireDeleted:
if r.Via != link.ViaController && s.record != nil {
verb := map[string]string{link.RetireApproved: "retire approve", link.RetireRejected: "retire reject",
link.RetireDeleted: "cleanup delete"}[r.Change]
cause := kindRetireWaiting
if r.Change == link.RetireDeleted {
cause = kindCleanupWaiting
}
why := r.Why
if strings.TrimSpace(why) == "" {
why = "no reason given to the provider"
}
act := link.HandAct{Verb: verb, Args: append([]string{r.ProviderNode, r.Module}, r.Names()...),
Why: why + " (asked of the provider directly, not through the controller)", Cause: cause,
By: r.By, At: r.At}
if act.By == "" {
act.By = "unknown, asked of " + r.Module + " on " + r.ProviderNode + " directly"
}
if err := s.record(ctx, act); err != nil {
return fmt.Errorf("%s on %s %s by hand, and it could not be recorded: %v: %w", r.Module,
r.ProviderNode, r.Change, err, link.ErrTryAgain)
}
}
}
return nil
}
// recordHandActOnTheBus writes an act through the serving controller's connection.
func recordHandActOnTheBus(ctx context.Context, act link.HandAct) error {
return onTheBus(func(conn *nats.Conn) error {
_, err := link.RecordHandAct(ctx, conn, act)
return err
})
}
// providerInstance is one provider module on one machine.
type providerInstance struct {
Node, Module string
}
// providerInstances are every module assigned on every machine whose manifest receives contributions:
// every provider, which serves the retirement tools (ADR 0230) — or is older than them.
func providerInstances(ctx context.Context, inv *inventory.Inventory) ([]providerInstance, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
}
var out []providerInstance
for _, n := range nodes {
modules, err := inv.Assigned(ctx, n.Name)
if err != nil {
return nil, fmt.Errorf("what %s is assigned cannot be read: %w", n.Name, err)
}
for _, m := range modules {
if man, ok := shelf[m]; ok && len(man.Receives) > 0 {
out = append(out, providerInstance{Node: n.Name, Module: m})
}
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Node != out[j].Node {
return out[i].Node < out[j].Node
}
return out[i].Module < out[j].Module
})
return out, nil
}
// askRetirement asks one provider for what it holds retired and what waits.
func askRetirement(ctx context.Context, conn *nats.Conn, p providerInstance) (link.RetirementState, error) {
var state link.RetirementState
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetirement, p.Node, map[string]any{}, retirementAsk)
if err != nil {
return state, err
}
if answer.Error != "" {
return state, fmt.Errorf("%s on %s answered %s with an error: %s", p.Module, p.Node, link.ToolRetirement, answer.Error)
}
if err := unmarshalAnswer(answer, &state); err != nil {
return state, fmt.Errorf("%s on %s answered %s with something unreadable: %w", p.Module, p.Node, link.ToolRetirement, err)
}
return state, nil
}
// retiredAt reads a retired consumer's moment; zero when the provider could not say.
func retiredAt(c link.RetiredConsumer) time.Time {
t, err := time.Parse(time.RFC3339, c.RetiredAt)
if err != nil {
return time.Time{}
}
return t
}
// cleanupObservation is a provider holding consumers retired longer than cleanupAfter.
func cleanupObservation(p providerInstance, old []link.RetiredConsumer, now time.Time) conditions.Observation {
parts := make([]string, 0, len(old))
for _, c := range old {
parts = append(parts, fmt.Sprintf("%s (%d days, %s)", c.Consumer, int(now.Sub(retiredAt(c)).Hours()/24),
sizeWords(c.SizeBytes)))
}
return conditions.Observation{Scope: conditions.ScopeProvider, ID: p.Module + "." + p.Node, Token: "cleanup",
Kind: kindCleanupWaiting, Machine: p.Node, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s on %s holds %d consumer(s) retired more than %d days, waiting for a person to "+
"delete or bring them back: %s — `cleanup list`, then `cleanup delete %s %s <consumer> --why …`",
p.Module, p.Node, len(old), int(cleanupAfter.Hours()/24), strings.Join(parts, ", "), p.Node, p.Module),
Resolver: conditions.ResolverOperator}
}
func sizeWords(size *int64) string {
if size == nil || *size < 0 {
return "size unknown"
}
b := float64(*size)
for _, unit := range []string{"B", "KB", "MB", "GB"} {
if b < 1024 || unit == "GB" {
if unit == "B" {
return fmt.Sprintf("%d B", *size)
}
return fmt.Sprintf("%.1f %s", b, unit)
}
b /= 1024
}
return ""
}
// probeRetired is D11: no provider holds a consumer retired more than thirty days. Every provider
// assigned is asked what it holds retired; one that does not serve the question — an older build, a
// TypeScript provider not yet on the SDK that retires — has nothing it can say and is passed over,
// which is not a failure of the probe. A provider that cannot be asked otherwise is.
func probeRetired(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
if d.js == nil {
return nil, fmt.Errorf("no bus to ask the providers over")
}
instances, err := providerInstances(ctx, d.open.inventory)
if err != nil {
return nil, err
}
return retiredTooLong(ctx, d.js.Conn(), instances, time.Now())
}
// retiredTooLong asks each provider and answers one observation per provider holding anything retired
// longer than cleanupAfter.
func retiredTooLong(ctx context.Context, conn *nats.Conn, instances []providerInstance,
now time.Time) ([]conditions.Observation, error) {
var out []conditions.Observation
var problems []string
for _, p := range instances {
state, err := askRetirement(ctx, conn, p)
if err != nil {
if isNothingServes(err) {
continue
}
problems = append(problems, err.Error())
continue
}
var old []link.RetiredConsumer
for _, c := range state.Retired {
if at := retiredAt(c); !at.IsZero() && now.Sub(at) > cleanupAfter {
old = append(old, c)
}
}
if len(old) > 0 {
out = append(out, cleanupObservation(p, old, now))
}
}
if len(problems) > 0 {
// Not "nothing retired": a provider that could not be asked may hold the oldest of all.
return nil, fmt.Errorf("%s", strings.Join(problems, "; "))
}
return out, nil
}
cmd/mesh-controller/retirement_test.go
-481
View File
@@ -1,481 +0,0 @@
package main
import (
"context"
"encoding/json"
"flag"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
// (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that
// answer it ask the provider — never anything else — for exactly what it said.
func waitingWord(module, node string, names ...string) link.Retirement {
r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting,
Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()}
for _, n := range names {
r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"})
}
return r
}
func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition {
t.Helper()
all, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range all {
out[c.Key] = c
}
return out
}
func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) {
k, _ := withConditionsInMemory(t)
var recorded []link.HandAct
r := retirements{keeper: func() *conditions.Keeper { return k },
record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }}
ctx := t.Context()
waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d")
if err := r.Retired(ctx, waiting); err != nil {
t.Fatal(err)
}
open := openKeys(t, k)
c, ok := open["provider.postgres.anchor.retire"]
if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent {
t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open)
}
for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres",
"retire reject anchor postgres", "more than 3"} {
if !strings.Contains(c.Summary, want) {
t.Errorf("the condition does not say %q: %s", want, c.Summary)
}
}
// Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here —
// the verb recorded it before it asked.
rejected := waiting
rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController
if err := r.Retired(ctx, rejected); err != nil {
t.Fatal(err)
}
open = openKeys(t, k)
if _, still := open["provider.postgres.anchor.retire"]; still {
t.Fatal("a rejection left the provider waiting")
}
if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning ||
c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") {
t.Fatalf("a rejection is not a warning saying why: %+v", open)
}
if len(recorded) != 0 {
t.Fatalf("an act through the controller was recorded twice: %+v", recorded)
}
// Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here.
approved := waiting
approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", ""
if err := r.Retired(ctx, approved); err != nil {
t.Fatal(err)
}
if open := openKeys(t, k); len(open) != 0 {
t.Fatalf("an approval left conditions open: %+v", open)
}
if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" ||
!strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") {
t.Fatalf("an approval outside the controller was not recorded: %+v", recorded)
}
// Waiting again, then the set settles (asked for again): cleared. And retired clears too.
for _, change := range []string{link.RetireSettled, link.RetireRetired} {
if err := r.Retired(ctx, waiting); err != nil {
t.Fatal(err)
}
done := waiting
done.Change = change
if err := r.Retired(ctx, done); err != nil {
t.Fatal(err)
}
if open := openKeys(t, k); len(open) != 0 {
t.Fatalf("%s left conditions open: %+v", change, open)
}
}
// A deletion asked of the provider directly is recorded too.
deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x",
Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}}
if err := r.Retired(ctx, deleted); err != nil {
t.Fatal(err)
}
if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting {
t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded)
}
}
func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) {
body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting",
"held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"})
r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body)
if err != nil || r.Module != "idp" || len(r.Consumers) != 2 {
t.Fatalf("%+v %v", r, err)
}
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement",
[]byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil {
t.Fatal("a change the mesh has no name for was read")
}
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil {
t.Fatal("a failing word was read as a retirement")
}
k, _ := withConditionsInMemory(t)
if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil {
t.Fatal(err)
}
if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok {
t.Fatal("not kept under the emitter the subject names")
}
}
func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "pg", Version: "1",
Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}})
if _, err := assign(ctx, open, "anchor", "pg"); err != nil {
t.Fatal(err)
}
r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }}
for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} {
if err := r.Retired(ctx, w); err != nil {
t.Fatal(err)
}
}
if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{
cleanupObservation(providerInstance{Node: "anchor", Module: "gone"},
[]link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()),
}); err != nil {
t.Fatal(err)
}
all, _ := conditionsFrom.Open(ctx)
if len(all) != 3 {
t.Fatalf("%+v", all)
}
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil {
t.Fatal(err)
}
left := openKeys(t, conditionsFrom)
if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" {
t.Fatalf("only the assigned provider's waiting should stay: %+v", left)
}
}
// fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked.
type fakeProvider struct {
mu sync.Mutex
state link.RetirementState
asked []map[string]any
deleted []string
approved []string
rejected []string
}
func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) {
t.Helper()
answer := func(m *nats.Msg, result any, refusal string) {
body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node})
_ = m.Respond(body)
}
names := func(args map[string]any) []string {
var out []string
for _, v := range args["consumers"].([]any) {
out = append(out, v.(string))
}
sort.Strings(out)
return out
}
set := func(cs []link.RetiredConsumer) []string {
var out []string
for _, c := range cs {
out = append(out, c.Consumer)
}
sort.Strings(out)
return out
}
for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} {
tool := tool
sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) {
f.mu.Lock()
defer f.mu.Unlock()
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
f.asked = append(f.asked, map[string]any{"tool": tool, "args": args})
switch tool {
case link.ToolRetirement:
answer(m, f.state, "")
case link.ToolRetireApprove:
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
answer(m, nil, "not the set I wait with")
return
}
f.approved = names(args)
answer(m, map[string]any{"retired": f.approved}, "")
case link.ToolRetireReject:
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
answer(m, nil, "not the set I wait with")
return
}
f.rejected = names(args)
answer(m, map[string]any{"kept": f.rejected}, "")
case link.ToolRetiredDelete:
if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController {
answer(m, nil, "confirm, why and via")
return
}
f.deleted = append(f.deleted, args["consumer"].(string))
answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "")
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
}
func onATestBus(t *testing.T) *nats.Conn {
t.Helper()
url := testbus.URL(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
return js.Conn()
}
func whyFlags(t *testing.T, why string) handActFlags {
t.Helper()
set := flag.NewFlagSet("t", flag.ContinueOnError)
f := addHandActFlags(set)
if err := set.Parse([]string{"--why", why}); err != nil {
t.Fatal(err)
}
return f
}
func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct {
t.Helper()
acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute))
if err != nil {
t.Fatal(err)
}
var out []link.HandAct
for _, a := range acts {
if a.Verb == verb {
out = append(out, a)
}
}
return out
}
func retiredDaysAgo(name string, days int) link.RetiredConsumer {
size := int64(4096)
return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it",
RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size}
}
func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6}
fake.serve(t, conn, "pg-approve", "anchor")
p := providerInstance{Node: "anchor", Module: "pg-approve"}
said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) })
if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") {
t.Fatalf("approved %v; said %s", fake.approved, said)
}
acts := handActsBy(t, conn, "retire approve")
if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting ||
acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" {
t.Fatalf("the approval is not in the hand-act log: %+v", acts)
}
// Reject, on another provider: the same set, and nothing approved.
other := &fakeProvider{state: fake.state}
other.serve(t, conn, "pg-reject", "anchor")
printed(t, func() error {
return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false,
whyFlags(t, "still moving"))
})
if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil {
t.Fatalf("rejected %v approved %v", other.rejected, other.approved)
}
// Nothing waiting: refused, nothing asked but the question.
idle := &fakeProvider{}
idle.serve(t, conn, "pg-idle", "anchor")
if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true,
whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") {
t.Fatalf("%v", err)
}
if len(idle.asked) != 1 {
t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked)
}
}
func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
fake.state.Held = []string{"active"}
fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)}
fake.serve(t, conn, "pg-clean", "anchor")
p := providerInstance{Node: "anchor", Module: "pg-clean"}
said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) })
if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") {
t.Fatalf("deleted %v; said %s", fake.deleted, said)
}
// An active consumer, or one it does not hold, is refused before the provider is asked to delete.
for _, name := range []string{"active", "nobody"} {
if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil ||
!strings.Contains(err.Error(), "only a retired consumer is deleted") {
t.Fatalf("%s: %v", name, err)
}
}
if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("more was deleted: %v", fake.deleted)
}
if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" {
t.Fatalf("the deletion is not in the hand-act log: %+v", acts)
}
// Older than: listed, and nothing deleted without confirm; with it, only the old one.
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 {
t.Fatalf("%+v", listing)
}
fake.deleted = nil
said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
strings.Contains(said, "young") {
t.Fatalf("deleted %v; said %s", fake.deleted, said)
}
printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("confirmed, deleted %v", fake.deleted)
}
}
func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) {
conn := onATestBus(t)
old := &fakeProvider{}
old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)}
old.serve(t, conn, "pg-d11-old", "anchor")
young := &fakeProvider{}
young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)}
young.serve(t, conn, "pg-d11-young", "anchor")
instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"},
// Nothing serves this one: a provider older than the question is passed over, not a failure.
{Node: "anchor", Module: "pg-d11-predates"}}
found, err := retiredTooLong(t.Context(), conn, instances, time.Now())
if err != nil {
t.Fatal(err)
}
if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting ||
found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") ||
strings.Contains(found[0].Summary, "fresh") {
t.Fatalf("%+v", found)
}
}
func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want string
}{
{"retire", map[string]any{}, "retire --json"},
{"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"},
"retire approve anchor postgres --why moved"},
{"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"},
"retire reject anchor postgres --why no"},
{"cleanup", map[string]any{}, "cleanup list --json"},
{"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"},
"cleanup delete anchor postgres x --why gone"},
{"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"},
{"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"},
"cleanup delete --older-than 30 --why tidy --confirm"},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
}
}
for _, args := range []map[string]any{
{"answer": "approve", "node": "anchor", "module": "postgres"}, // no why
{"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"},
} {
if _, err := argvFor("retire", args); err == nil {
t.Errorf("retire %v was composed", args)
}
}
for _, args := range []map[string]any{
{"node": "anchor", "module": "postgres", "consumer": "x"}, // no why
{"older-than": "30"},
{"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"},
} {
if _, err := argvFor("cleanup", args); err == nil {
t.Errorf("cleanup %v was composed", args)
}
}
if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" ||
repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" {
t.Error("the generic verb would let a retirement or a deletion through without a why")
}
}
// A mark-only provider's retired consumer keeps its access; the listing and the approval say so
// rather than claiming it was disabled (ADR 0230).
func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
kept := retiredDaysAgo("ledger", 3)
kept.Access = "kept"
fake.state.Retired = []link.RetiredConsumer{kept}
fake.state.RetiresBy = "mark-only"
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2}
fake.serve(t, conn, "vault-mark", "anchor")
p := providerInstance{Node: "anchor", Module: "vault-mark"}
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
said := printed(t, func() error { return printRetired(listing, false) })
if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" {
t.Fatalf("%s %+v", said, listing)
}
said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) })
if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") {
t.Fatal(said)
}
}
cmd/mesh-controller/rollout.go
+9 -2
View File
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
// would bury the ones that matter under a list nobody can act on.
func speaksOnTheBus(m catalogue.Manifest) bool {
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
}
@@ -388,7 +388,14 @@ func rolloutMint(ctx context.Context, again bool) error {
}
// providesBus is whether a manifest provides the mesh's bus.
func providesBus(m catalogue.Manifest) bool { return catalogue.ProvidesBus(m) }
func providesBus(m catalogue.Manifest) bool {
for _, o := range m.Provides {
if o.Name == "mesh-bus" {
return true
}
}
return false
}
// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
cmd/mesh-controller/rotate.go
+1 -34
View File
@@ -6,8 +6,6 @@ import (
"flag"
"fmt"
"sort"
"github.com/novox/mesh-controller/internal/inventory"
)
// rotateCommand replaces a credential and moves both ends together.
@@ -35,16 +33,12 @@ func rotateCommand(ctx context.Context, args []string) error {
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
// and rotating the other nine would be a great deal of disruption for one suspicion.
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
// One consuming module rather than every module on the machine. A machine runs many consumers
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
// issue 268) is no reason to restart every other one on the machine.
module := set.String("module", "", "only this consuming module's credential")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
return errors.New("rotate <provision> [--consumer <machine>]")
}
provision := positionals[0]
@@ -59,12 +53,6 @@ func rotateCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
holders = ofModule(holders, *module)
if len(holders) == 0 && *module != "" {
return fmt.Errorf(
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
"says what a machine holds", *module, onMachine(*only), provision)
}
if len(holders) == 0 {
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
// and a rotation somebody believes happened is worse than one they know did not.
@@ -128,27 +116,6 @@ func rotateCommand(ctx context.Context, args []string) error {
return nil
}
// ofModule is the holders whose consuming module is this one; all of them when none is named.
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
if module == "" {
return holders
}
var out []inventory.Holder
for _, h := range holders {
if h.ConsumerModule == module {
out = append(out, h)
}
}
return out
}
func onMachine(machine string) string {
if machine == "" {
return ""
}
return " on " + machine
}
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
func asLocal(local string) string {
if local == "" {
cmd/mesh-controller/rotate_test.go
-27
View File
@@ -1,27 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
// 268): a module that leaked its database password is no reason to restart every other consumer.
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
holders := []inventory.Holder{
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
}
got := ofModule(holders, "letta")
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
t.Fatalf("narrowed to letta: %+v", got)
}
if len(ofModule(holders, "")) != 3 {
t.Fatal("no module named narrowed anyway")
}
if len(ofModule(holders, "absent")) != 0 {
t.Fatal("a module holding nothing matched")
}
}
cmd/mesh-controller/same_source_test.go
-209
View File
@@ -1,209 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A build whose source is unchanged is never a move (novox/hq issue 280): an image is not
// byte-reproducible, so two builds of one source make two digests, and the rule that a rebuild with
// identical artifacts is no move (ADR 0236) never held for one.
// anImageBuild is a build outcome of an image module: its manifest names the image by the digest made.
func anImageBuild(t *testing.T, module, id, commit, digest, source string, asked time.Time) link.BuildResult {
t.Helper()
image := "registry.invalid:5000/" + module + "/server@sha256:" + digest
manifest, _ := json.Marshal(map[string]any{"module": module, "version": "1",
"resources": []any{map[string]any{"id": "svc", "type": "container", "name": module, "image": image}}})
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module,
On: "anchor", Module: module, Commit: commit, Manifest: manifest, SourceFingerprint: source,
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: image}},
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
}
// shelfNames is whether the module the mesh holds names this digest.
func shelfNames(t *testing.T, inv *inventory.Inventory, module, digest string) bool {
t.Helper()
shelf, err := inv.Catalogue(t.Context())
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(shelf[module])
return strings.Contains(string(raw), digest)
}
// A rebuild of an unchanged source is registered with the artifacts of the build it was first made
// as: no machine is sent a new digest, a module standing on it is handed the same base, and the two
// builds are one to every rule that asks whether a send moves something. A real source change moves.
func TestARebuildOfAnUnchangedSourceKeepsItsArtifacts(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
for i, b := range []link.BuildResult{
anImageBuild(t, "app", "", "c1aaaaaa", strings.Repeat("a", 64), "src1:same", start),
// Another module's merge rebuilt it: a new commit, a new image digest, the same source.
anImageBuild(t, "app", "", "c2bbbbbb", strings.Repeat("b", 64), "src1:same", start.Add(time.Minute)),
} {
if _, _, err := takeIn(ctx, inv, b); err != nil {
t.Fatalf("build %d: %v", i, err)
}
}
if !shelfNames(t, inv, "app", strings.Repeat("a", 64)) || shelfNames(t, inv, "app", strings.Repeat("b", 64)) {
t.Fatal("a rebuild of an unchanged source registered the digest it made, not the one the mesh holds")
}
if src, err := inv.SourceOf(ctx, "app"); err != nil || src.BuiltFrom != "c2bbbbbb" {
t.Fatalf("the module is not at the commit it was rebuilt from: %+v %v", src, err)
}
held, err := inv.Held(ctx)
if err != nil || !strings.HasSuffix(held["app/server"], strings.Repeat("a", 64)) {
t.Fatalf("a module standing on it is handed %q, not the build the mesh holds (%v)", held["app/server"], err)
}
f, err := readMoveFacts(ctx, inv)
if err != nil {
t.Fatal(err)
}
if !f.identical("app", "c1aaaaaa", "c2bbbbbb") {
t.Fatal("two builds of one source are not one build")
}
// A real change to the source moves: its own digest registered, and not identical.
if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c3cccccc", strings.Repeat("c", 64), "src1:changed",
start.Add(2*time.Minute))); err != nil {
t.Fatal(err)
}
if !shelfNames(t, inv, "app", strings.Repeat("c", 64)) {
t.Fatal("a changed source did not register what it made")
}
if f, _ = readMoveFacts(ctx, inv); f.identical("app", "c2bbbbbb", "c3cccccc") {
t.Fatal("a changed source is read as the same build")
}
// And a builder that says no fingerprint is told apart by its artifacts alone, as before.
if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c4dddddd", strings.Repeat("d", 64), "",
start.Add(3*time.Minute))); err != nil {
t.Fatal(err)
}
if !shelfNames(t, inv, "app", strings.Repeat("d", 64)) {
t.Fatal("a build with no fingerprint did not register what it made")
}
}
// The bus rebuilt for another module's merge, its source untouched and its image digest new: no move —
// no push to its machine is held, no bus step is demanded, and `bus upgrade` has nothing to do. A real
// change to the bus's source is the planned step again.
func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
bus := func(id, commit, digest, source string, at time.Time) link.BuildResult {
b := anImageBuild(t, "nats", id, commit, digest, source, at)
manifest, _ := json.Marshal(map[string]any{"module": "nats", "version": "2",
"provides": []any{map[string]any{"name": "mesh-bus"}},
"resources": []any{map[string]any{"id": "svc", "type": "container", "name": "nats",
"image": b.Made[0].Reference}}})
b.Manifest = manifest
return b
}
if _, _, err := takeIn(ctx, inv, bus("", "n1111111", strings.Repeat("a", 64), "src1:bus", start)); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1111111"}); err != nil {
t.Fatal(err)
}
// The wide rebuild: a new commit, a new image digest, the same source.
if _, _, err := takeIn(ctx, inv, bus("", "n2222222", strings.Repeat("b", 64), "src1:bus", start.Add(time.Minute))); err != nil {
t.Fatal(err)
}
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
t.Fatalf("a bus rebuilt from an unchanged source held its machine: %v %v", held, err)
}
wasSnapshot := takeBusSnapshot
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
takeBusSnapshot = func(context.Context, string, string) (string, error) {
return "", errors.New("no snapshot is taken for a bus step nobody needs")
}
var sent [][]string
wasSend := sendRollout
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
sent = append(sent, names)
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
if err := busCommand(ctx, []string{"upgrade", "--why", "nothing changed", "--reversible"}); err != nil || len(sent) != 0 {
t.Fatalf("a bus step ran for a bus whose source is unchanged: %v, sent %v", err, sent)
}
if !shelfNames(t, inv, "nats", strings.Repeat("a", 64)) {
t.Fatal("the bus the mesh holds is not the image its machine runs")
}
// A real change to the bus's source: the planned step.
if _, _, err := takeIn(ctx, inv, bus("", "n3333333", strings.Repeat("c", 64), "src1:bus-2.12", start.Add(2*time.Minute))); err != nil {
t.Fatal(err)
}
held, err := busHeld(ctx, inv, []string{"anchor"})
if err != nil || !strings.Contains(held["anchor"], "planned step") {
t.Fatalf("a changed bus did not demand its planned step: %v %v", held, err)
}
}
// A plan's build made from the source every machine running the module runs is not sent and not
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
for _, tc := range []struct {
name string
source string
sent [][]string
}{
{"unchanged", "src1:app", nil},
{"changed", "src1:app-changed", [][]string{{"anchor"}}},
} {
t.Run(tc.name, func(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// What anchor and laptop run (c1) was made from src1:app; the plan's build of c2 from the source
// the case says.
for _, b := range []inventory.Build{
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: tc.source,
Asked: time.Now(), At: time.Now()},
} {
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
p := g.plan(t)
p.Modules["app"].Build = "build-2s"
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
if !reflect.DeepEqual(g.sent, tc.sent) {
t.Fatalf("sent %v, want %v", g.sent, tc.sent)
}
p = g.plan(t)
s := p.Modules["app"]
if tc.sent == nil && (s.SentAt == nil || s.Gate != nil || !strings.Contains(s.Why, "no move")) {
t.Fatalf("an unchanged source was not read as no move: %+v", s)
}
if tc.sent != nil && (s.Gate == nil || len(s.First) == 0) {
t.Fatalf("a changed source was not sent to its first machine, gated: %+v", s)
}
})
}
}
cmd/mesh-controller/sayable_test.go
-287
View File
@@ -1,287 +0,0 @@
package main
import (
"context"
"fmt"
"net"
"os"
"sort"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"golang.org/x/net/dns/dnsmessage"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/outward"
)
// **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277,
// ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row
// suppressed past its bound, every probe's finding, every event's condition — is held to the operator
// channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's
// shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
// this is where the producer is made to say it rightly in the first place.
func TestMain(m *testing.M) {
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
unsaid.note(o, field, r)
}
code := m.Run()
if said := unsaid.all(); len(said) > 0 {
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+
"(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n",
len(said), strings.Join(said, "\n "))
if code == 0 {
code = 1
}
}
os.Exit(code)
}
// unsaid collects, across the suite, every finding whose words the operator's channel would withhold.
var unsaid unsayable
type unsayable struct {
mu sync.Mutex
seen map[string]bool
}
func (u *unsayable) note(o conditions.Observation, field string, r outward.Refusal) {
text := o.Summary
if field == "key" {
text = o.Key()
}
u.mu.Lock()
defer u.mu.Unlock()
if u.seen == nil {
u.seen = map[string]bool{}
}
u.seen[fmt.Sprintf("%s (source %s, kind %s): its %s carries %s — %q", o.Key(), o.Source, o.Kind, field, r.What,
text)] = true
}
func (u *unsayable) all() []string {
u.mu.Lock()
defer u.mu.Unlock()
var out []string
for s := range u.seen {
out = append(out, s)
}
sort.Strings(out)
return out
}
// linted is a producer's findings, held to the content rule as a keeper would hold them: for a test
// that reads a producer's findings without raising them.
func linted(obs []conditions.Observation) []conditions.Observation {
for _, o := range obs {
machines := append([]string{o.Machine}, o.Also...)
if r, ok := outward.Check(o.Key(), machines...); !ok {
unsaid.note(o, "key", r)
}
if r, ok := outward.Check(o.Summary, machines...); !ok {
unsaid.note(o, "summary", r)
}
}
return obs
}
// resolverStandIn is a resolver on loopback that answers as answer says; nil answers nothing.
func resolverStandIn(t *testing.T, answer func(q dnsmessage.Question, n int64) *dnsmessage.Message) (port string, asked *atomic.Int64) {
t.Helper()
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = conn.Close() })
asked = &atomic.Int64{}
go func() {
buf := make([]byte, 1500)
for {
n, from, err := conn.ReadFrom(buf)
if err != nil {
return
}
var q dnsmessage.Message
if q.Unpack(buf[:n]) != nil || len(q.Questions) == 0 {
continue
}
reply := answer(q.Questions[0], asked.Add(1))
if reply == nil {
continue
}
reply.Header.ID, reply.Header.Response, reply.Questions = q.ID, true, q.Questions
packed, _ := reply.Pack()
_, _ = conn.WriteTo(packed, from)
}
}()
_, port, _ = net.SplitHostPort(conn.LocalAddr().String())
return port, asked
}
// answersRightly answers A with the address the test's machine has, and NODATA for AAAA.
func answersRightly(q dnsmessage.Question) *dnsmessage.Message {
reply := &dnsmessage.Message{}
if q.Type == dnsmessage.TypeA {
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Name, Type: dnsmessage.TypeA,
Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
}
return reply
}
// quickResolvers makes D2's patience short for a test.
func quickResolvers(t *testing.T, port string) {
t.Helper()
before, within, pause := resolverPort, resolverWithin, resolverPause
resolverPort, resolverWithin, resolverPause = port, 150*time.Millisecond, 10*time.Millisecond
t.Cleanup(func() { resolverPort, resolverWithin, resolverPause = before, within, pause })
}
var anchorOnTheNetwork = []inventory.Overlay{{Name: "anchor", Address: "10.77.0.1"}}
// **D2 asks again before it says anything** (novox/hq issue 277): a resolver that misses a question on
// a loaded machine and answers the next try is a resolver that answers.
func TestAResolverThatMissesOneTryAndAnswersTheNextIsWell(t *testing.T) {
port, asked := resolverStandIn(t, func(q dnsmessage.Question, n int64) *dnsmessage.Message {
if n <= 2 {
return nil // the first question of each kind lost, as under a push and a build starting
}
return answersRightly(q)
})
quickResolvers(t, port)
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
if len(got) != 0 {
t.Fatalf("a resolver that answered its second try was said: %+v", got)
}
if asked.Load() < 3 {
t.Fatalf("asked %d times", asked.Load())
}
}
// **A resolver that answers nothing is held for the next run, and said in machine names**: the address
// it was asked at and the socket's words are the evidence, never the summary the operator reads.
func TestAResolverThatAnswersNothingIsHeldAndSaidInMachineNames(t *testing.T) {
port, asked := resolverStandIn(t, func(dnsmessage.Question, int64) *dnsmessage.Message { return nil })
quickResolvers(t, port)
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
if len(got) != 1 {
t.Fatalf("%+v", got)
}
o := got[0]
if !o.Confirm || o.Key() != "seat.mesh-dns-resolver.anchor.wrong" || o.Severity != conditions.Urgent {
t.Fatalf("not held for a second look, or not the resolver's condition: %+v", o)
}
if r, ok := outward.Check(o.Summary, "anchor"); !ok {
t.Fatalf("the summary carries %s: %q", r, o.Summary)
}
if !strings.Contains(o.Summary, "anchor's") || !strings.Contains(o.Said, "127.0.0.1") ||
!strings.Contains(o.Said, "anchor.internal") {
t.Fatalf("summary %q, evidence %q", o.Summary, o.Said)
}
if want := int64(2 * resolverTries); asked.Load() != want {
t.Fatalf("asked %d times, want %d: each question %d times", asked.Load(), want, resolverTries)
}
}
// **A resolver that answers wrongly is said at once**, in machine names: an answer is not the absence
// of one (issue 262's NXDOMAIN for IPv6).
func TestAResolverAnsweringWronglyIsSaidAtOnceInMachineNames(t *testing.T) {
port, _ := resolverStandIn(t, func(q dnsmessage.Question, _ int64) *dnsmessage.Message {
if q.Type == dnsmessage.TypeAAAA {
return &dnsmessage.Message{Header: dnsmessage.Header{RCode: dnsmessage.RCodeNameError}}
}
return answersRightly(q)
})
quickResolvers(t, port)
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
if len(got) != 1 || got[0].Confirm {
t.Fatalf("%+v", got)
}
if r, ok := outward.Check(got[0].Summary, "anchor"); !ok || !strings.Contains(got[0].Summary, "NXDOMAIN") ||
!strings.Contains(got[0].Summary, "anchor's IPv6 address") {
t.Fatalf("summary %q (%v)", got[0].Summary, r)
}
}
// **The self-check raises a finding one look can be wrong about on the second run in a row**, keeps it
// open while it is seen, and clears it when it is not — and a probe that cannot run is said as a
// condition only when the next run cannot run it either. Neither is ever a pass in the verdict.
func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) {
var unanswered, broken atomic.Bool
held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong",
Machine: "anchor", Severity: conditions.Urgent, Confirm: true,
Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"}
withProbes(t,
probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
if unanswered.Load() {
return []conditions.Observation{held}, nil
}
return nil, nil
}},
probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1,
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
if broken.Load() {
return nil, fmt.Errorf("a question timed out")
}
return nil, nil
}},
)
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
d := &doctor{keeper: k, host: "anchor"}
keys := func() []string {
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
var out []string
for _, c := range open {
out = append(out, c.Key)
}
sort.Strings(out)
return out
}
unanswered.Store(true)
broken.Store(true)
run := d.runOnce(t.Context(), "a test")
if open := keys(); len(open) != 0 {
t.Fatalf("one look raised %v", open)
}
if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun {
t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes)
}
// Seen twice in a row: raised, and so is the probe that could not run twice.
run = d.runOnce(t.Context(), "a test")
if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" {
t.Fatalf("two looks in a row left open %v", open)
}
if run.Probes[0].Verdict != verdictFail {
t.Fatalf("%+v", run.Probes[0])
}
// Open, and seen again: kept, never cleared and raised again.
d.runOnce(t.Context(), "a test")
if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 {
t.Fatalf("the open condition was not kept as it was: %+v", c)
}
// Answered again: cleared. Then one look alone raises nothing again.
unanswered.Store(false)
broken.Store(false)
d.runOnce(t.Context(), "a test")
if open := keys(); len(open) != 0 {
t.Fatalf("a passing run left open %v", open)
}
unanswered.Store(true)
d.runOnce(t.Context(), "a test")
if open := keys(); len(open) != 0 {
t.Fatalf("one look after a pass raised %v", open)
}
}
cmd/mesh-controller/seats.go
+9 -35
View File
@@ -29,13 +29,11 @@ type seatHolder struct {
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
type seatRow struct {
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
Replicated bool `json:"replicated,omitempty"`
Holders []seatHolder `json:"holders"`
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
Holders []seatHolder `json:"holders"`
}
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -49,7 +47,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
rows := make([]seatRow, 0, len(seats))
for _, s := range seats {
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
Replicated: s.Replicated, Holders: []seatHolder{}}
Holders: []seatHolder{}}
seen := map[seatHolder]bool{}
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -106,13 +104,9 @@ func seatCommand(ctx context.Context, args []string) error {
return nil
}
if len(args) == 3 && args[1] == "--to" {
return handOver(ctx, args[0], args[2], false)
return handOver(ctx, args[0], args[2])
}
if len(args) == 3 && args[1] == "--add" {
return handOver(ctx, args[0], args[2], true)
}
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
"seat <name> --add <node>/<module>")
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
}
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -125,12 +119,7 @@ func seatCommand(ctx context.Context, args []string) error {
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
// and refusing to record a handover to a module the node has not started would make the handover
// impossible to do before the switch instead of as the switch.
//
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
// records the named assignment as a further holder and leaves every holder on record as it is. A
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
// the one named, as it always did.
func handOver(ctx context.Context, seatName, to string, adding bool) error {
func handOver(ctx context.Context, seatName, to string) error {
nodeName, module, ok := strings.Cut(to, "/")
if !ok || nodeName == "" || module == "" {
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -146,10 +135,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
if !known {
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
}
if adding && !seat.Replicated {
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
}
assigned, err := inv.Assigned(ctx, nodeName)
if err != nil {
return err
@@ -172,7 +157,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
}
var was string
var held []string
holdings, err := inv.Holdings(ctx)
if err != nil {
return err
@@ -180,7 +164,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
for _, h := range holdings {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
was = h.Node
held = append(held, h.Node)
}
}
@@ -204,15 +187,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
} else if err := catalogue.CanHold(*m, seat); err != nil {
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
}
if adding {
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
strings.Join(held, ", "))
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
return nil
}
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
cmd/mesh-controller/seatverbs.go
+41 -646
View File
@@ -9,11 +9,9 @@ import (
"github.com/nats-io/nats.go/micro"
"os"
"os/exec"
"slices"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
@@ -38,194 +36,19 @@ type verbAnswer struct {
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
//
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
// the verb declares but did not use for the command line it composed — given beside another that
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
// not take that" would have stopped it before anything was sent.
func argvFor(verb string, args map[string]any) ([]string, error) {
a, err := readArguments(verb, args)
if err != nil {
return nil, err
str := func(key string) string {
v, _ := args[key].(string)
return strings.TrimSpace(v)
}
argv, err := a.commandLine()
if len(a.misread) > 0 {
// The table and the command line disagree: the verb reads an argument no caller can see
// in its schema, so no caller could ever pass it.
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
"table and its command lines disagree", verb, quoteAll(a.misread))
}
if err != nil {
return nil, err
}
if unused := a.unused(); len(unused) > 0 {
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
}
return argv, nil
}
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
// command line was composed from.
type verbArguments struct {
verb string
given map[string]string
used map[string]bool
declared map[string]bool
misread []string // arguments the command line read that the schema does not declare: a bug here
}
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
// wrote.
func controllerVerb(name string) (catalogue.Verb, bool) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == name {
return v, true
}
}
return catalogue.Verb{}, false
}
// declaredArguments are a schema's properties, and which of them are switches.
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
switches = map[string]bool{}
props, _ := v.Input["properties"].(map[string]any)
for name, p := range props {
names = append(names, name)
desc, _ := p.(map[string]any)
switch enum := desc["enum"].(type) {
case []string:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
case []any:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
}
}
sort.Strings(names)
return names, switches
}
// readArguments refuses what the verb does not take, before anything is composed.
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
v, known := controllerVerb(verb)
if !known {
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
names, switches := declaredArguments(v)
declared := map[string]bool{}
for _, n := range names {
declared[n] = true
}
takes := "none"
if len(names) > 0 {
takes = quoteAll(names)
}
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
keys := make([]string, 0, len(args))
for k := range args {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if !declared[k] {
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
}
var value string
switch x := args[k].(type) {
case nil:
continue
case string:
value = strings.TrimSpace(x)
case bool:
if !switches[k] {
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
}
value = fmt.Sprint(x)
default:
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
}
if switches[k] {
switch value {
case "true":
case "false", "":
continue // said and off: the same as not given, and nothing passed over
default:
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
need := func(keys ...string) error {
for _, k := range keys {
if str(k) == "" {
return fmt.Errorf("%s needs %q", verb, k)
}
}
if value != "" {
a.given[k] = value
}
return nil
}
return a, nil
}
// str is one argument's value, marked as used.
func (a *verbArguments) str(key string) string {
if !a.declared[key] {
a.misread = append(a.misread, key)
}
a.used[key] = true
return a.given[key]
}
// on is a switch, marked as used.
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
// need refuses a call missing a required argument, in the verb's own words.
func (a *verbArguments) need(keys ...string) error {
for _, k := range keys {
if a.str(k) == "" {
return fmt.Errorf("%s needs %q", a.verb, k)
}
}
return nil
}
// unused are the arguments given that the command line was not composed from.
func (a *verbArguments) unused() []string {
var out []string
for k := range a.given {
if !a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
return out
}
func (a *verbArguments) usedGiven() []string {
var out []string
for k := range a.given {
if a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
if len(out) == 0 {
return []string{"nothing"}
}
return out
}
func quoteAll(xs []string) string {
q := make([]string, len(xs))
for i, x := range xs {
if x == "nothing" {
q[i] = x
continue
}
q[i] = fmt.Sprintf("%q", x)
}
return strings.Join(q, ", ")
}
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
func (a *verbArguments) commandLine() ([]string, error) {
verb, str, on, need := a.verb, a.str, a.on, a.need
switch verb {
case "command":
// The generic verb: the command line as given, split as a shell would split it, with
@@ -241,15 +64,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
if len(argv) == 0 {
return nil, errors.New("command names no command")
}
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
// it says why, as it would through its own verb.
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
"line (recorded in the hand-act log). Nothing was done", repair)
}
return argv, nil
case "tools":
return nil, errors.New("tools is answered from the records, not by a command")
case "status":
return []string{"status", "--json"}, nil
case "nodes":
@@ -267,14 +82,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
if id := str("log"); id != "" {
return []string{"builds", "--log", id}, nil
}
argv := []string{"builds"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
if m := str("module"); m != "" {
argv = append(argv, m)
return []string{"builds", m}, nil
}
return argv, nil
return []string{"builds"}, nil
case "plans":
if r := str("repository"); r != "" {
argv := []string{"plans", "--what-if", r}
@@ -284,87 +95,21 @@ func (a *verbArguments) commandLine() ([]string, error) {
if m := str("modules"); m != "" {
argv = append(argv, "--modules", m)
}
if d := str("module-dirs"); d != "" {
argv = append(argv, "--module-dirs", d)
}
return argv, nil
}
for _, act := range []string{"stop", "close", "retry", "go"} {
if id := str(act); id != "" {
argv := []string{"plans", act, id}
if act == "retry" {
return argv, nil
}
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
if id := str("stop"); id != "" {
return []string{"plans", "stop", id}, nil
}
if id := str("close"); id != "" {
return []string{"plans", "close", id}, nil
}
if id := str("retry"); id != "" {
return []string{"plans", "retry", id}, nil
}
if id := str("id"); id != "" {
return []string{"plans", id}, nil
}
argv := []string{"plans"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
return argv, nil
// The delivery's owner's verbs (novox/hq ADR 0239).
case "delivery-plan":
if err := need("repository", "paths"); err != nil {
return nil, err
}
argv := []string{"delivery", "plan", "--repository", str("repository"), "--paths", str("paths")}
for _, flag := range []string{"head", "base", "module-dirs", "removed"} {
if v := str(flag); v != "" {
argv = append(argv, "--"+flag, v)
}
}
return argv, nil
case "delivery-order":
if err := need("members"); err != nil {
return nil, err
}
return []string{"delivery", "order", "--members", str("members")}, nil
case "delivery-check":
if err := need("members"); err != nil {
return nil, err
}
if g := str("group"); g != "" {
return []string{"delivery", "check", "--group", g, "--members", str("members")}, nil
}
return []string{"delivery", "check", "--members", str("members")}, nil
case "deliver":
if err := need("plan"); err != nil {
return nil, err
}
argv := []string{"delivery", "go", str("plan"), "--by", catalogue.DeliverySeat}
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
}
return argv, nil
case "delivery-stop":
if err := need("plan", "why"); err != nil {
return nil, err
}
argv := []string{"delivery", "stop", str("plan"), "--why", str("why")}
if b := str("by"); b != "" {
argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
}
return argv, nil
case "delivery-walks":
argv := []string{"delivery", "walks"}
if id := str("plan"); id != "" {
return append(argv, "--plan", id), nil
}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
return argv, nil
return []string{"plans"}, nil
// The build queue (novox/hq ADR 0219).
case "queue":
return []string{"queue"}, nil
@@ -374,7 +119,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return []string{verb, str("id")}, nil
case "clear":
if on("dead") {
if str("dead") == "true" {
return []string{"clear", "--dead"}, nil
}
return []string{"clear"}, nil
@@ -388,10 +133,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, err
}
argv := []string{"replay", str("id")}
if on("register") {
if str("register") == "true" {
argv = append(argv, "--register")
}
if on("older") {
if str("older") == "true" {
argv = append(argv, "--older")
}
return argv, nil
@@ -404,9 +149,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err := need("node"); err != nil {
return nil, err
}
if on("files") {
return []string{"plan", str("node"), "--files"}, nil
}
return []string{"plan", str("node"), "--json"}, nil
case "assign", "unassign":
if err := need("node", "module"); err != nil {
@@ -429,232 +171,23 @@ func (a *verbArguments) commandLine() ([]string, error) {
// Sent and not waited for: the asker reads `status` for what the machine did, which is
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
// time out on every machine that takes a minute, and say nothing about the ones that did not.
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
if err := need("why"); err != nil {
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
}
why := []string{"--why", str("why")}
if c := str("cause"); c != "" {
why = append(why, "--cause", c)
}
if n := str("node"); n != "" {
// behind is not read here: given with a machine, it is refused as passed over — naming
// a machine and asking for every machine behind are two requests, and guessing one
// would push a machine nobody named, or not push one somebody did.
return append([]string{"push", n, "--wait", "0"}, why...), nil
return []string{"push", n, "--wait", "0"}, nil
}
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
// first, so a caller who meant one machine reads that it was not one.
on("behind")
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
case "hand-act":
if err := need("what", "why", "cause"); err != nil {
return nil, err
}
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
if c := str("condition"); c != "" {
argv = append(argv, "--condition", c)
}
return argv, nil
case "hand-acts":
argv := []string{"hand-acts", "--json"}
if d := str("days"); d != "" {
argv = append(argv, "--days", d)
}
return argv, nil
case "healers":
argv := []string{"healers", "--json"}
if d := str("days"); d != "" {
argv = append(argv, "--days", d)
}
return argv, nil
case "durations":
argv := []string{"durations", "--json"}
if k := str("kind"); k != "" {
argv = append(argv, "--kind", k)
}
if d := str("days"); d != "" {
argv = append(argv, "--days", d)
}
return argv, nil
case "conditions":
// One verb, four shapes, as `plans` (novox/hq to-be 45 §2): a silence, one condition, the
// history, or the open ones filtered.
if key := str("silence"); key != "" {
if err := need("for", "why"); err != nil {
return nil, err
}
argv := []string{"conditions", "silence", key, "--for", str("for"), "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
if on("history") {
argv := []string{"conditions", "history", "--json"}
if d := str("days"); d != "" {
argv = append(argv, "--days", d)
}
if k := str("key"); k != "" {
argv = append(argv, "--key", k)
}
return argv, nil
}
if k := str("key"); k != "" {
return []string{"conditions", "show", k, "--json"}, nil
}
argv := []string{"conditions", "--json"}
for _, filter := range []string{"scope", "severity", "machine"} {
if v := str(filter); v != "" {
argv = append(argv, "--"+filter, v)
}
}
return argv, nil
case "retire":
answer := str("answer")
if answer == "" {
return []string{"retire", "--json"}, nil
}
if answer != "approve" && answer != "reject" {
return nil, fmt.Errorf("retire answers approve or reject, not %q", answer)
}
if err := need("node", "module", "why"); err != nil {
return nil, err
}
argv := []string{"retire", answer, str("node"), str("module"), "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
case "cleanup":
consumer, older := str("consumer"), str("older-than")
switch {
case consumer != "" && older != "":
return nil, errors.New("cleanup deletes one consumer or those older than some days, not both")
case consumer != "":
if err := need("node", "module", "why"); err != nil {
return nil, err
}
argv := []string{"cleanup", "delete", str("node"), str("module"), consumer, "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
case older != "":
if err := need("why"); err != nil {
return nil, err
}
argv := []string{"cleanup", "delete", "--older-than", older, "--why", str("why")}
if on("confirm") {
argv = append(argv, "--confirm")
}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
return []string{"cleanup", "list", "--json"}, nil
case "data":
argv := []string{"data", "--json"}
if m := str("machine"); m != "" {
argv = append(argv, "--machine", m)
}
if on("retired") {
argv = append(argv, "--retired")
}
return argv, nil
case "upgrade":
if on("backlog") {
return []string{"upgrade", "backlog"}, nil
}
if on("release-backlog") {
argv := []string{"upgrade", "release-backlog", "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
m, policy := str("module"), str("policy")
if m == "" {
if policy != "" {
return nil, errors.New("upgrade takes a policy only for a module")
}
return []string{"upgrade"}, nil
}
argv := []string{"upgrade", m}
if policy != "" {
argv = append(argv, policy)
if on("together") {
argv = append(argv, "--together")
}
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
}
}
return argv, nil
case "bus":
if !on("upgrade") {
return []string{"bus"}, nil
}
argv := []string{"bus", "upgrade", "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
if on("reversible") {
argv = append(argv, "--reversible")
}
if on("irreversible") {
argv = append(argv, "--irreversible")
}
if w := str("snapshot-taken"); w != "" {
argv = append(argv, "--snapshot-taken", w)
}
return argv, nil
case "doctor":
which := 0
argv := []string{"doctor"}
for _, sub := range []string{"run", "probes", "signals"} {
if on(sub) {
which++
argv = append(argv, sub)
}
}
if which > 1 {
return nil, errors.New("doctor answers one of run, probes or signals at a time")
}
return append(argv, "--json"), nil
return []string{"push", "--behind", "--wait", "0"}, nil
case "rotate":
if p := str("provision"); p != "" {
argv := []string{"rotate", p}
if c := str("consumer"); c != "" {
argv = append(argv, "--consumer", c)
}
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
// and secret stay the other shape's, and are refused as passed over.
if m := str("module"); m != "" {
argv = append(argv, "--module", m)
}
return argv, nil
}
_, node := a.given["node"]
_, module := a.given["module"]
_, secret := a.given["secret"]
if node || module || secret {
if err := need("node", "module", "secret"); err != nil {
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
}
argv := []string{"secret", "rotate", str("node"), str("module"), str("secret")}
// Why, recorded in the hand-act log (novox/hq ADR 0228); a cause only beside a why.
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
}
return argv, nil
if str("node") != "" && str("module") != "" && str("secret") != "" {
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
}
// Neither shape: the command says its usage, which names both, and that is the answer the
// caller needs.
// Half of either shape: the command says its usage, which names both shapes, and that is
// the answer the caller needs.
return []string{"rotate"}, nil
case "settings":
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
@@ -662,16 +195,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err := need("module"); err != nil {
return nil, err
}
var argv []string
if on("clear") {
argv := []string{"settings", "set", str("module")}
switch {
case str("clear") == "true":
argv = []string{"settings", "clear", str("module")}
} else {
argv = []string{"settings", "set", str("module")}
// Neither values nor clear: the command says its usage, which names both.
if v := str("values"); v != "" {
argv = append(argv, v)
}
case str("values") != "":
argv = append(argv, str("values"))
}
// Neither values nor clear: the command says its usage, which names both, and that is the
// answer the caller needs — the same as `rotate` given half of either shape.
if n := str("node"); n != "" {
argv = append(argv, "--node", n)
}
@@ -703,41 +235,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
}
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
verb, catalogue.ControllerSeatName)
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true,
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
"delivery": true}
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
func repairingCommand(argv []string) string {
switch {
case argv[0] == "push":
return "push"
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close" || argv[1] == "go"):
return "plans " + argv[1]
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
return "broker consumer-reset"
case argv[0] == "hand-act":
return "hand-act record"
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
return "conditions silence"
case argv[0] == "retire" && len(argv) > 1 && (argv[1] == "approve" || argv[1] == "reject"):
return "retire " + argv[1]
case argv[0] == "cleanup" && len(argv) > 1 && argv[1] == "delete":
return "cleanup delete"
}
return ""
}
func isWhyFlag(word string) bool {
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
}
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
// runVerb runs this binary with the given command line and gathers what it said.
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
@@ -749,12 +251,6 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
// The same environment: the stores' credentials, the bus, the broker — everything a command run
// from a shell in this container would have, because it is that.
cmd.Env = os.Environ()
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
caller := link.CallerIn(ctx)
if caller == "" {
caller = "a seat call whose caller the bus did not name"
}
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
// prints its warnings beside the document, and a JSON parsed from the two together parsed
@@ -791,35 +287,8 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
handlers := map[string]link.ToolHandler{}
for _, v := range seat.Serves {
verb := v.Name
if inProcess[verb] {
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
args := map[string]any{}
if len(bytes.TrimSpace(raw)) > 0 {
if err := json.Unmarshal(raw, &args); err != nil {
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
}
}
// Refused like any verb's: what a verb does not take is not ignored.
a, err := readArguments(verb, args)
if err != nil {
return nil, err
}
if verb == "calls" {
return callsAnswer(link.Calls, a.given["call"])
}
if verb == "doctor" {
// From the serving controller, which runs the self-check and hears the signals
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
argv, err := a.commandLine()
if err != nil {
return nil, err
}
sub := ""
if len(argv) > 2 {
sub = argv[1]
}
return doctorAnswer(ctx, sub)
}
if verb == "tools" {
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
return seatTools(), nil
}
continue
@@ -858,85 +327,12 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
if err != nil {
return nil, err
}
if verb == "status" && statusFrom != nil {
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
return statusFrom.answer(ctx)
}
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
// Whatever it did, `status` is composed again once it has.
defer statusFrom.nudge()
}
if answersFirst(argv) {
// Before anything is sent: a push sends the bus's own machine first, and a broker
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
link.Acknowledge(ctx)
}
return runVerb(ctx, argv)
}
}
return handlers, behind, nil
}
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
func actsOnAPlan(args map[string]any) bool {
for _, act := range []string{"stop", "close", "retry"} {
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
return true
}
}
return false
}
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
// the records, `calls` from what this process served.
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
// through `command`. A push sends the machine holding the bus first when its user list changed, the
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
func answersFirst(argv []string) bool {
return len(argv) > 0 && argv[0] == "push"
}
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
// the reason said beside it.
func callsAnswer(log *link.CallLog, id string) (any, error) {
if id != "" {
c, ok, err := log.Get(id)
if err != nil {
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
"bus could not be read: %w", id, err)
}
if !ok {
if log.IsDurable() {
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
}
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
}
return c, nil
}
recent, err := log.Recent()
for i := range recent {
recent[i].Answer = nil
}
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
if log.IsDurable() {
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
broker.KeptCallsDurably, broker.CallsKeptFor)
} else {
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
}
if err != nil {
answer["unread"] = err.Error()
}
return answer, nil
}
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
func seatTools() map[string]any {
@@ -957,10 +353,9 @@ func seatTools() map[string]any {
}
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
// more, since an argument a verb does not declare is refused.
// verb runnable rather than that it happens to want the arguments the check guessed.
func sampleArguments(v catalogue.Verb) map[string]any {
sample := map[string]any{}
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
switch required := v.Input["required"].(type) {
case []string:
for _, k := range required {
cmd/mesh-controller/seatverbs_schema_test.go
-377
View File
@@ -1,377 +0,0 @@
package main
import (
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The schemas the controller serves, verb by verb, as the console receives them: from the
// announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244).
func servedSchemas(t *testing.T) map[string]catalogue.Verb {
t.Helper()
handlers, behind, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
if len(behind) != 0 {
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
}
served := map[string]catalogue.Verb{}
for _, e := range seatAnnouncement(handlers).Endpoints {
var input map[string]any
if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil {
t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err)
}
served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input}
}
if len(served) != len(catalogue.ControllerVerbs) {
t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs))
}
return served
}
// subsetsOf is every subset of the names, the empty one included.
func subsetsOf(names []string) [][]string {
var out [][]string
for mask := 0; mask < 1<<len(names); mask++ {
var s []string
for i, n := range names {
if mask&(1<<i) != 0 {
s = append(s, n)
}
}
out = append(out, s)
}
return out
}
func argumentsFor(subset []string, switches map[string]bool) map[string]any {
args := map[string]any{}
for _, n := range subset {
if switches[n] {
args[n] = "true"
} else {
args[n] = "x-" + n
}
}
return args
}
// saidOutright are the switches that say the default aloud, so the line is the same without them —
// on purpose, and only these.
var saidOutright = map[string]string{
"push": "behind", // the whole mesh is what no machine means; behind lets a caller say they meant it
}
// **No argument a caller gives is passed over** (novox/hq issue 244). For every verb served and
// every combination of the arguments its schema declares, the verb either refuses the call, or
// composes a command line that each given argument changed: taking any one away changes the line
// or makes it refused. An argument the line is the same without is one the verb ignored — the
// shape of the push that named a machine and pushed every machine behind.
func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
for name, v := range servedSchemas(t) {
if inProcess[name] {
continue
}
names, switches := declaredArguments(v)
for _, subset := range subsetsOf(names) {
args := argumentsFor(subset, switches)
argv, err := argvFor(name, args)
if err != nil {
if strings.Contains(err.Error(), "does not declare") {
t.Errorf("%s %v: %v", name, args, err)
}
continue
}
for _, dropped := range subset {
fewer := map[string]any{}
for k, val := range args {
if k != dropped {
fewer[k] = val
}
}
without, err := argvFor(name, fewer)
if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped {
t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv)
}
}
}
}
}
// **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with
// what it requires and one argument more; and `node` in particular, for every verb whose schema
// does not take a machine.
func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
for name, v := range servedSchemas(t) {
names, _ := declaredArguments(v)
strangers := []string{"no-such-argument"}
if !contains(names, "node") {
strangers = append(strangers, "node")
}
for _, stranger := range strangers {
args := sampleArguments(v)
args[stranger] = "x"
_, err := argvFor(name, args)
if inProcess[name] {
_, err = readArguments(name, args)
}
if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err)
}
}
}
if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil {
t.Error("a switch took a word that is neither true nor false, and would have read it as false")
}
if _, err := argvFor("node", map[string]any{"node": 7}); err == nil {
t.Error("a number was taken as a machine's name, or as no machine")
}
if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" {
t.Errorf("a switch given as JSON true: %v %v", argv, err)
}
}
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
// naming one beside behind is refused rather than one of the two guessed.
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
t.Fatalf("a named push: %v %v", argv, err)
}
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
argv, err := argvFor("push", args)
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
}
}
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil ||
!strings.Contains(err.Error(), `"behind"`) {
t.Fatalf("a named push with behind was taken: %v", err)
}
if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) {
t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err)
}
}
// commandFlags is every flag set this package's commands parse, by the name the set is made with,
// and the flags defined on it — read from the source, so a flag added to a command is seen here
// without anyone remembering to.
func commandFlags(t *testing.T) map[string][]string {
t.Helper()
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
fset := token.NewFileSet()
out := map[string][]string{}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
file, err := parser.ParseFile(fset, f, nil, 0)
if err != nil {
t.Fatal(err)
}
for _, decl := range file.Decls {
fn, ok := decl.(*ast.FuncDecl)
if !ok || fn.Body == nil {
continue
}
sets := map[string]string{} // variable → the set's name
ast.Inspect(fn.Body, func(n ast.Node) bool {
if assign, ok := n.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 && len(assign.Rhs) == 1 {
if call, ok := assign.Rhs[0].(*ast.CallExpr); ok && isSelector(call.Fun, "flag", "NewFlagSet") {
if id, ok := assign.Lhs[0].(*ast.Ident); ok {
if name, ok := stringLit(call.Args[0]); ok {
sets[id.Name] = name
out[name] = append(out[name], []string{}...)
}
}
}
}
call, ok := n.(*ast.CallExpr)
if !ok {
return true
}
sel, ok := call.Fun.(*ast.SelectorExpr)
if !ok {
return true
}
recv, ok := sel.X.(*ast.Ident)
if !ok || sets[recv.Name] == "" {
return true
}
arg := 0
switch sel.Sel.Name {
case "Bool", "String", "Int", "Int64", "Uint", "Uint64", "Float64", "Duration", "Func", "BoolFunc", "TextVar":
case "BoolVar", "StringVar", "IntVar", "Int64Var", "UintVar", "Uint64Var", "Float64Var", "DurationVar", "Var":
arg = 1
default:
return true
}
if arg < len(call.Args) {
if flagName, ok := stringLit(call.Args[arg]); ok {
out[sets[recv.Name]] = append(out[sets[recv.Name]], flagName)
}
}
return true
})
}
}
return out
}
func isSelector(e ast.Expr, pkg, name string) bool {
sel, ok := e.(*ast.SelectorExpr)
if !ok {
return false
}
id, ok := sel.X.(*ast.Ident)
return ok && id.Name == pkg && sel.Sel.Name == name
}
func stringLit(e ast.Expr) (string, bool) {
lit, ok := e.(*ast.BasicLit)
if !ok || lit.Kind != token.STRING {
return "", false
}
s, err := strconv.Unquote(lit.Value)
return s, err == nil
}
// accountedFlags are the flags of a verb's command that are not an argument of the same name:
// carried by an argument named otherwise ("=argument"), or set by the verb itself, or withheld from
// the named verb on purpose — each with why. `command` reaches every flag of the binary regardless.
var accountedFlags = map[string]map[string]string{
"status": {"json": "set by the verb: the answer is data"},
"seats": {"json": "set by the verb: the answer is data"},
"queue": {"json": "not set: the verb answers the table a person reads"},
"plan": {"json": "set by the verb unless files is asked"},
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
"build": {
"wait": "set by the verb to 0: the id follows the build (issue 176)",
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
"behind": "withheld: the named verb builds one named repository; `command` reaches the rest",
"on": "withheld: the named verb builds one named repository; `command` reaches the rest",
},
"builds": {"n": "=limit"},
"plans": {"n": "=limit", "what-if": "=repository"},
"durations": {
"json": "set by the verb: the answer is data",
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
},
"hand-acts": {"json": "set by the verb: the answer is data"},
"conditions": {"json": "set by the verb: the answer is data"},
"retire": {"json": "set by the verb: the answer is data"},
"cleanup": {"json": "set by the verb: the answer is data"},
"data": {"json": "set by the verb: the answer is data"},
"conditions history": {"json": "set by the verb: the answer is data"},
"conditions show": {"json": "set by the verb: the answer is data"},
"healers": {"json": "set by the verb: the answer is data"},
}
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
// from the source, so a flag added to a command — or a verb added whose command takes flags —
// fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And
// a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody
// reads.
func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
flags := commandFlags(t)
if len(flags["push"]) == 0 || len(flags["plan"]) == 0 {
t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags)
}
reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
served := servedSchemas(t)
for name, v := range served {
if inProcess[name] || name == "command" {
continue
}
names, switches := declaredArguments(v)
for _, subset := range subsetsOf(names) {
argv, err := argvFor(name, argumentsFor(subset, switches))
if err != nil {
continue
}
// The flag set is named by the longest run of leading words that names one.
var words []string
for _, w := range argv {
if strings.HasPrefix(w, "-") {
break
}
words = append(words, w)
}
for n := len(words); n > 0; n-- {
if _, has := flags[strings.Join(words[:n], " ")]; has {
if reached[name] == nil {
reached[name] = map[string]bool{}
}
reached[name][strings.Join(words[:n], " ")] = true
break
}
}
}
}
used := map[string]map[string]bool{}
verbs := make([]string, 0, len(reached))
for verb := range reached {
verbs = append(verbs, verb)
}
sort.Strings(verbs)
for _, verb := range verbs {
declared, _ := declaredArguments(served[verb])
for set := range reached[verb] {
if used[set] == nil {
used[set] = map[string]bool{}
}
for _, flagName := range flags[set] {
why, accounted := accountedFlags[set][flagName]
switch {
case accounted && strings.HasPrefix(why, "="):
used[set][flagName] = true
if !contains(declared, strings.TrimPrefix(why, "=")) {
t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare",
verb, flagName, set, strings.TrimPrefix(why, "="))
}
case accounted:
used[set][flagName] = true
case contains(declared, flagName):
default:
t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+
"or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName)
}
}
}
}
for set, fs := range accountedFlags {
for flagName := range fs {
if !used[set][flagName] {
t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set)
}
}
}
}
// Every verb's required arguments are properties of its schema: a schema that requires what it
// does not describe is the uncallable verb of issue 244 from the other side.
func TestEveryRequiredArgumentIsDescribed(t *testing.T) {
for name, v := range servedSchemas(t) {
names, _ := declaredArguments(v)
for k := range sampleArguments(v) {
if !contains(names, k) {
t.Errorf("%s requires %q and does not describe it", name, k)
}
}
}
}
cmd/mesh-controller/seatverbs_test.go
+29 -15
View File
@@ -10,6 +10,29 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
)
// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
// schema names and no other (novox/hq ADR 0154, ADR 0035).
func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == "tools" {
continue
}
args := map[string]any{}
props, _ := v.Input["properties"].(map[string]any)
for name := range props {
args[name] = "x"
}
argv, err := argvFor(v.Name, args)
if err != nil {
t.Errorf("%s: %v", v.Name, err)
continue
}
if argv[0] == "" {
t.Errorf("%s: empty command", v.Name)
}
}
}
// `builds` given a build's id reads that build's log from the bus rather than listing builds
// (novox/hq ADR 0157).
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
@@ -43,22 +66,13 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
t.Fatalf("a pair credential: %v", argv)
}
argv, _ = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "module": "letta"})
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace --module letta" {
t.Fatalf("one consuming module's pair credential: %v", argv)
}
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
t.Fatalf("an own secret: %v", argv)
}
if _, err := argvFor("rotate", map[string]any{"node": "ace"}); err == nil || !strings.Contains(err.Error(), `"module"`) {
t.Fatalf("half an own secret is refused, naming what it lacks: %v", err)
}
if argv, _ := argvFor("rotate", nil); strings.Join(argv, " ") != "rotate" {
t.Fatalf("neither shape falls to the command's usage: %v", argv)
}
if _, err := argvFor("rotate", map[string]any{"provision": "p", "node": "ace", "module": "m", "secret": "s"}); err == nil {
t.Fatal("both shapes at once were taken, and one of them passed over")
argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
if strings.Join(argv, " ") != "rotate" {
t.Fatalf("half an own secret falls to the command's usage: %v", argv)
}
}
@@ -99,7 +113,7 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
if _, err := argvFor("node", map[string]any{}); err == nil || !strings.Contains(err.Error(), `node needs "node"`) {
t.Fatalf("node without a machine was accepted: %v", err)
}
if _, err := argvFor("no-such-verb", map[string]any{}); err == nil {
if _, err := argvFor("upgrade", map[string]any{}); err == nil {
t.Fatal("a verb the seat does not serve was accepted")
}
}
@@ -107,8 +121,8 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
func TestActsDoNotBlockTheCall(t *testing.T) {
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
argv, _ := argvFor("push", map[string]any{"node": "one"})
if strings.Join(argv, " ") != "push one --wait 0" {
t.Fatalf("push waits: %v", argv)
}
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
cmd/mesh-controller/secret.go
+5 -38
View File
@@ -93,31 +93,18 @@ func secretCommand(ctx context.Context, args []string) error {
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
return nil
}
untilStart, err := open.inventory.AcceptGivenSecret(ctx, node, module, name, value)
if err != nil {
if err := open.inventory.AcceptSecretForModule(ctx, node, module, name, value); err != nil {
return err
}
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
// machine and the mesh cannot read it again.
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
if untilStart {
// Accepted, and said what it is for (novox/hq ADR 0228): a value given by hand adopts
// something that already holds it, and lives only until the module has started on it.
fmt.Printf(" it lives until %s next starts well under the mesh on it, and is then replaced with a value\n"+
" the mesh makes, sealed and sent (ADR 0228): a value given by hand is for adopting something\n"+
" already running that holds it\n", module)
if record, err := open.inventory.NodeByName(ctx, node); err == nil && !record.Adopted {
fmt.Printf(" %s is not an adopted machine: if %s is installed fresh there, it needs no given value —\n"+
" the mesh makes one at the first push\n", node, module)
}
} else {
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
}
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
fmt.Printf(" run `push %s` to send it\n", node)
return nil
}
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
const secretUsage = "secret rotate <node> <module> <name>\n" +
"secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
"secret export [--out <file>]"
@@ -380,20 +367,9 @@ func valueFor(node, module, name, from string) (string, error) {
// secretRotate makes a module's own secret anew and sends the machine, so the module starts again on
// the new value (novox/hq ADR 0114, issue 180). A pair credential rotates with `rotate <provision>`;
// this is the secret with one party. Said in the log with who asked and when, never the value.
//
// **A value given to the mesh rotates the same way** (novox/hq ADR 0228): what a module reads at start
// is held by nobody else, so the old value is not needed to replace it. Refused for a value an
// outside party issued, which no value of the mesh's would replace. Why it was rotated is recorded in
// the hand-act log when given — a rotation asked by a person is an act by hand, and a leak is a cause
// worth counting.
func secretRotate(ctx context.Context, args []string) error {
rest, flags := split(args)
set := flag.NewFlagSet("secret rotate", flag.ContinueOnError)
why := addHandActFlags(set)
if err := set.Parse(flags); err != nil {
return err
}
if len(rest) != 3 || set.NArg() != 0 {
rest, _ := split(args)
if len(rest) != 3 {
return errors.New(secretUsage)
}
node, module, name := rest[0], rest[1], rest[2]
@@ -402,10 +378,6 @@ func secretRotate(ctx context.Context, args []string) error {
return err
}
defer open.Close()
origin, given, err := open.inventory.OwnSecretOrigin(ctx, node, module, name)
if err != nil {
return err
}
if err := open.inventory.RotateModuleSecret(ctx, node, module, name); err != nil {
var refused inventory.ErrNotRotatable
if errors.As(err, &refused) {
@@ -413,13 +385,8 @@ func secretRotate(ctx context.Context, args []string) error {
}
return err
}
why.record(ctx, "secret rotate", []string{node, module, name})
fmt.Printf("rotated %q of %s on %s at %s, asked by %s; the value is sealed and not shown\n",
name, module, node, time.Now().UTC().Format(time.RFC3339), whoAsked())
if origin == inventory.OriginAccepted {
fmt.Printf(" it replaces the value given to the mesh on %s: the mesh made this one, so `secret rotate` "+
"replaces it again whenever asked (ADR 0228)\n", given.UTC().Format("2006-01-02"))
}
// A shared credential (ADR 0158) has as many holders as the provision has consumers, and all
// of them are sent in one act, so no machine is left reading a value the provider no longer takes.
machines, err := open.inventory.SharedHolders(ctx, node, module, name)
cmd/mesh-controller/sendable.go
-21
View File
@@ -22,11 +22,6 @@ type sendable struct {
// under the node's hold just before the body is made (novox/hq 04-ISSUES/107). Zero is not sent
// at all, which a host reads as "no order claimed" — the shape of every declaration before this.
Sequence int64
// Epoch is the controller lease's epoch it was composed under (novox/hq to-be 45 §6): a machine that
// heard a later epoch refuses it. Zero is not sent at all — every machine whose node-engine has not
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
// (link/order.go, the contract).
Epoch uint64
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
// adoption existed: an older host parses the envelope strictly and would refuse the key.
Adoption *adoptionEnvelope
@@ -48,23 +43,10 @@ type sendable struct {
LeftOut []string
// leftOutWhy is why each was, for push and plan to say; never on the wire.
leftOutWhy map[string]string
// withheld is every consumer this machine's grants leave out, because its identity overflows the
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
withheld []catalogue.Overflow
// unbound is every consumer whose credential from this machine is on record and that is bound
// elsewhere (novox/hq issue 274); for push and plan to say, never on the wire.
unbound []catalogue.Unbound
// foreseen is every own secret composed with a stand-in, as `module/name`: what the next send will
// make (Foreseeing, novox/hq issue 275). Never on the wire, and a declaration that has any is never
// sent.
foreseen []string
// Builds is the build of each module this declaration carries — module to the commit its build
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
// Composed only on the send path; nil records that it is not known.
Builds map[string]string
// Bindings is where each consumer of a provision that keeps its data is bound in this declaration
// (novox/hq ADR 0232), recorded with the send and never on the wire. Composed only on the send path.
Bindings []inventory.Binding
}
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
@@ -85,9 +67,6 @@ func (s sendable) Body() ([]byte, error) {
if s.Sequence > 0 {
envelope["sequence"] = s.Sequence
}
if s.Epoch > 0 {
envelope["epoch"] = s.Epoch
}
if len(s.LeftOut) > 0 {
envelope["left_out"] = s.LeftOut
}
cmd/mesh-controller/signals.go
-707
View File
@@ -1,707 +0,0 @@
package main
import (
"fmt"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The signals table (novox/hq to-be 45 §3, ADR 0227 rule 5), compiled in.
//
// **Every signal the core expects has a watchdog; its absence is a condition.** A row says what is
// expected, from whom, after what, within what bound, and what is raised when it does not come. One
// table, read three ways: by the watchdog loop (watchdogs.go), which runs every row's watch over the
// facts it gathered; by `doctor signals`, which says the age of every row's newest signal; and by the
// test generated from it (signals_test.go), which suppresses each signal in turn and asserts its
// condition — so a row added without a watch, or a watch that does not fire, fails the build.
//
// A row not watched yet says why, and in which phase it will be: a watchdog of a signal nothing emits
// would be a condition that can only cry wolf. Bounds marked provisional are set from what Phase 0
// measured (`durations`) and corrected in Phase 1's first live week; a corrected bound is a change to
// this table, reviewed like code.
// signalRow is one row of the table.
type signalRow struct {
Row string
Signal string
Emitter string
Trigger string
Bound string
Kind string
Severity conditions.Severity
// Phase is the phase of to-be 45 its watchdog is built in.
Phase int
// Deferred says why it is not watched yet; empty for a row that is.
Deferred string
// needs is the part of the facts the row reads: an error there is the row blind, which is
// itself said (probe-failed) rather than read as nothing wrong.
needs func(f *signalFacts) error
// watch is what is wrong now, from the facts.
watch func(f *signalFacts) []conditions.Observation
// newest is the time of the newest signal of this row, for `doctor signals`; zero when none.
newest func(f *signalFacts) time.Time
}
// The bounds, provisional where to-be 45 says so.
const (
// heartbeatEvery is the interval a node-engine or node tools that say none have always used.
heartbeatEvery = 60 * time.Second
// heartbeatsMissed is how many intervals may pass in silence (S1, S11).
heartbeatsMissed = 3
// controlNodeUrgentAfter is how long the control node may be silent before it is urgent (S1).
controlNodeUrgentAfter = 30 * time.Minute
// reportAtLeast is the least a machine is given to report a send (S2).
reportAtLeast = 2 * time.Minute
// tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at.
tierAtLeast = planWaitBound
// loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4).
loopDeafAfter = 2 * time.Minute
// askAtLeast is the least a build ask is given, and askDefault the bound while nothing is
// measured (S6): the build seat declares no timeout of its own.
askAtLeast = 20 * time.Minute
askDefault = time.Hour
// callDefault is the bound of a verb that declares none (S7); push's and build's are longer.
callDefault = 10 * time.Minute
// advisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9).
advisoryQuiet = time.Hour
// staleRefusalsAllowed in staleRefusalsWithin are what S13 lets pass from one writer.
staleRefusalsAllowed = 5
staleRefusalsWithin = 5 * time.Minute
// leaseBound is how long the lease may go unrenewed (S12): the key's age.
leaseBound = broker.LeaseTTL
// waitBound is how long a walk may wait for its delivery's word before it is said, and waitUrgentAfter
// before it is urgent (S16, novox/hq ADR 0239): a group member may wait behind another for a while; four
// hours without a word is the delivery's owner broken, whatever it says of itself.
waitBound = 30 * time.Minute
waitUrgentAfter = 4 * time.Hour
)
// callBounds are the verbs that may run longer than callDefault, and how long (S7).
var callBounds = map[string]time.Duration{
"push": 30 * time.Minute, "rotate": 30 * time.Minute, "assign": 15 * time.Minute,
"unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute,
}
// callBound is a verb's bound.
func callBound(verb string) time.Duration {
if b, ok := callBounds[verb]; ok {
return b
}
return callDefault
}
// signalsTable is the table, in to-be 45's order.
var signalsTable = []signalRow{
{Row: "S1", Signal: "machine heartbeat", Emitter: "node-engine", Trigger: "its interval",
Bound: "3 × the interval it says (60 s when it says none), provisional; not raised while the machine " +
"said it is asleep or shutting down (ADR 0211); urgent after 30 min for a control node",
Kind: "silent", Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchHeartbeats,
newest: func(f *signalFacts) time.Time {
return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard })
}},
{Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent",
Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep",
Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports,
newest: func(f *signalFacts) time.Time {
return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt })
}},
{Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered",
Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional; not while the " +
"build seat is paused under it",
Kind: "stalled", Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans,
newest: func(f *signalFacts) time.Time {
return newestOf(f.plans, func(p planFacts) time.Time { return p.entered })
}},
{Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller",
Trigger: "while its consumers have pending messages", Bound: "2 min",
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
newest: func(f *signalFacts) time.Time { return f.loop.took }},
{Row: "S5", Signal: "a merge announced becomes a plan, or nothing reads it", Emitter: "announcer → controller",
Trigger: "each merge", Bound: "10 min (the catch-up pass of issue 266)",
Kind: "merge-not-acted", Severity: conditions.Urgent, Phase: 1,
needs: func(f *signalFacts) error { return f.mergesErr }, watch: watchMerges,
newest: func(f *signalFacts) time.Time { return f.mergesPassed }},
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
Kind: "ask-lost", Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.asksErr }, watch: watchAsks,
newest: func(f *signalFacts) time.Time { return newestOf(f.asks, func(a askFacts) time.Time { return a.since }) }},
{Row: "S7", Signal: "a call running → finished", Emitter: "controller", Trigger: "each call",
Bound: "the verb's bound: push, rotate and command 30 min, assign and unassign 15 min, doctor 3 min, " +
"any other 10 min",
Kind: "call-hung", Severity: conditions.Warning, Phase: 1,
needs: func(*signalFacts) error { return nil }, watch: watchCalls,
newest: func(f *signalFacts) time.Time {
return newestOf(f.calls, func(c link.Call) time.Time { return c.Started })
}},
{Row: "S8", Signal: "a provider's failing word repeated", Emitter: "provider",
Trigger: "every 15 min while failing (ADR 0224)", Bound: "30 min",
Kind: kindProviderSilent, Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.standingsErr }, watch: watchProviders,
newest: func(f *signalFacts) time.Time {
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
}},
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
"once it exists again or the mesh no longer expects it",
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
newest: func(f *signalFacts) time.Time {
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
}},
{Row: "S10", Signal: "the self-check's heartbeat", Emitter: "controller's doctor", Trigger: "every run",
Bound: "2 × its interval; watched from a second machine by mesh-watcher, and here as well",
Kind: "self-check-silent", Severity: conditions.Urgent, Phase: 1,
needs: func(*signalFacts) error { return nil }, watch: watchSelfCheck,
newest: func(f *signalFacts) time.Time { return f.selfCheck.last }},
{Row: "S11", Signal: "node tools heartbeat", Emitter: "node tools", Trigger: "its interval",
Bound: "3 × the interval it says (60 s when it says none), provisional; only where node-tools is assigned",
Kind: "tools-silent", Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchTools,
newest: func(f *signalFacts) time.Time {
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
}},
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
Bound: "15 s (the key's age); a holder that lost the lease, or stopped renewing and was taken over, and a " +
"lease bucket found raised again from nothing, are said for an hour after; a controller serving without " +
"the lease, for as long as it does",
Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
needs: func(f *signalFacts) error { return f.leaseErr }, watch: watchLease,
newest: func(f *signalFacts) time.Time { return f.lease.renewed }},
{Row: "S13", Signal: "stale refusals", Emitter: "every receiver (rule 2)", Trigger: "each refusal",
Bound: "more than 5 from one writer in 5 min: a controller epoch, a controller that claimed none, or a " +
"machine's node-engine whose accounts the controller refused",
Kind: "stale-writer", Severity: conditions.Warning, Phase: 2,
needs: func(*signalFacts) error { return nil }, watch: watchStaleRefusals,
newest: func(f *signalFacts) time.Time {
return newestOf(f.staleRefusals, func(w link.WriterRefusals) time.Time { return w.Last })
}},
{Row: "S14", Signal: "facts snapshot exported", Emitter: "controller", Trigger: "when it moved, and daily",
Bound: "2 days: a snapshot older than that, or none kept since this controller began two days ago",
Kind: "facts-stale", Severity: conditions.Warning, Phase: 5,
needs: func(*signalFacts) error { return nil }, watch: watchFacts,
newest: func(f *signalFacts) time.Time { return f.facts.taken }},
{Row: "S15", Signal: "a hand act with a cause already recorded", Emitter: "hand-act log",
Trigger: "each act", Bound: "the second within 14 days; clears when fewer than two remain within 14 days",
Kind: "healer-wanted", Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.handActsErr }, watch: watchHandActs,
newest: func(f *signalFacts) time.Time {
return newestOf(f.handActs, func(a link.HandAct) time.Time { return a.At })
}},
{Row: "S16", Signal: "a walk waiting for its delivery's word is let go", Emitter: "mesh-delivery, through deliver",
Trigger: "each merge whose walk waits (novox/hq ADR 0239)",
Bound: "30 min, then urgent after 4 h — raised by the controller whatever mesh-delivery says of itself, " +
"naming `plans go` as the way on",
Kind: kindWalkWaiting, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchWaits,
newest: func(f *signalFacts) time.Time {
return newestOf(f.waits, func(w waitFacts) time.Time { return w.since })
}},
}
// watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since
// this controller began that long ago (novox/hq to-be 45 §9). A check fed a stale snapshot judges a change
// against a mesh that no longer is, which is the fault the snapshot exists to end.
func watchFacts(f *signalFacts) []conditions.Observation {
since := f.facts.taken
if since.IsZero() {
since = f.facts.began
}
if since.IsZero() || f.now.Sub(since) <= factsStaleAfter {
return nil
}
said := "none kept since this controller began " + ago(f.now.Sub(since)) + " ago"
if !f.facts.taken.IsZero() {
said = "the newest kept was taken " + ago(f.now.Sub(f.facts.taken)) + " ago"
}
if f.facts.err != nil {
said += "; the last attempt: " + oneLine(f.facts.err.Error())
}
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "facts", Kind: "facts-stale", Token: "stale",
Severity: conditions.Warning,
Summary: fmt.Sprintf("the facts snapshot merge checks are fed is stale (bound %s): a change is judged against a "+
"mesh that no longer is", ago(factsStaleAfter)),
Said: said}}
}
// newestOf is the newest time among things.
func newestOf[T any](list []T, at func(T) time.Time) time.Time {
var newest time.Time
for _, x := range list {
if t := at(x); t.After(newest) {
newest = t
}
}
return newest
}
// ago is a duration as the summaries say it.
func ago(d time.Duration) string {
if d < time.Minute {
return d.Round(time.Second).String()
}
return d.Round(time.Minute).String()
}
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
func heartbeatBound(every time.Duration) time.Duration {
if every <= 0 {
every = heartbeatEvery
}
return heartbeatsMissed * every
}
func watchHeartbeats(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, m := range f.machines {
if m.lastHeard.IsZero() || m.asleep() {
// Never heard is a machine that has not joined, which status says; asleep is not lost.
continue
}
bound := heartbeatBound(m.every)
silent := f.now.Sub(m.lastHeard)
if silent <= bound {
continue
}
severity := conditions.Warning
if m.control && silent > controlNodeUrgentAfter {
severity = conditions.Urgent
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "silent",
Machine: m.name, Severity: severity,
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", m.name,
m.lastHeard.UTC().Format("2006-01-02 15:04 MST"), bound),
Said: fmt.Sprintf("silent for %s", ago(silent))})
}
return out
}
func watchReports(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, m := range f.machines {
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
continue
}
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
continue // silent: S1 says it, and a silent machine reports nothing
}
bound := max(reportAtLeast, 3*m.lastApply)
waited := f.now.Sub(m.sentAt)
if waited <= bound {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name,
Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)",
m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))})
}
return out
}
func watchPlans(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, p := range f.plans {
if p.paused {
continue
}
in := f.now.Sub(p.entered)
if in <= p.bound {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
Severity: conditions.Warning,
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
ago(p.bound), p.waiting),
Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)})
}
return out
}
// kindWalkWaiting is S16's kind: plan.<id>.waiting.
const kindWalkWaiting = "waiting"
// watchWaits is S16: a walk that has waited for its delivery's word past its bound, said by the controller
// itself — a delivery's owner that is up and never says go (a bug, a delivery stuck in its own table) would
// otherwise leave a merge waiting for ever with nothing open (novox/hq ADR 0239).
func watchWaits(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, w := range f.waits {
in := f.now.Sub(w.since)
if in <= waitBound {
continue
}
severity := conditions.Warning
if in > waitUrgentAfter {
severity = conditions.Urgent
}
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.id, Kind: kindWalkWaiting,
Severity: severity,
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
"delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
short(w.commit), ago(in), w.awaits, short(w.commit), w.id),
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits)})
}
return out
}
func watchLoop(f *signalFacts) []conditions.Observation {
if f.loop.pending == 0 {
return nil
}
since := f.loop.took
if since.IsZero() {
since = f.started
}
if f.now.Sub(since) <= loopDeafAfter {
return nil
}
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "controller", Kind: "controller-deaf",
Token: "deaf", Machine: f.host, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the controller's event loop has taken nothing for %s while its consumers hold %d "+
"message(s): reports, builds and merges are not being acted on", ago(f.now.Sub(since)), f.loop.pending),
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
}
func watchMerges(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, m := range f.merges {
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, ID: m.Repo + "." + short(m.Commit),
Kind: "merge-not-acted", Token: "not-acted", Severity: conditions.Urgent,
Summary: fmt.Sprintf("%s/%s merged into %s (%s) was never handed to the controller by the bus; "+
"acted on late by the catch-up", m.Owner, m.Repo, m.Base, short(m.Commit)),
Said: fmt.Sprintf("announced %s, %s behind it", m.At.UTC().Format(time.RFC3339), readableList(m.Modules))})
}
return out
}
func watchAsks(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, a := range f.asks {
var said string
switch {
case a.state == link.AskDead:
said = fmt.Sprintf("the %s queue handed it out as often as it may and it was never settled", a.seat)
case a.state == link.AskInFlight && f.now.Sub(a.since) > a.bound:
said = fmt.Sprintf("in flight on %s for %s (bound %s)", orSomewhere(a.on), ago(f.now.Sub(a.since)), ago(a.bound))
default:
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeBuild, ID: a.id, Kind: "ask-lost",
Token: "lost", Machine: a.on, Severity: conditions.Warning,
Summary: fmt.Sprintf("the build %s of %s has no outcome: %s", a.id, a.what, said), Said: said})
}
return out
}
func orSomewhere(node string) string {
if node == "" {
return "a machine that did not say which"
}
return node
}
func watchCalls(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, c := range f.calls {
bound := callBound(c.Verb)
running := f.now.Sub(c.Started)
if running <= bound {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeCall, ID: c.ID, Kind: "call-hung",
Token: "hung", Machine: f.host, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s.%s (call %s) has been running since %s, past its bound of %s", c.Seat, c.Verb,
c.ID, c.Started.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
Said: fmt.Sprintf("running %s, asked by %s", ago(running), orSomebody(c.Caller))})
}
return out
}
func orSomebody(caller string) string {
if caller == "" {
return "a caller the bus did not name"
}
return caller
}
func watchProviders(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, c := range f.standings {
quiet := f.now.Sub(c.LastObserved)
if quiet <= providerSaysAgainWithin {
continue
}
module, node, consumer, ok := providerOf(c)
if !ok {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, ID: module + "." + node + "." + consumer,
Token: "silent", Kind: kindProviderSilent, Machine: node, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped "+
"saying anything, so its last word is all the mesh has", module, node, consumer,
c.LastObserved.UTC().Format("2006-01-02 15:04 MST")),
Said: fmt.Sprintf("not said again for %s", ago(quiet))})
}
return out
}
func watchAdvisories(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, a := range f.advisories {
if f.now.Sub(a.Last) > advisoryQuiet {
continue
}
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
}
severity := conditions.Warning
times := ""
if a.Count > 1 {
times = fmt.Sprintf(" (%d times since %s)", a.Count, a.First.UTC().Format("15:04 MST"))
}
machine := ""
if a.ID == "controller" {
machine = f.host
}
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
}
return out
}
func watchSelfCheck(f *signalFacts) []conditions.Observation {
every := f.selfCheck.every
if every <= 0 {
every = doctorEvery
}
since := f.selfCheck.last
if since.IsZero() {
since = f.started
}
if f.now.Sub(since) <= 2*every {
return nil
}
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "doctor", Kind: "self-check-silent",
Machine: f.host, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's "+
"invariants are not being checked", since.UTC().Format("2006-01-02 15:04 MST"), every),
Said: fmt.Sprintf("no run for %s", ago(f.now.Sub(since)))}}
}
func watchTools(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, m := range f.machines {
if !m.tools || m.asleep() {
continue
}
bound := heartbeatBound(m.toolsEvery)
since := m.toolsHeard
if since.IsZero() {
// Not heard since this controller started: silent since then at the most.
since = f.toolsHeardFrom
}
silent := f.now.Sub(since)
if silent <= bound {
continue
}
heard := "not since this controller started at " + since.UTC().Format("2006-01-02 15:04 MST")
if !m.toolsHeard.IsZero() {
heard = "since " + m.toolsHeard.UTC().Format("2006-01-02 15:04 MST")
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "tools-silent",
Machine: m.name, Severity: conditions.Warning,
Summary: fmt.Sprintf("the node tools on %s have not said they are there %s (bound %s): nothing can "+
"ask that machine anything", m.name, heard, bound),
Said: fmt.Sprintf("silent for %s", ago(silent))})
}
return out
}
func watchStaleRefusals(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, w := range f.staleRefusals {
if w.Count <= staleRefusalsAllowed {
continue
}
scope, id, machine := conditions.ScopeCore, "controller.unnamed", ""
named := w.Writer
switch {
case w.Epoch > 0:
id = fmt.Sprintf("controller.epoch-%d", w.Epoch)
if e, ok := f.epochs[w.Epoch]; ok {
named = fmt.Sprintf("the controller of epoch %d (%s%s)", w.Epoch, e.Instance, endedWords(e))
}
case w.Writer == link.WriterNodeEngine(firstOf(w.Receivers)) || strings.HasPrefix(w.Writer, "the node-engine on "):
node := strings.TrimPrefix(w.Writer, "the node-engine on ")
scope, id, machine = conditions.ScopeMachine, node, node
}
if machine == "" && len(w.Receivers) > 0 {
machine = w.Receivers[0]
}
var also []string
for _, r := range w.Receivers {
if r != machine && r != "controller" {
also = append(also, r)
}
}
out = append(out, conditions.Observation{Scope: scope, ID: id, Kind: "stale-writer", Token: "stale-writer",
Machine: machine, Also: also, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s was refused %d time(s) in %s as older than what its receivers hold (%s): a "+
"writer is sending what the mesh has moved past", named, w.Count, staleRefusalsWithin,
strings.Join(w.Receivers, ", ")),
Said: fmt.Sprintf("%d stale refusals in %s, the last at %s", w.Count, staleRefusalsWithin,
w.Last.UTC().Format(time.RFC3339))})
}
return out
}
// firstOf is a list's first, empty for none.
func firstOf(list []string) string {
if len(list) == 0 {
return ""
}
return list[0]
}
// endedWords is how an epoch ended, for a sentence naming it; nothing while it is held.
func endedWords(e inventory.Epoch) string {
if e.Ended == nil {
return ", still holding the lease"
}
return fmt.Sprintf(", %s at %s", e.How, e.Ended.UTC().Format("15:04:05 MST"))
}
// watchLease is S12: the lease held and renewed by this controller, and every holder that lost it.
func watchLease(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
l := f.lease
if l.unleased != "" {
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "unleased",
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
Summary: "the controller serves WITHOUT the lease: nothing keeps a second controller from acting " +
"beside it, and its declarations carry no epoch. A bus whose user list is older than this " +
"controller does not grant it the lease's bucket: a push of the machine holding the bus sends " +
"the list that does, and the controller takes the lease within five seconds",
Said: l.unleased})
} else if l.held && !l.renewed.IsZero() && f.now.Sub(l.renewed) > leaseBound {
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "late",
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the controller has not renewed its lease (epoch %d) since %s, past the key's age "+
"of %s: another controller may take it", l.epoch, l.renewed.UTC().Format(time.RFC3339), leaseBound),
Said: fmt.Sprintf("not renewed for %s", ago(f.now.Sub(l.renewed)))})
}
if !l.reset.IsZero() && f.now.Sub(l.reset) <= advisoryQuiet {
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "reset",
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
Summary: "the controller lease's bucket was raised again from nothing: whatever held the lease before " +
"does not know it lost it",
Said: l.resetSaid})
}
var lost []string
newest := inventory.Epoch{}
for _, e := range l.ended {
if e.Ended == nil || e.How == inventory.EpochReleased || f.now.Sub(*e.Ended) > advisoryQuiet {
continue
}
lost = append(lost, fmt.Sprintf("epoch %d (%s) %s at %s", e.Epoch, e.Instance, e.How,
e.Ended.UTC().Format("15:04:05 MST")))
if newest.Ended == nil || e.Ended.After(*newest.Ended) {
newest = e
}
}
if len(lost) > 0 {
how := "lost it: its renewal was refused or could not be made"
if newest.How == inventory.EpochExpired {
how = "stopped renewing it without giving it back, and was taken over"
}
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent,
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
newest.Instance, how, l.epoch),
Said: strings.Join(lost, "; ")})
}
return out
}
// handActsWithin is how far back a repeated cause counts (S15).
const handActsWithin = 14 * 24 * time.Hour
// watchHandActs is S15: a cause recorded by hand twice within a fortnight is a healer wanted, named by
// the cause. **A heal is never a hand act** (healers.go), so a cause a healer exists for and a person
// still repaired twice says the healer is not enough — its reach or its budget — and is said so. **An act
// that is a person's decision by design is no repair** (handActVerbs), so it never counts; one counted
// before this was so clears on the next tick, as any condition the row no longer sees.
func watchHandActs(f *signalFacts) []conditions.Observation {
recent := make([]link.HandAct, 0, len(f.handActs))
for _, a := range repairs(f.handActs) {
if f.now.Sub(a.At) <= handActsWithin {
recent = append(recent, a)
}
}
repeated := link.RepeatedCauses(recent, f.now)
causes := make([]string, 0, len(repeated))
for c := range repeated {
causes = append(causes, c)
}
sort.Strings(causes)
var out []conditions.Observation
for _, cause := range causes {
var acts []string
var newest link.HandAct
for _, a := range recent {
if a.Cause != cause {
continue
}
acts = append(acts, fmt.Sprintf("%s %s by %s: %s", a.At.UTC().Format("2006-01-02 15:04"),
strings.TrimSpace(a.Verb+" "+strings.Join(a.Args, " ")), a.By, a.Why))
if a.At.After(newest.At) {
newest = a
}
}
wanted := "a healer is wanted for it"
if h := healerNamedFor(cause); h != "" {
wanted = fmt.Sprintf("healer %s answers this cause and a person still repaired it: its reach or its "+
"budget is not enough", h)
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "hand-acts." + cause,
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause,
repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted),
Said: strings.Join(acts, "; ")})
}
return out
}
// watchedRows are the rows a watchdog runs for.
func watchedRows() []signalRow {
var out []signalRow
for _, r := range signalsTable {
if r.watch != nil {
out = append(out, r)
}
}
return out
}
// kindsOf is a row's condition kinds, one or several.
func kindsOf(r signalRow) []string {
var out []string
for _, k := range strings.Split(r.Kind, ",") {
out = append(out, strings.TrimSpace(k))
}
return out
}
cmd/mesh-controller/signals_test.go
-552
View File
@@ -1,552 +0,0 @@
package main
import (
"context"
"errors"
"os"
"path/filepath"
"regexp"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
// and the condition clears. A row that is not watched says why. A row added to the table without a
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
// running, the self-check a minute old.
func calm(now time.Time) *signalFacts {
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
mergesPassed: now.Add(-time.Minute),
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
facts: factsFacts{taken: now.Add(-time.Hour), began: now.Add(-time.Hour)},
}
}
// refusedBy is n refusals of one writer, the last a moment ago.
func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals {
return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n,
Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}}
}
// suppression is one row's signal held back: inside its bound, and past it.
type suppression struct{ inside, past func(f *signalFacts) }
// suppressions are every watched row's, by row.
var suppressions = map[string]suppression{
"S1": {
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
},
"S2": {
inside: func(f *signalFacts) {
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
},
past: func(f *signalFacts) {
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
},
},
"S3": {
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
},
"S4": {
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
},
"S5": {
inside: func(f *signalFacts) {},
past: func(f *signalFacts) {
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
},
},
"S6": {
inside: func(f *signalFacts) {
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
},
past: func(f *signalFacts) {
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
},
},
"S7": {
inside: func(f *signalFacts) {
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
},
past: func(f *signalFacts) {
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
},
},
"S8": {
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
},
"S9": {
inside: func(f *signalFacts) {
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
},
past: func(f *signalFacts) {
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
},
},
"S10": {
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
},
"S11": {
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
},
"S12": {
inside: func(f *signalFacts) { f.lease.renewed = f.now.Add(-15 * time.Second) },
past: func(f *signalFacts) { f.lease.renewed = f.now.Add(-16 * time.Second) },
},
"S13": {
inside: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 5) },
past: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 6) },
},
// The snapshot a merge check is fed, older than two days; or none kept by a controller two days up.
"S14": {
inside: func(f *signalFacts) { f.facts.taken = f.now.Add(-47 * time.Hour) },
past: func(f *signalFacts) { f.facts.taken = f.now.Add(-49 * time.Hour) },
},
// A walk waiting for its delivery's word for 30 minutes (novox/hq ADR 0239).
"S16": {
inside: func(f *signalFacts) {
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
since: f.now.Add(-29 * time.Minute)}}
},
past: func(f *signalFacts) {
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
since: f.now.Add(-31 * time.Minute)}}
},
},
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
"S15": {
inside: func(f *signalFacts) {
f.handActs = []link.HandAct{actByHand(f.now.Add(-15*24*time.Hour), "consumer-behind"),
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
},
past: func(f *signalFacts) {
f.handActs = []link.HandAct{actByHand(f.now.Add(-13*24*time.Hour), "consumer-behind"),
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
},
},
}
// actByHand is one entry in the hand-act log, with its cause.
func actByHand(at time.Time, cause string) link.HandAct {
return link.HandAct{ID: "act-" + at.Format("150405"), At: at, By: "jochen at a shell on the laptop",
Verb: "broker consumer-reset", Args: []string{"EVENTS", "controller"}, Why: "it replayed a week", Cause: cause}
}
// **S15 names the cause and, where a healer answers it, that the healer was not enough.**
func TestARepeatedHandActNamesItsCauseAndItsHealer(t *testing.T) {
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
f := calm(now)
f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), "consumer-behind"),
actByHand(now.Add(-time.Hour), "consumer-behind"), actByHand(now.Add(-time.Hour), "restarted the proxy"),
actByHand(now.Add(-time.Minute), "restarted the proxy")}
got := watchHandActs(f)
if len(got) != 2 {
t.Fatalf("%+v", got)
}
if got[0].Key() != "mesh.hand-acts.consumer-behind.healer-wanted" || !strings.Contains(got[0].Summary, "healer H4") {
t.Errorf("a cause a healer answers: %s — %s", got[0].Key(), got[0].Summary)
}
if got[1].Key() != "mesh.hand-acts.restarted_the_proxy.healer-wanted" ||
!strings.Contains(got[1].Summary, "a healer is wanted") {
t.Errorf("a cause no healer answers: %s — %s", got[1].Key(), got[1].Summary)
}
}
// actOf is an act in the log recorded by a verb, with its cause.
func actOf(at time.Time, verb, cause string) link.HandAct {
a := actByHand(at, cause)
a.Verb, a.Args = verb, nil
return a
}
// **An act a person decides by design is no repair** (handActVerbs): approving or rejecting a
// retirement and deleting what was retired (ADR 0230), a bus upgrade and a backlog released (ADR 0236),
// and a rotation after a leak — repeated, none is a healer wanted, whatever cause it gives.
func TestAPersonsDecisionRepeatedWantsNoHealer(t *testing.T) {
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
for _, c := range []struct{ verb, cause string }{
{"retire approve", kindRetireWaiting}, {"retire reject", kindRetireWaiting},
{"cleanup delete", kindCleanupWaiting}, {"bus upgrade", "bus-upgrade"},
{"bus upgrade", "a word the person chose"}, {"upgrade release-backlog", "upgrade release-backlog"},
{"secret rotate", causeLeakedInLogs},
} {
f := calm(now)
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), c.verb, c.cause),
actOf(now.Add(-time.Hour), c.verb, c.cause)}
if got := watchHandActs(f); len(got) != 0 {
t.Errorf("%s (cause %s) repeated asked for a healer: %+v", c.verb, c.cause, got)
}
}
}
// **What repairs still counts**: a push by hand above all (ADR 0236 exists to end it), a rotation for
// any cause but a leak, an act recorded outside the mesh whatever cause it names, and a verb the table
// does not know.
func TestARepairRepeatedStillWantsAHealer(t *testing.T) {
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
for _, c := range []struct{ verb, cause string }{
{"push", "push"}, {"plans close", "plans close"}, {"conditions silence", "consumer-behind"},
{"secret rotate", "stopped-working"}, {"hand-act record", "bus-upgrade"},
{"hand-act record", kindCleanupWaiting}, {"a verb nobody listed", "x"},
} {
f := calm(now)
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), c.verb, c.cause),
actOf(now.Add(-time.Hour), c.verb, c.cause)}
if got := watchHandActs(f); len(got) != 1 || got[0].Kind != "healer-wanted" {
t.Errorf("%s (cause %s) repeated wanted no healer: %+v", c.verb, c.cause, got)
}
}
// A decision does not make up the second of a cause a repair recorded once.
f := calm(now)
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act record", "bus-upgrade"),
actOf(now.Add(-time.Hour), "bus upgrade", "bus-upgrade")}
if got := watchHandActs(f); len(got) != 0 {
t.Errorf("one repair and one decision asked for a healer: %+v", got)
}
}
// **A healer-wanted already open for a decision clears on the next tick** — the log of 2026-10-06:
// a bus upgrade recorded after the fact and one through its verb, and two rotations after a leak.
func TestAHealerWantedOpenForADecisionClearsOnTheNextTick(t *testing.T) {
now := time.Date(2026, 10, 6, 18, 0, 0, 0, time.UTC)
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
defer k.Close(context.Background())
// What the build before this one raised, as it raised it.
var before []conditions.Observation
for _, cause := range []string{"bus-upgrade", causeLeakedInLogs} {
before = append(before, conditions.Observation{Scope: conditions.ScopeMesh, ID: "hand-acts." + cause,
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning, Summary: "repaired twice"})
}
if err := k.Reconcile(t.Context(), "S15", before); err != nil {
t.Fatal(err)
}
if open, _ := k.Open(t.Context()); len(open) != 2 {
t.Fatalf("the conditions of the build before were not open: %+v", open)
}
f := calm(now)
f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", "bus-upgrade"),
actOf(now.Add(-8*time.Hour), "secret rotate", causeLeakedInLogs),
actOf(now.Add(-8*time.Hour), "secret rotate", causeLeakedInLogs),
actOf(now.Add(-time.Hour), "bus upgrade", "bus-upgrade")}
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
w.see(t.Context(), f)
if open, _ := k.Open(t.Context()); len(open) != 0 {
t.Fatalf("a healer-wanted for a person's decision stayed open: %+v", open)
}
}
// **Every verb that writes the hand-act log is in handActVerbs**, so whether it is a repair is said
// where S15 reads it, not left to a default nobody chose.
func TestEveryVerbThatRecordsAHandActIsInTheTable(t *testing.T) {
listed := map[string]bool{}
for _, v := range handActVerbs {
if listed[v.Verb] {
t.Errorf("%s is in the table twice", v.Verb)
}
listed[v.Verb] = true
if len(v.DecidedFor) > 0 && v.Decision == "" {
t.Errorf("%s limits a decision it does not state", v.Verb)
}
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
literal := regexp.MustCompile(`(?:\.record\(ctx, |HandAct\{Verb: |RetireApproved: |RetireRejected: |RetireDeleted: )"([^"]+)"\s*[,})]`)
composed := regexp.MustCompile(`\.record\(ctx, "([^"]+ )"\s*\+`)
found := 0
for _, name := range files {
if strings.HasSuffix(name, "_test.go") {
continue
}
body, err := os.ReadFile(name)
if err != nil {
t.Fatal(err)
}
for _, m := range literal.FindAllStringSubmatch(string(body), -1) {
found++
if !listed[m[1]] {
t.Errorf("%s records %q, which handActVerbs does not list", name, m[1])
}
}
// "plans " + stop|close, "retire " + approve|reject: some listed verb begins with it.
for _, m := range composed.FindAllStringSubmatch(string(body), -1) {
found++
if !slices.ContainsFunc(handActVerbs, func(v handActVerb) bool { return strings.HasPrefix(v.Verb, m[1]) }) {
t.Errorf("%s records %q…, which no verb of handActVerbs begins with", name, m[1])
}
}
}
if found < 12 {
t.Fatalf("found %d recording verbs, fewer than the table's own: the search no longer sees them", found)
}
}
// standingSaid is a provider's failing word last said at a moment.
func standingSaid(at time.Time) conditions.Condition {
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
}
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
seen := map[string]bool{}
for _, row := range signalsTable {
t.Run(row.Row, func(t *testing.T) {
if seen[row.Row] {
t.Fatalf("%s is in the table twice", row.Row)
}
seen[row.Row] = true
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
}
if row.Deferred != "" {
if row.watch != nil || row.Phase <= 1 {
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
}
if _, has := suppressions[row.Row]; has {
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
}
return
}
if row.watch == nil || row.needs == nil || row.newest == nil {
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
}
s, ok := suppressions[row.Row]
if !ok {
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
}
if got := row.watch(calm(now)); len(got) != 0 {
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
}
inside := calm(now)
s.inside(inside)
if got := row.watch(inside); len(got) != 0 {
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
}
past := calm(now)
s.past(past)
got := row.watch(past)
if len(got) == 0 {
t.Fatalf("%s raised nothing past its bound", row.Row)
}
for _, o := range got {
if !slices.Contains(kindsOf(row), o.Kind) {
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
}
if o.Severity != row.Severity {
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
}
if strings.TrimSpace(o.Summary) == "" {
t.Errorf("%s raised a condition that says nothing", row.Row)
}
}
// Through the store: raised past the bound, cleared when the signal returns.
store := conditions.NewInMemory()
told := &conditions.Told{}
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
Now: func() time.Time { return now }})
defer k.Close(context.Background())
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
w.see(t.Context(), past)
open, err := k.Open(t.Context())
if err != nil || len(open) != len(got) {
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
}
w.see(t.Context(), calm(now))
if open, _ := k.Open(t.Context()); len(open) != 0 {
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
}
})
}
for name := range suppressions {
if !seen[name] {
t.Errorf("a suppression for %s, which the table does not have", name)
}
}
}
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
now := time.Now()
f := calm(now)
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
got := watchHeartbeats(f)
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
t.Fatalf("%+v", got)
}
}
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
now := time.Now()
f := calm(now)
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
t.Fatalf("a sleeping machine raised %+v", got)
}
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
if got := watchHeartbeats(f); len(got) != 1 {
t.Fatalf("a woken machine silent past its bound raised %+v", got)
}
}
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
now := time.Now()
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
silent := calm(now)
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
w.see(t.Context(), silent)
blind := calm(now)
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
// Blind for one tick is a read that did not answer in time; for two in a row, a watchdog that cannot
// see (novox/hq issue 277).
w.see(t.Context(), blind)
if open, _ := k.Open(t.Context()); len(open) != 1 {
t.Fatalf("one blind tick raised %+v", open)
}
w.see(t.Context(), blind)
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
var keys []string
for _, c := range open {
keys = append(keys, c.Key)
}
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
if !slices.Contains(keys, want) {
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
}
}
w.see(t.Context(), calm(now))
if open, _ := k.Open(t.Context()); len(open) != 0 {
t.Fatalf("seeing again left open %+v", open)
}
}
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
// every machine silent.
func TestAControllerStandingBySaysNothing(t *testing.T) {
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
defer k.Close(context.Background())
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
w.tick(t.Context())
if w.lastTick().IsZero() {
t.Fatal("a tick standing by was not counted")
}
if open, _ := k.Open(t.Context()); len(open) != 0 {
t.Fatalf("%+v", open)
}
}
// **A stale writer is named** (novox/hq to-be 45 §3, S13): by the controller instance that held the epoch
// its refused declarations claimed, and how that epoch ended — the question issue 204 could not answer.
func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
now := time.Now()
f := calm(now)
ended := now.Add(-time.Minute)
f.staleRefusals = refusedBy(now, 41, 6)
f.epochs[41] = inventory.Epoch{Epoch: 41, Instance: "controller@anchor pid 7 since 2026-10-06T10:00:00Z",
Host: "anchor", Ended: &ended, How: inventory.EpochExpired}
got := watchStaleRefusals(f)
if len(got) != 1 || got[0].Key() != "core.controller.epoch-41.stale-writer" ||
!strings.Contains(got[0].Summary, "pid 7") || !strings.Contains(got[0].Summary, "expired") ||
got[0].Machine != "anchor" || !slices.Equal(got[0].Also, []string{"laptop"}) {
t.Fatalf("the stale writer is not named: %+v", got)
}
// An account the controller refused names the machine whose node-engine sent it.
f.staleRefusals = []link.WriterRefusals{{Writer: link.WriterNodeEngine("laptop"), Count: 6,
Receivers: []string{"controller"}, Last: now}}
got = watchStaleRefusals(f)
if len(got) != 1 || got[0].Key() != "machine.laptop.stale-writer" || got[0].Machine != "laptop" {
t.Fatalf("a node-engine sending older accounts is not named: %+v", got)
}
}
// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
now := time.Now()
f := calm(now)
expired := now.Add(-59 * time.Minute)
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
Ended: &expired, How: inventory.EpochExpired}}
got := watchLease(f)
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
got[0].Severity != conditions.Urgent {
t.Fatalf("a holder that stopped renewing was not said: %+v", got)
}
long := now.Add(-61 * time.Minute)
f.lease.ended[0].Ended = &long
if got := watchLease(f); len(got) != 0 {
t.Fatalf("a loss an hour old is still said: %+v", got)
}
f.lease.ended[0].How, f.lease.ended[0].Ended = inventory.EpochReleased, &expired
if got := watchLease(f); len(got) != 0 {
t.Fatalf("a lease given back is said as lost: %+v", got)
}
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
}
f.lease = leaseFacts{unleased: "the bus refused the key"}
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.unleased" {
t.Fatalf("serving without the lease was not said: %+v", got)
}
}
// **A walk waiting four hours for its delivery's word is urgent** (S16, novox/hq ADR 0239), and says how on.
func TestAWalkWaitingFourHoursIsUrgentAndNamesTheWayOn(t *testing.T) {
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
f := calm(now)
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
since: now.Add(-4*time.Hour - time.Minute)}}
got := watchWaits(f)
if len(got) != 1 || got[0].Severity != conditions.Urgent || got[0].Key() != "plan.plan-2.waiting" {
t.Fatalf("four hours of waiting raised %+v", got)
}
if !strings.Contains(got[0].Summary, "plans go plan-2") || !strings.Contains(got[0].Summary, "mesh-delivery") {
t.Fatalf("it does not say how on: %q", got[0].Summary)
}
}
cmd/mesh-controller/source.go
+3 -13
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"fmt"
"os"
"strconv"
"strings"
@@ -135,28 +134,19 @@ func seatBase(world catalogue.World, seatName string) (string, error) {
// seatBases is the clone base of every seat a recipe's context may name, for a build request
// (novox/hq ADR 0155). A seat nobody holds is left out rather than refused here: the build may not
// name it at all, and if it does the builder refuses with the seat's name.
//
// **What cannot be read is said, not passed over as no seats** (novox/hq to-be 45 Phase 2, the
// empty-on-error lint): the build still goes ahead — one that names no seat needs none — and one that
// does is refused naming it, but the reason is the store, and that is said here where it is known.
func seatBases(ctx context.Context) map[string]string {
unread := func(what string, err error) map[string]string {
fmt.Fprintf(os.Stderr, "could not read %s, so a build naming a seat's clone base will be told that "+
"seat is not held: %v\n", what, err)
return nil
}
open, err := openStores(ctx)
if err != nil {
return unread("the mesh's store", err)
return nil
}
defer open.Close()
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return unread("the catalogue", err)
return nil
}
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
if err != nil {
return unread("who holds which seat", err)
return nil
}
bases := map[string]string{}
for _, seatName := range []string{gitSeat} {
cmd/mesh-controller/standing.go
-189
View File
@@ -1,189 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224) —
// **the condition store's first kind** (to-be 45 §2, ADR 0227).
//
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
// provider announces a consumer it has failed for minutes; the controller keeps it until the provider
// says it recovered; and `status`, its JSON and `node show` name it, breaking "all well". Unchanged in
// what it says and when; kept as a condition, `provider.<module>.<node>.<consumer>.failing`, rather
// than a row of its own, so it is said outward like every other fault and silenced like one.
// Kinds of the provider standing.
const (
kindProviderFailing = "provider-failing"
kindProviderSilent = "provider-silent"
// sourceProvisioner is what raised a standing: the provider's own event.
sourceProvisioner = "provisioner.failing"
)
// providerSaysAgainWithin is how long a failing word stays current without being said again: twice
// the quarter of an hour a provider repeats it at (ADR 0224). Past it, S8.
const providerSaysAgainWithin = 30 * time.Minute
// standings keeps what providers say, as conditions.
type standings struct {
keeper func() *conditions.Keeper
}
// standingObservation is a provider's failing word as an observation: the provider, its machine and
// the consumer name it, so the same consumer failed again is the same condition.
func standingObservation(st link.Standing) conditions.Observation {
whom := st.Consumer
if st.Node != "" {
whom += " on " + st.Node
}
summary := fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode,
whom, orUnclassed(st.Class), st.Attempts, st.Since.UTC().Format("2006-01-02 15:04 MST"))
said := orUnclassed(st.Class)
if e := firstLine(st.Error); e != "" {
said += ": " + e
}
if st.Provider != "" {
said += fmt.Sprintf(" (provision %s, %d attempts)", st.Provider, st.Attempts)
}
return conditions.Observation{Scope: conditions.ScopeProvider,
ID: st.Module + "." + st.ProviderNode + "." + st.Consumer,
Token: "failing", Kind: kindProviderFailing, Machine: st.ProviderNode, Also: alsoOn(st.Node, st.ProviderNode),
Severity: conditions.Warning, Summary: summary, Said: said, Source: sourceProvisioner}
}
// Stood keeps a provider's newest word: failing raises or observes its condition, recovered clears
// it. An error is the store away, and the link holds the message to be asked again — a recovery is
// said once, and dropping it would leave a consumer named failing that is fine.
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
k := s.keeper()
if k == nil {
return false, fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
}
o := standingObservation(st)
if !st.Failing {
why := "the provider says it recovered"
if st.Why != "" {
why += ": " + st.Why
}
cleared, err := k.Clear(ctx, o.Key(), why)
return cleared, storeAway(err)
}
_, err := k.Observe(ctx, o)
return false, storeAway(err)
}
// storeAway reads the condition store failing as the bus being away for the moment: the link holds the
// message and asks again, as it does for a store restarting (ADR 0083), rather than taking it unkept.
func storeAway(err error) error {
if err == nil {
return nil
}
return fmt.Errorf("%v: %w", err, link.ErrTryAgain)
}
// providerStandings is every open provider-failing condition, from what is open.
func providerStandings(open []conditions.Condition) []conditions.Condition {
var out []conditions.Condition
for _, c := range open {
if c.Kind == kindProviderFailing {
out = append(out, c)
}
}
return out
}
// providerConditions is every open condition a provider's word raised: a consumer failing (ADR 0224),
// and a retirement waiting for a person, kept by a rejection, or cleanup waiting (ADR 0230).
func providerConditions(open []conditions.Condition) []conditions.Condition {
var out []conditions.Condition
for _, c := range open {
switch c.Kind {
case kindProviderFailing, kindRetireWaiting, kindRetireRejected, kindCleanupWaiting:
out = append(out, c)
}
}
return out
}
// providerOf reads a provider condition's module and machine back from its key: a standing's
// `provider.<module>.<node>.<consumer>.failing`, and a retirement's `provider.<module>.<node>.<token>`,
// which names no consumer.
func providerOf(c conditions.Condition) (module, node, consumer string, ok bool) {
parts := strings.Split(c.Key, ".")
if parts[0] != conditions.ScopeProvider {
return "", "", "", false
}
switch len(parts) {
case 5:
return parts[1], parts[2], parts[3], true
case 4:
return parts[1], parts[2], "", true
}
return "", "", "", false
}
// unassignedProviders clears the standing of every provider no longer assigned where it ran — and its
// retirement and cleanup conditions with it (ADR 0230): nothing runs there to retire or delete anything.
//
// **A provider no longer assigned is not asked about** (ADR 0224 §4): nothing runs there to fail
// anybody, and nothing there will ever say it recovered. The observation that resolves it is the
// assignment. Assigned again, its first failure raises it again.
func unassignedProviders(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper,
open []conditions.Condition) error {
assigned := map[string]map[string]bool{}
for _, c := range providerConditions(open) {
module, node, _, ok := providerOf(c)
if !ok {
continue
}
on, asked := assigned[node]
if !asked {
modules, err := inv.Assigned(ctx, node)
if err != nil {
if errors.Is(err, inventory.ErrNoSuchNode) {
modules = nil // a machine the mesh no longer knows runs nothing
} else {
return fmt.Errorf("what %s is assigned cannot be read: %w", node, err)
}
}
on = map[string]bool{}
for _, m := range modules {
on[m] = true
}
assigned[node] = on
}
if !on[module] {
if _, err := k.Clear(ctx, c.Key, module+" is no longer assigned to "+node+
": nothing runs there to fail anybody"); err != nil {
return err
}
}
}
return nil
}
func orUnclassed(class string) string {
if class == "" {
return "failing"
}
return class
}
// alsoOn is a consumer's machine, when it is not the provider's.
func alsoOn(consumerNode, providerNode string) []string {
if consumerNode == "" || consumerNode == providerNode {
return nil
}
return []string{consumerNode}
}
cmd/mesh-controller/standing_test.go
-118
View File
@@ -1,118 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224), kept as
// the condition store's first kind (to-be 45 §2). On 2026-10-05 the identity provider refused every
// consumer for a day and status called the mesh well (04-ISSUES/179): this is that day, told to the
// controller the way the provider now tells it.
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
t.Fatal(err)
}
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
since := time.Now().Add(-23 * time.Hour)
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
if _, err := kept.Stood(ctx, failing); err != nil {
t.Fatal(err)
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if asked.well() {
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
}
said := printed(t, func() error { return printStatus(asked) })
for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing",
"idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} {
if !strings.Contains(said, want) {
t.Fatalf("status does not say %q:\n%s", want, said)
}
}
if strings.Contains(said, "all doing what they were told") {
t.Fatalf("status said all well beside a failing provider:\n%s", said)
}
if !strings.HasPrefix(said, "1 open condition(s)") {
t.Fatalf("status does not lead with what is open:\n%s", said)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
Conditions []conditions.Condition `json:"conditions"`
Failing []conditions.Condition `json:"failing"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 ||
!strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") {
t.Fatalf("the document does not carry it: %v\n%s", err, body)
}
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
for _, node := range []string{"anchor", "laptop"} {
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
t.Fatalf("node show %s does not name it:\n%s", node, shown)
}
}
// Recovered: gone, and the mesh may be well again as far as this is concerned.
failing.Failing = false
if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
t.Fatalf("%v %v", cleared, err)
}
asked, err = theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(asked.conditions) != 0 {
t.Fatalf("a recovered consumer is still named: %+v", asked.conditions)
}
}
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
// cleared on the next look, said as resolved by the assignment.
func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true,
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
t.Fatal(err)
}
all, err := conditionsFrom.Open(ctx)
if err != nil || len(all) != 1 {
t.Fatalf("%+v %v", all, err)
}
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil {
t.Fatal(err)
}
if all, _ := conditionsFrom.Open(ctx); len(all) != 0 {
t.Fatalf("%+v", all)
}
}
// **The store away holds a recovery** (ADR 0224 §3): a standing that cannot be kept is asked again.
func TestAStandingTheStoreCannotKeepIsAskedAgain(t *testing.T) {
kept := standings{keeper: func() *conditions.Keeper { return nil }}
if _, err := kept.Stood(t.Context(), link.Standing{Module: "idp", ProviderNode: "anchor", Consumer: "x"}); err == nil ||
!strings.Contains(err.Error(), link.ErrTryAgain.Error()) {
t.Fatalf("answered %v", err)
}
}
cmd/mesh-controller/status.go
+6 -74
View File
@@ -8,7 +8,6 @@ import (
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
@@ -82,12 +81,8 @@ func printStatus(asked answers) error {
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
behind, sources := asked.behind, asked.sources
// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
// first, silenced ones with when their silence ends.
printConditions(asked.conditions, asked.conditionsUnread, time.Now())
if len(asked.refused) > 0 {
// First of what follows. A machine that cannot be worked out is not running an old
// First, above everything else. A machine that cannot be worked out is not running an old
// declaration — it has no declaration, and nothing below this line is about it.
var names []string
for name := range asked.refused {
@@ -303,38 +298,6 @@ func printStatus(asked answers) error {
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
}
if len(asked.overflowing) > 0 {
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
// out of its provider's grants, so the module holds a login nothing created; the provider's
// machine is sent everything else rather than refused for one consumer elsewhere.
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
len(asked.overflowing))
for _, o := range asked.overflowing {
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
}
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
}
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
switch {
case asked.handActsUnread != "":
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
asked.handActsUnread)
case asked.handActs != nil && *asked.handActs > 0:
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
}
// And what the mesh repaired by itself (novox/hq to-be 45 §7): seen, not only done.
switch {
case asked.healsUnread != "":
fmt.Printf("what the healers did could not be read: %s\n\n", asked.healsUnread)
case asked.heals != nil && (asked.heals.Acts > 0 || asked.heals.Escalated > 0):
fmt.Printf("%d repair(s) made by the healers in the last seven days, %d handed to the operator — `healers` "+
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
}
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
// Said, because nothing forces the flip: a node left adopted is visible here rather than
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -344,9 +307,8 @@ func printStatus(asked answers) error {
if asked.well() {
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
// and getting here means every question was asked and answered. **No open conditions first**
// (novox/hq to-be 45 §2): it is what the sentence means now, silenced ones included.
fmt.Printf("no open conditions; %d machine(s), all doing what they were told, all heard from, running what "+
// and getting here means every question was asked and answered.
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
"the mesh would send them, and every module current with its source\n", len(nodes))
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
@@ -444,27 +406,15 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
// the private network is built from and should say nothing else; a machine that does not
// resolve is already in refused, and is passed over here.
plans := map[string]planned{}
for _, n := range out.nodes {
plan, settings, err := planFor(ctx, open, n.Name)
plan, _, err := planFor(ctx, open, n.Name)
if err != nil {
if unresolvable(err) {
continue
}
return answers{}, err
}
plans[n.Name] = planned{plan, settings}
out.unheld = append(out.unheld, plan.Unheld...)
// And which of its modules a provider leaves out of its grants, for an identity too long
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
// resolution, as the provider's composition judges it.
out.overflowing = append(out.overflowing, plan.Overflowing()...)
}
// And every open condition (novox/hq to-be 45 §2): what the watchdogs, the self-check and the
// providers' own words say is wrong — a provider failing a consumer among them (ADR 0224). Kept on
// the bus; a process that cannot read them says so, and the mesh is then not called well.
if out.conditions, err = openConditions(ctx); err != nil {
out.conditionsUnread = err.Error()
}
out.plans, err = inv.RecentPlans(ctx, 5)
if err != nil {
@@ -472,23 +422,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
}
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
out.paused = buildSeatPause(ctx, inv, out.plans)
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
// to read the log from; a process with none has no log to count.
if _, onBus := broker.BusAddress(); onBus == nil {
n, unread := handActsThisWeek(ctx)
if unread != "" {
out.handActsUnread = unread
} else {
out.handActs = &n
}
}
// And what the healers did this week (novox/hq to-be 45 §7).
if heals, err := inv.HealsSince(ctx, time.Now().Add(-7*24*time.Hour)); err != nil {
out.healsUnread = err.Error()
} else {
out.heals = countHeals(heals)
}
// And which machines are not running what the mesh would send them. The same question as a
// module being behind its source, one level down: that one says the catalogue is out of date,
@@ -500,7 +433,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
// reason is kept and reported as data; every question that does not depend on it is still
// answered.
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
would, err := wouldSend(ctx, open, out.nodes)
if err != nil {
out.network = err.Error()
would = map[string]string{}
@@ -595,8 +528,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
len(a.conditions) == 0 && a.conditionsUnread == ""
len(a.filtered) == 0 && len(a.unheld) == 0
}
// hostSplit is which machines report which host version, for every version more than one machine
cmd/mesh-controller/status_summary.go
-213
View File
@@ -1,213 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/novox/mesh-controller/internal/link"
)
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
// §4's D9 and §8's health of the controller).
//
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
// controller composes it in the background — at its start, after anything that changes what it says
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
// the last composition at once, saying when it was composed and how long that took. A caller who
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
// long a nudge waits for the next, so a push answered by four machines is composed once.
var (
statusEvery = time.Minute
statusSettle = 2 * time.Second
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
// good; the attempt is said as failed, and the last summary stands with its age.
statusComposeWithin = 2 * time.Minute
)
// statusSummary is the last composed `status --json`, and when.
type statusSummary struct {
compose func(context.Context) ([]byte, error)
// every, settle and within are the clocks above, read once when it is made.
every, settle, within time.Duration
mu sync.Mutex
body []byte
composedAt time.Time
took time.Duration
failed string
failedAt time.Time
started time.Time
first chan struct{} // closed when the first attempt ends, either way
nudged chan struct{}
}
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
}
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
// composed when asked, as at a shell.
var statusFrom *statusSummary
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
func (s *statusSummary) nudge() {
if s == nil {
return
}
select {
case s.nudged <- struct{}{}:
default:
}
}
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
func (s *statusSummary) keep(ctx context.Context) {
once := sync.Once{}
for {
s.composeOnce(ctx)
once.Do(func() { close(s.first) })
timer := time.NewTimer(s.every)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
case <-s.nudged:
timer.Stop()
// Let what else is arriving arrive, then compose once for all of it.
select {
case <-ctx.Done():
return
case <-time.After(s.settle):
}
select {
case <-s.nudged:
default:
}
}
}
}
func (s *statusSummary) composeOnce(ctx context.Context) {
start := time.Now()
asking, cancel := context.WithTimeout(ctx, s.within)
body, err := s.compose(asking)
cancel()
took := time.Since(start)
s.mu.Lock()
defer s.mu.Unlock()
if err != nil {
s.failed, s.failedAt = err.Error(), time.Now()
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
"with its age\n", took.Round(time.Millisecond), err)
return
}
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
}
// answer is what the `status` verb answers: the last summary at once, the same document `status
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
// but never past the caller's window; a controller that has none says so and why, rather than
// answering an empty mesh as a well one.
func (s *statusSummary) answer(ctx context.Context) (any, error) {
wait := time.NewTimer(link.AnswerWithin - time.Second)
defer wait.Stop()
select {
case <-s.first:
case <-wait.C:
case <-ctx.Done():
}
s.mu.Lock()
defer s.mu.Unlock()
if s.body == nil {
why := "its first composition has not finished"
if s.failed != "" {
why = "it could not be composed: " + s.failed
}
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
}
var parsed any
_ = json.Unmarshal(s.body, &parsed)
out := map[string]any{
"output": string(s.body), "ok": true, "answer": parsed,
"composed": s.composedAt.UTC().Format(time.RFC3339),
"age": time.Since(s.composedAt).Round(time.Second).String(),
"composedIn": s.took.Round(time.Millisecond).String(),
"note": "composed by the serving controller at its start, after each report, build or act, and " +
"every minute; answered at once from the last composition",
}
if s.failed != "" && s.failedAt.After(s.composedAt) {
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
s.failedAt.UTC().Format(time.RFC3339), s.failed)
}
return out, nil
}
// composeStatus is `status --json`, composed in this process against its stores.
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
return func(ctx context.Context) ([]byte, error) {
asked, err := theThreeQuestions(ctx, open)
if err != nil {
return nil, err
}
return statusAsJSON(asked)
}
}
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
// controller composing for ever.
var readingVerbs = map[string]bool{
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
"doctor": true, "conditions": true, "healers": true,
}
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
type nudgingListener struct {
link.Enrolment
summary *statusSummary
// open is the serving controller's stores, for replacing a given value after a module's first
// good start (novox/hq ADR 0228).
open *stores
}
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
// A declaration refused as older than the one the machine holds is counted by its writer — the
// controller epoch it claimed (novox/hq to-be 45 §6, S13) — and so is what the machine's own count
// says it refused beyond the refusals heard.
now := time.Now()
if report.StaleRefusalOf() {
link.StaleRefusals.Refused(link.Refusal{Writer: link.WriterEpoch(report.Epoch), Epoch: report.Epoch,
Receiver: report.Node, At: now})
}
if report.Ordered() {
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
}
// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
if report.Superseded == "" && report.Rekey == nil && report.Node != "" {
witnessed.heard(report.Node, report.Rollbacks, now)
}
news, err := l.Enrolment.Heard(ctx, report)
if news {
l.summary.nudge()
}
if err == nil && l.open != nil && startedWell(report) {
// Off the report's path: replacing a given value sends the machine, and a report waits for
// nothing it caused (novox/hq ADR 0228).
go replaceGiven(context.WithoutCancel(ctx), l.open, report)
}
return news, err
}
cmd/mesh-controller/status_summary_test.go
-152
View File
@@ -1,152 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/link"
)
// quickly shortens the summary's clocks for one test.
func quickly(t *testing.T) {
t.Helper()
every, settle := statusEvery, statusSettle
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
}
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
// verb answered "still running"; from the summary it answers at once, in full, saying when.
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
quickly(t)
var composed atomic.Int32
slow := make(chan struct{})
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
if composed.Add(1) > 1 {
<-slow // every composition after the first outlasts any caller
}
return []byte(`{"wrong":[],"machines":4}`), nil
})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
defer close(slow)
go s.keep(ctx)
for i := 0; i < 5; i++ {
s.nudge() // a composition is under way and does not finish
start := time.Now()
got, err := s.answer(ctx)
if err != nil {
t.Fatal(err)
}
if took := time.Since(start); took > time.Second {
t.Fatalf("answer %d took %s", i+1, took)
}
m := got.(map[string]any)
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
t.Fatalf("answer %d was not in full: %v", i+1, m)
}
}
}
// The first composition is waited for, never past the caller's window; a controller with none yet
// says so rather than answering an empty mesh as a well one.
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
quickly(t)
was := link.AnswerWithin
link.AnswerWithin = 1100 * time.Millisecond
t.Cleanup(func() { link.AnswerWithin = was })
never := make(chan struct{})
defer close(never)
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go s.keep(ctx)
start := time.Now()
_, err := s.answer(ctx)
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
t.Fatalf("answered %v", err)
}
if took := time.Since(start); took > link.AnswerWithin {
t.Fatalf("waited %s, past the caller's window", took)
}
}
// A nudge composes it again, once for several close together; a failed composition leaves the last
// summary standing and says it is the last that could be composed.
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
quickly(t)
var composed atomic.Int32
fail := atomic.Bool{}
s := newStatusSummary(func(context.Context) ([]byte, error) {
n := composed.Add(1)
if fail.Load() {
return nil, errors.New("the store did not answer")
}
body, _ := json.Marshal(map[string]any{"n": n})
return body, nil
})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go s.keep(ctx)
if _, err := s.answer(ctx); err != nil {
t.Fatal(err)
}
s.nudge()
s.nudge()
s.nudge()
waitFor(t, func() bool { return composed.Load() == 2 })
time.Sleep(50 * time.Millisecond)
if n := composed.Load(); n != 2 {
t.Fatalf("three nudges together composed %d times after the first", n-1)
}
fail.Store(true)
s.nudge()
waitFor(t, func() bool { return composed.Load() == 3 })
waitFor(t, func() bool {
got, err := s.answer(ctx)
if err != nil {
t.Fatal(err)
}
m := got.(map[string]any)
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
})
}
// The seat's `status` answers from the summary when this process keeps one.
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
quickly(t)
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go s.keep(ctx)
statusFrom = s
t.Cleanup(func() { statusFrom = nil })
handlers, _, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
t.Fatalf("answered %v", got)
}
}
func waitFor(t *testing.T, ok func() bool) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for !ok() {
if time.Now().After(deadline) {
t.Fatal("never happened")
}
time.Sleep(5 * time.Millisecond)
}
}
cmd/mesh-controller/stores.go
-18
View File
@@ -73,22 +73,6 @@ func migrate(ctx context.Context) error {
}
fmt.Printf("provided %s\n", m.Module)
}
// And what an earlier release shipped and this one does not goes (novox/hq ADR 0226) — unless a
// machine still has it, which is said rather than overridden.
var shipped []string
for _, m := range provided {
shipped = append(shipped, m.Module)
}
retired, kept, err := inv.RetireUnshipped(ctx, shipped)
if err != nil {
return err
}
for _, name := range retired {
fmt.Printf("retired %s: the control plane no longer ships it\n", name)
}
for name, why := range kept {
fmt.Printf("kept %s, which the control plane no longer ships: %s\n", name, why)
}
// The seats the mesh ships with, into the table that now holds the set (novox/hq ADR 0122).
// Idempotent: fills an empty table on first boot, adds a seat a release ships, and leaves an
// operator's changes in the table as they are.
@@ -110,8 +94,6 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
inv.Close()
return nil, err
}
// A plan is written only under the lease, carrying its epoch (novox/hq to-be 45 §6).
inv.ActsUnder(theLease.epoch)
// Load the seat set from the store, so the control plane reads the set as data rather than as
// the slice it was compiled with (novox/hq ADR 0122). A store not yet seeded — or one whose
// seat table a migration has not reached — returns nothing, and UseSeats leaves the compiled
cmd/mesh-controller/supersede_test.go
+6 -10
View File
@@ -72,28 +72,24 @@ func TestAPersonClosesAStuckPlan(t *testing.T) {
stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
if err := open.inventory.SavePlan(ctx, &stuck); err != nil {
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
t.Fatal(err)
}
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
t.Fatalf("a plan was closed by hand without saying why: %v", err)
}
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
t.Fatal(err)
}
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
if err != nil {
t.Fatal(err)
}
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
!strings.Contains(closed.Note, "its report will not come") {
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
}
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
t.Fatal("a plan already closed was closed again")
}
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
}
}
cmd/mesh-controller/tier_send_test.go
-375
View File
@@ -1,375 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// A plan's tier is sent to each machine once, and judged there by one gate (novox/hq issue 281).
//
// On 2026-10-06 a merge rebuilt about seventy modules in one tier, and the plan sent the first machine
// one module at a time — one whole declaration per module, seconds apart. The node-engine set aside one
// declaration for the next, the node tools never settled, the mesh said the machine's core was behind,
// and the gate failed the module it happened to be judging for that machine-level condition; it then
// could not put the module back, having read as "the build it ran before" the very build an earlier send
// of the same tier had already carried there.
// tierMesh is a mesh with every module of one tier running on anchor and laptop at build c1, a newer
// build c2 of each registered, a plan whose one tier built them all, and every send recorded and
// answered: the machine applies what it is sent and reports it. A judging reads the store's reports, the
// node tools as answering everywhere, and the conditions the test opens.
type tierMesh struct {
open *stores
modules []string
sent [][]string
conds []conditions.Condition
keeper *conditions.Keeper
}
func aTierMesh(t *testing.T, modules ...string) *tierMesh {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
tm := &tierMesh{open: open, modules: modules}
tm.keeper, _ = withConditionsInMemory(t)
was := doctorFrom
doctorFrom = &doctor{open: open, keeper: tm.keeper, teller: &conditions.Told{}}
t.Cleanup(func() { doctorFrom = was })
register := func(m, commit string, asked time.Time) {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, BuiltFrom: commit,
Head: commit, Asked: asked}); err != nil {
t.Fatal(err)
}
}
first := map[string]string{}
for _, m := range modules {
for i, commit := range []string{"c1", "c2"} {
at := time.Now().Add(-2 * time.Hour)
if commit == "c2" {
at = time.Now().Add(-time.Minute)
}
b := inventory.Build{ID: fmt.Sprintf("build-%s-%d", m, i+1), Module: m, Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/" + m, Asked: at, At: at}
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
register(m, "c1", time.Now().Add(-2*time.Hour))
first[m] = "c1"
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, n, m); err != nil {
t.Fatal(err)
}
}
}
for _, n := range []string{"anchor", "laptop"} {
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", first); err != nil {
t.Fatal(err)
}
}
for _, m := range modules {
register(m, "c2", time.Now().Add(-time.Minute))
}
n := 0
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
tm.sent = append(tm.sent, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
builds := map[string]string{}
for _, m := range tm.modules {
builds[m] = current[m].Commit
}
for _, node := range names {
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
wasGather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, judged: true,
open: append([]conditions.Condition(nil), tm.conds...)}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
f.reports[r.Node] = r
f.served[r.Node] = served{runtime: true, tools: map[string]bool{}}
}
return f, nil
}
t.Cleanup(func() { gatherGateFacts = wasGather })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
built := time.Now().UTC()
states := map[string]*inventory.PlanModule{}
for _, m := range modules {
states[m] = &inventory.PlanModule{State: "built", BuiltAt: &built, Commit: "c2", Build: "build-" + m + "-2"}
}
plan := inventory.Plan{ID: "plan-tier", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{modules}, Modules: states}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
return tm
}
func (tm *tierMesh) plan(t *testing.T) inventory.Plan {
t.Helper()
p, err := tm.open.inventory.PlanByID(t.Context(), "plan-tier")
if err != nil {
t.Fatal(err)
}
return p
}
// raise opens a condition about a machine itself, raised now: after the send.
func (tm *tierMesh) raise(machine, kind, summary string) {
tm.conds = append(tm.conds, conditions.Condition{Key: "machine." + machine + "." + kind, Kind: kind,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, Machine: machine},
Raised: time.Now().Add(time.Second), Summary: summary, Source: "D10"})
}
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
var modules []string
for i := 1; i <= 12; i++ {
modules = append(modules, fmt.Sprintf("m%02d", i))
}
tm := aTierMesh(t, modules...)
ctx := t.Context()
advancePlans(ctx, tm.open)
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v: the first machine once, for the whole tier", tm.sent)
}
p := tm.plan(t)
lead := p.Modules["m01"]
if lead.Gate == nil || len(lead.Gate.Carried) != len(modules) || lead.GatedBy != "" {
t.Fatalf("the gate on the first module does not judge the whole send: %+v", lead.Gate)
}
for _, m := range modules {
s := p.Modules[m]
if s.FirstAt == nil || !reflect.DeepEqual(s.First, []string{"anchor"}) || s.Previous != "c1" {
t.Fatalf("%s's first send is not recorded, or what anchor ran before is not c1: %+v", m, s)
}
if m != "m01" && s.GatedBy != "m01" {
t.Fatalf("%s is not gated by the send's gate: %+v", m, s)
}
}
gateEvery = 0
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
p = tm.plan(t)
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("sent %v: one send per machine for the tier", tm.sent)
}
if p.State != inventory.PlanDone {
t.Fatalf("the plan is %s: %s", p.State, p.Note)
}
for _, m := range modules {
if g := p.Modules[m].Gate; g == nil || g.Verdict != inventory.GatePassed {
t.Fatalf("%s's gate: %+v", m, g)
}
if v, found, err := tm.open.inventory.GateOf(ctx, "build-"+m+"-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("%s's pass was not kept: %+v %v %v", m, v, found, err)
}
}
}
// The core behind about the node tools, while the node tools are themselves among what was sent and
// settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the
// modules sent with them: at the bound only the node tools are put back.
func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule)
ctx := t.Context()
inv := tm.open.inventory
advancePlans(ctx, tm.open)
if len(tm.sent) != 1 {
t.Fatalf("sent %v", tm.sent)
}
tm.raise("anchor", kindCoreBehind, "anchor runs the node tools it was last sent, not yet reported applied, "+
"and no plan is rolling them out")
gateEvery = 0
advancePlans(ctx, tm.open)
g := tm.plan(t).Modules["app1"].Gate
if g.Verdict != "" || !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) {
t.Fatalf("inside the settle window the gate is %+v: only the node tools wait on it", g)
}
gateBound = -time.Second
advancePlans(ctx, tm.open)
p := tm.plan(t)
g = p.Modules["app1"].Gate
if p.State != inventory.PlanFailed || g.Verdict != inventory.GateFailed ||
!reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) {
t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g)
}
current, _ := inv.CurrentBuilds(ctx)
if current[broker.RuntimeModule].Commit != "c1" {
t.Fatalf("the node tools were not put back: %s", current[broker.RuntimeModule].Commit)
}
for _, m := range []string{"app1", "app2"} {
if current[m].Commit != "c2" {
t.Fatalf("%s was put back for the node tools' condition: %s", m, current[m].Commit)
}
if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed {
t.Fatalf("%s's build was marked failed for the node tools' condition", m)
}
}
if !strings.Contains(p.Note, "app1 was not found wanting") {
t.Fatalf("the plan blames the module its gate was kept on: %s", p.Note)
}
}
// A machine-level condition about anything else holds the machine back as a whole: everything the send
// moved there fails together, once, with that reason — not one module for it.
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
tm := aTierMesh(t, "app1", "app2", "app3")
ctx := t.Context()
inv := tm.open.inventory
advancePlans(ctx, tm.open)
tm.raise("anchor", "disk-full", "anchor's root file system is full")
gateEvery, gateBound = 0, -time.Second
advancePlans(ctx, tm.open)
p := tm.plan(t)
g := p.Modules["app1"].Gate
if p.State != inventory.PlanFailed || !strings.Contains(g.Why, "anchor as a whole") ||
!reflect.DeepEqual(g.Failing, []string{"app1", "app2", "app3"}) {
t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g)
}
current, _ := inv.CurrentBuilds(ctx)
for _, m := range []string{"app1", "app2", "app3"} {
if current[m].Commit != "c1" {
t.Fatalf("%s was not put back with the send: %s", m, current[m].Commit)
}
}
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"anchor"}}) {
t.Fatalf("sent %v: the first send and one send putting all of it back, and nothing to laptop", tm.sent)
}
if !strings.Contains(p.Note, "put back app1 to c1, app2 to c1, app3 to c1 on anchor, in one send") {
t.Fatalf("the plan does not say what was put back: %s", p.Note)
}
}
// The machine-level conditions, sorted: one naming a module is that module's; the core behind is the
// core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole.
func TestTheConditionsAboutAMachineItself(t *testing.T) {
since := time.Now().Add(-time.Minute)
about := func(kind, id string) conditions.Condition {
return conditions.Condition{Key: "machine." + id + "." + kind, Kind: kind, Raised: time.Now(),
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: id, Machine: "anchor"}}
}
f := gateFacts{judged: true, open: []conditions.Condition{about(kindCoreBehind, "anchor")}}
if w := aboutTheMachine("anchor", []string{"app", "blueman"}, since, f); w.whole != "" || len(w.on) != 0 || len(w.facts.open) != 0 {
t.Errorf("the core behind, with no core component sent, counted against what was sent: %+v", w)
}
if w := aboutTheMachine("anchor", []string{"app", broker.RuntimeModule}, since, f); w.whole != "" ||
w.on[broker.RuntimeModule] == "" || w.on["app"] != "" {
t.Errorf("the core behind, with the node tools sent, is not theirs alone: %+v", w)
}
f.open = []conditions.Condition{about("x", "anchor.app")}
if w := aboutTheMachine("anchor", []string{"app", "other"}, since, f); w.on["app"] == "" || w.on["other"] != "" || w.whole != "" {
t.Errorf("a condition naming a module is not that module's: %+v", w)
}
f.open = []conditions.Condition{about("unreachable", "anchor")}
if w := aboutTheMachine("anchor", []string{"app"}, since, f); !strings.Contains(w.whole, "anchor as a whole") {
t.Errorf("a condition about the machine is not the machine's as a whole: %+v", w)
}
f.open[0].Raised = since.Add(-time.Hour)
if w := aboutTheMachine("anchor", []string{"app"}, since, f); w.whole != "" || len(w.facts.open) != 1 {
t.Errorf("a condition older than the send counted: %+v", w)
}
}
// A module its first send changed nothing of — an earlier send already carried this very build there —
// fails a gate as no verdict on its build: left as it was, never marked failed, nothing said as a
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
// judged (the incident's "NOT put back: no build kept").
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// An earlier send carried c2 to anchor before the plan's own.
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-early", map[string]string{"app": "c2"}); err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
if p := g.plan(t); p.Modules["app"].Previous != "c2" {
t.Fatalf("what anchor ran before the send: %+v", p.Modules["app"])
}
g.health["anchor"] = healthNotYet
gateEvery, gateBound = 0, -time.Second
advancePlans(ctx, g.open)
p := g.plan(t)
gate := p.Modules["app"].Gate
if p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || gate.Rollback != gateUnchanged ||
!strings.Contains(p.Note, "app was left as it was") {
t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, gate)
}
if failed, _ := inv.GateFailed(ctx, "build-2"); failed {
t.Fatal("a build its send changed nothing of was marked failed")
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
t.Fatalf("the module was moved: %s", current["app"].Commit)
}
open, _ := g.keeper.Open(ctx)
for _, c := range open {
if c.Kind == kindRollbackFailed || c.Kind == kindRolledBack {
t.Fatalf("said as a rollback: %+v", c)
}
}
was := askABuild
askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil }
t.Cleanup(func() { askABuild = was })
g.health["anchor"] = healthGood
said, err := retryPlan(ctx, g.open, "plan-gate")
if err != nil {
t.Fatalf("a plan stopped at a gate that judged no build was not retried: %v", err)
}
p = g.plan(t)
s := p.Modules["app"]
if p.State != inventory.PlanBuilding || s.State != "asked" || s.Build != "build-3" || s.FirstAt != nil || s.Gate != nil {
t.Fatalf("retried as %q: the plan is %s, app %+v", said, p.State, s)
}
}

Some files were not shown because too many files have changed in this diff Show More