Compare commits
39
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
20c147ffdb | ||
|
|
2eb9a22c24 | ||
|
|
070ecafc07 | ||
|
|
e51c6a2cb9 | ||
|
|
722682f1c4 | ||
|
|
b853439792 | ||
|
|
9cf47f4429 | ||
|
|
8ddc019cd2 | ||
|
|
fab6b0059e | ||
|
|
e1f5d4fdf0 | ||
|
|
bb1607e424 | ||
|
|
cf4834a36c | ||
|
|
9d8cbe7b81 | ||
|
|
e74c32ed50 | ||
|
|
146c48fd96 | ||
|
|
1f3abd3e0e | ||
|
|
6d620f77c3 | ||
|
|
f8286c063d | ||
|
|
e096b4595a | ||
|
|
09c0c6b367 | ||
|
|
d1fc25f682 | ||
|
|
eda457f415 | ||
|
|
59f4d486b1 | ||
|
|
801552c0eb | ||
|
|
ede9bce6ef | ||
|
|
0f0028785c | ||
|
|
6fdcfad8d3 | ||
|
|
8d9d33ae85 | ||
|
|
cc25baa563 | ||
|
|
68a2ebdcc3 | ||
|
|
69b99eec68 | ||
|
|
09bd0eec4f | ||
|
|
222a38e050 | ||
|
|
ee99a24f77 | ||
|
|
f68521da28 | ||
|
|
296064c799 | ||
|
|
df9231c734 | ||
|
|
843b709b59 | ||
|
|
f506fb34ec |
+5
-3
@@ -21,13 +21,15 @@ ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
|
||||
# Dependencies first, so a change to the source does not refetch them.
|
||||
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
|
||||
# the image builds from this repository alone — the host's validator among them, whose module no
|
||||
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY vendor/ vendor/
|
||||
|
||||
COPY . .
|
||||
ARG VERSION=development
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o /mesh-controller ./cmd/mesh-controller
|
||||
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
|
||||
//
|
||||
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
|
||||
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
|
||||
// not happen. Three ways a process stands to the lease:
|
||||
//
|
||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
||||
// its service manager restarts it as a candidate (serve, in push.go).
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
|
||||
// between makes it stale and refused like any other. **When nobody holds the lease, the command
|
||||
// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
|
||||
// it, as it would for another controller.
|
||||
// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
|
||||
// refused nothing: there is nothing to order against, and nothing it does reaches a machine.
|
||||
//
|
||||
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
|
||||
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
|
||||
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
|
||||
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
|
||||
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
|
||||
// instead would be a controller that can never send the user list that lets it act.
|
||||
|
||||
// actor is this process's standing to the lease.
|
||||
type actor struct {
|
||||
mu sync.Mutex
|
||||
// held is the lease this process holds: the serving controller's, or a command's own.
|
||||
held *lease.Lease
|
||||
// unleased is why a serving controller acts without the lease; empty while it holds it or is not
|
||||
// serving.
|
||||
unleased string
|
||||
// serving is a serving controller, which never borrows another's epoch.
|
||||
serving bool
|
||||
// kv is the lease bucket, for a command to read the holder's epoch from.
|
||||
kv jetstream.KeyValue
|
||||
close func()
|
||||
// noBus is a process with no bus configured.
|
||||
noBus bool
|
||||
// reset is when the lease bucket was found raised again from nothing and its revisions moved past
|
||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
var theLease = &actor{}
|
||||
|
||||
// instance names this process among controller instances: its machine, its process and when it
|
||||
// started. The lease's holder and every call this process keeps carry it.
|
||||
var instance = func() string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||
}()
|
||||
|
||||
// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
|
||||
func (a *actor) epoch(ctx context.Context) (uint64, error) {
|
||||
a.mu.Lock()
|
||||
held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
|
||||
a.mu.Unlock()
|
||||
switch {
|
||||
case held != nil:
|
||||
return held.Epoch()
|
||||
case serving && unleased != "":
|
||||
return 0, nil
|
||||
case serving:
|
||||
return 0, lease.ErrNotHeld
|
||||
case noBus:
|
||||
return 0, nil
|
||||
}
|
||||
return a.forACommand(ctx)
|
||||
}
|
||||
|
||||
// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
|
||||
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if a.held != nil {
|
||||
return a.held.Epoch()
|
||||
}
|
||||
if a.kv == nil {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
// No bus: this process reaches no machine, and has nothing to order against.
|
||||
a.noBus = true
|
||||
return 0, nil
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
|
||||
"read and nothing is done: %w", err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := broker.EnsureLeaseBucket(reading, api); err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
kv, err := api.KeyValue(reading, broker.LeaseBucket)
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
a.kv, a.close = kv, js.Close
|
||||
if _, found, err := lease.Current(reading, kv); err == nil && !found {
|
||||
// Nobody: this command takes it for as long as it runs.
|
||||
l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
epoch, err := l.TryTake(reading)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
|
||||
}
|
||||
keeping, stop := context.WithCancel(context.Background())
|
||||
go l.Keep(keeping)
|
||||
a.held = l
|
||||
closeBus := a.close
|
||||
a.close = func() {
|
||||
stop()
|
||||
l.Release(context.Background())
|
||||
closeBus()
|
||||
}
|
||||
return epoch, nil
|
||||
}
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
holder, found, err := lease.Current(reading, a.kv)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
|
||||
"more is done under an epoch nobody holds — run the command again")
|
||||
}
|
||||
return holder.Epoch, nil
|
||||
}
|
||||
|
||||
// release gives back what this process holds, at its end.
|
||||
func (a *actor) release() {
|
||||
a.mu.Lock()
|
||||
closing := a.close
|
||||
a.close = nil
|
||||
a.mu.Unlock()
|
||||
if closing != nil {
|
||||
closing()
|
||||
}
|
||||
}
|
||||
|
||||
// holderOf is this process as the lease's holder.
|
||||
func holderOf(instance string) lease.Holder {
|
||||
host, _ := os.Hostname()
|
||||
return lease.Holder{Instance: instance, Host: host, Build: version}
|
||||
}
|
||||
|
||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
||||
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
|
||||
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
|
||||
a.mu.Lock()
|
||||
a.serving = true
|
||||
a.mu.Unlock()
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.EnsureLeaseBucket(asserting, api)
|
||||
cancel()
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Floor: inv.HighestEpoch, Say: say, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
|
||||
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
|
||||
"no machine refuses the next epoch", was, floor, floor)
|
||||
}})
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
gone := make(chan struct{})
|
||||
epoch, err := l.Take(ctx)
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
js.Close()
|
||||
return nil, ctx.Err()
|
||||
case err != nil && !errors.Is(err, lease.ErrUnwritable):
|
||||
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
|
||||
js.Close()
|
||||
return nil, err
|
||||
case err != nil:
|
||||
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
|
||||
a.mu.Lock()
|
||||
a.unleased = err.Error()
|
||||
a.mu.Unlock()
|
||||
say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
|
||||
"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
|
||||
go a.takeWhenGranted(ctx, l, inv, gone)
|
||||
default:
|
||||
a.took(ctx, l, inv, epoch, gone)
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.close = func() {
|
||||
if held, err := l.Epoch(); err == nil {
|
||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
|
||||
say("how epoch %d ended could not be recorded: %v", held, err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
l.Release(context.Background())
|
||||
js.Close()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
return gone, nil
|
||||
}
|
||||
|
||||
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
|
||||
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
|
||||
a.mu.Lock()
|
||||
a.held, a.unleased = l, ""
|
||||
a.mu.Unlock()
|
||||
h := holderOf(instance)
|
||||
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
|
||||
Build: h.Build, Taken: time.Now()})
|
||||
cancel()
|
||||
if err != nil {
|
||||
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
|
||||
}
|
||||
for _, e := range expired {
|
||||
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
|
||||
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
|
||||
}
|
||||
go l.Keep(ctx)
|
||||
go func() {
|
||||
<-l.Lost()
|
||||
if ctx.Err() == nil {
|
||||
why := l.LostWhy()
|
||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
|
||||
cancel()
|
||||
fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
|
||||
"be started again as a candidate\n", epoch, why)
|
||||
}
|
||||
close(gone)
|
||||
}()
|
||||
}
|
||||
|
||||
// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
|
||||
// controller if another took it meanwhile: two serving at once is what the lease is for.
|
||||
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
|
||||
tick := time.NewTicker(lease.RenewEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
epoch, err := l.TryTake(ctx)
|
||||
if errors.Is(err, lease.ErrTaken) {
|
||||
fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
|
||||
"stops and exits\n", err)
|
||||
close(gone)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
// Still not written, or not readable this time: unleased, said by S12, tried again.
|
||||
a.mu.Lock()
|
||||
a.unleased = err.Error()
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
a.took(ctx, l, inv, epoch, gone)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// standing is what `status` and the self-check say of this process and the lease.
|
||||
type standing struct {
|
||||
Epoch uint64
|
||||
Held bool
|
||||
Renewed time.Time
|
||||
Unleased string
|
||||
// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
|
||||
Reset time.Time
|
||||
ResetSaid string
|
||||
}
|
||||
|
||||
func (a *actor) standing() standing {
|
||||
a.mu.Lock()
|
||||
held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
|
||||
a.mu.Unlock()
|
||||
st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
|
||||
if held == nil {
|
||||
return st
|
||||
}
|
||||
epoch, err := held.Epoch()
|
||||
st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
|
||||
return st
|
||||
}
|
||||
|
||||
// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
|
||||
func init() {
|
||||
link.ActingGate = func(ctx context.Context) error {
|
||||
_, err := theLease.epoch(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("this controller may not send: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,14 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -676,6 +677,20 @@ type answers struct {
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||
unheld []catalogue.Unheld
|
||||
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
|
||||
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
|
||||
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
|
||||
// not be read when they could not — never read as none.
|
||||
conditions []conditions.Condition
|
||||
conditionsUnread string
|
||||
// overflowing is every module whose identity overflows the bound of a provision it requires
|
||||
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||
// this is the one place it is said across the mesh. Not well while there is any.
|
||||
overflowing []catalogue.Overflow
|
||||
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||
handActs *int
|
||||
handActsUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
@@ -692,12 +707,14 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
|
||||
"base will be told that base is missing: %v\n", err)
|
||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.inventory.Held(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
|
||||
// D6 and D7).
|
||||
//
|
||||
// **What the controller asserts at its start and what the self-check expects to find are one
|
||||
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
|
||||
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
|
||||
// the drift rather than the fault.
|
||||
|
||||
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
|
||||
// can now hear a declaration.
|
||||
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
if err := broker.Raise(r, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||
// consumer nothing had created (2026-09-28).
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return nil, errors.Join(failed...)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// What raises the condition a send says when the objects it implies could not be asserted, and its kind.
|
||||
const (
|
||||
sourceBusObjects = "bus-objects"
|
||||
kindBusObjectsUnasserted = "bus-objects-unasserted"
|
||||
)
|
||||
|
||||
// assertOnSend asserts, on the bus a send is about to use, every object assertBusObjects derives —
|
||||
// **whenever a declaration is sent, not only when the controller starts** (novox/hq issue 208).
|
||||
//
|
||||
// A module assigned after the controller started was sent its declaration and found no consumer to
|
||||
// bind (`consumer not found`, messenger on 2026-10-06), and a seat holder assigned after it found no
|
||||
// worker: the objects a declaration implies were asserted at start and nowhere else, so they existed
|
||||
// only for what was assigned before the last restart. The same derivation, not a second list of what
|
||||
// a send needs: what start asserts, the self-check expects and a send asserts are one answer. Every
|
||||
// part is idempotent, so asserting the whole of it again is the no-op a restart already relies on.
|
||||
//
|
||||
// **A failure is said and raised, and the send goes on.** The objects are the mesh's, not the
|
||||
// machines' being sent: holding every machine back for one consumer that none of them may use would
|
||||
// turn one fault into all of them, and the declarations are not what is wrong. It is never silent —
|
||||
// said in the send's own output and raised as a condition, which the next send that asserts them
|
||||
// clears — and the start-time raise still refuses to serve without them.
|
||||
func assertOnSend(ctx context.Context, inv *inventory.Inventory, r broker.Raiser, indent string) error {
|
||||
_, err := assertBusObjects(ctx, inv, r)
|
||||
var observed []conditions.Observation
|
||||
if err != nil {
|
||||
fmt.Printf("%sTHE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES: %v\n", indent, err)
|
||||
fmt.Printf("%s a module may find no consumer to bind, or a holder no worker; sent anyway, raised as "+
|
||||
"condition %s, and asserted again by the next send\n", indent, unassertedObservation(err).Key())
|
||||
observed = append(observed, unassertedObservation(err))
|
||||
}
|
||||
// Observed when it failed, cleared when it did not: a send that asserted everything is the
|
||||
// observation that the bus holds what it should.
|
||||
if kerr := withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
return k.Reconcile(ctx, sourceBusObjects, observed)
|
||||
}); kerr != nil {
|
||||
fmt.Printf("%sand whether the bus holds what this send implies could not be kept as a condition: %v\n",
|
||||
indent, kerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// unassertedObservation is a send's failure to assert the bus's objects, as a condition.
|
||||
func unassertedObservation(err error) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "objects", Token: "unasserted",
|
||||
Kind: kindBusObjectsUnasserted, Severity: conditions.Warning, Source: sourceBusObjects,
|
||||
Summary: "the bus's streams and consumers could not be asserted when a declaration was sent: " +
|
||||
"a module may find no consumer to bind, or a seat's holder no worker",
|
||||
Said: err.Error()}
|
||||
}
|
||||
|
||||
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return broker.ConsumersOf(users), nil
|
||||
}
|
||||
|
||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
return len(consumers), err
|
||||
}
|
||||
|
||||
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
|
||||
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||
var rec recordingRaiser
|
||||
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
|
||||
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
|
||||
}
|
||||
return rec.streams, rec.consumers, nil
|
||||
}
|
||||
|
||||
// recordingRaiser keeps what it was asked to assert and asserts nothing.
|
||||
type recordingRaiser struct {
|
||||
streams []broker.Stream
|
||||
consumers []broker.Consumer
|
||||
}
|
||||
|
||||
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
|
||||
r.streams = append(r.streams, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||
r.consumers = append(r.consumers, c)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
// only when the controller starts.
|
||||
|
||||
// aCarriedConsumer is the runtime on laptop and, carried by it, a module that consumes an event: the
|
||||
// shape of messenger on 2026-10-06, assigned after the controller started.
|
||||
func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}})
|
||||
register(t, open, catalogue.Manifest{Module: "messenger", Version: "1",
|
||||
Consumes: []string{"billing.order.placed"}})
|
||||
for _, m := range []string{catalogue.RuntimeModule, "messenger"} {
|
||||
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
consumers, err := moduleConsumers(t.Context(), open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if c.Module == "messenger" && c.Node == "laptop" {
|
||||
return c.Consumer
|
||||
}
|
||||
}
|
||||
t.Fatalf("messenger on laptop is derived no consumer: %+v", consumers)
|
||||
return broker.Consumer{}
|
||||
}
|
||||
|
||||
// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
|
||||
// started, and the worker its seat's queue should have for it.
|
||||
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
|
||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range inventory.MeshSeats() {
|
||||
if s.Name == "node-build-agent" {
|
||||
c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
|
||||
if !needed {
|
||||
t.Fatal("the build agent's seat needs no worker")
|
||||
}
|
||||
return c
|
||||
}
|
||||
}
|
||||
t.Fatal("the mesh declares no build agent's seat")
|
||||
return broker.Consumer{}
|
||||
}
|
||||
|
||||
// asserted says whether a recording holds a consumer by stream and name.
|
||||
func asserted(rec *recordingRaiser, want broker.Consumer) bool {
|
||||
for _, c := range rec.consumers {
|
||||
if c.Stream == want.Stream && c.Name == want.Name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// What a send asserts includes what was assigned after the start's assertion: a carried module's
|
||||
// consumer and a late holder's worker. The derivation is the start's own, so this holds without a bus.
|
||||
func TestASendAssertsWhatWasAssignedAfterStart(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
var atStart recordingRaiser
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, &atStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
if asserted(&atStart, consumer) || asserted(&atStart, worker) {
|
||||
t.Fatal("the start asserted what was not yet assigned; the test proves nothing")
|
||||
}
|
||||
var onSend recordingRaiser
|
||||
if err := assertOnSend(t.Context(), open.inventory, &onSend, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !asserted(&onSend, consumer) {
|
||||
t.Fatalf("messenger's consumer %s on %s is not asserted by the send: %+v", consumer.Name, consumer.Stream, onSend.consumers)
|
||||
}
|
||||
if !asserted(&onSend, worker) {
|
||||
t.Fatalf("the late holder's worker %s on %s is not asserted by the send: %+v", worker.Name, worker.Stream, onSend.consumers)
|
||||
}
|
||||
}
|
||||
|
||||
// failingRaiser refuses one consumer by name and records everything it was asked.
|
||||
type failingRaiser struct {
|
||||
recordingRaiser
|
||||
refuse string
|
||||
}
|
||||
|
||||
func (f *failingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||
f.consumers = append(f.consumers, c)
|
||||
if c.Name == f.refuse {
|
||||
return errors.New("nats: API error: code=503 description=insufficient resources")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A send whose objects cannot be asserted says so in its output and raises a condition — and the next
|
||||
// send that asserts them clears it. The consumers after the refused one are still asked for.
|
||||
func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
failing := &failingRaiser{refuse: consumer.Name}
|
||||
|
||||
var sendErr error
|
||||
out := stdoutOf(t, func() error {
|
||||
sendErr = assertOnSend(t.Context(), open.inventory, failing, " ")
|
||||
return nil
|
||||
})
|
||||
if sendErr == nil || !strings.Contains(sendErr.Error(), "how messenger on laptop hears what it consumes") {
|
||||
t.Fatalf("the failure is not answered: %v", sendErr)
|
||||
}
|
||||
if !strings.Contains(out, "THE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES") ||
|
||||
!strings.Contains(out, "insufficient resources") || !strings.Contains(out, "bus.objects.unasserted") {
|
||||
t.Fatalf("the failure is not said in the send's output:\n%s", out)
|
||||
}
|
||||
if !asserted(&failing.recordingRaiser, worker) {
|
||||
t.Fatal("the seat's worker was not asked for")
|
||||
}
|
||||
c, open1, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted")
|
||||
if err != nil || !open1 {
|
||||
t.Fatalf("no condition raised: %v", err)
|
||||
}
|
||||
if c.Kind != kindBusObjectsUnasserted || c.Source != sourceBusObjects ||
|
||||
len(c.Evidence) == 0 || !strings.Contains(c.Evidence[0].Said, "insufficient resources") {
|
||||
t.Fatalf("the condition does not say what failed: %+v", c)
|
||||
}
|
||||
|
||||
// The next send asserts them, and that observation clears it.
|
||||
if err := assertOnSend(t.Context(), open.inventory, &recordingRaiser{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, stillOpen, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); err != nil || stillOpen {
|
||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
// The controller starting, before either was assigned.
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, nil)
|
||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
||||
t.Fatal("a send that asserted everything raised a condition")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
|
||||
// RequestTimeout) — the shortest wait of a caller the mesh ships.
|
||||
const consoleWaits = 30 * time.Second
|
||||
|
||||
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
|
||||
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
|
||||
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
|
||||
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
||||
if link.AnswerWithin >= consoleWaits/2 {
|
||||
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
|
||||
}
|
||||
if link.AnswerWithin >= broker.ResponseTTL {
|
||||
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
|
||||
}
|
||||
}
|
||||
|
||||
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
||||
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
||||
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want bool
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
||||
}
|
||||
if got := answersFirst(argv); got != c.want {
|
||||
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,17 @@ import (
|
||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||
// refused what it could not see would teach people to ignore it.
|
||||
//
|
||||
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
||||
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||
// provider's machine when a real machine's name first meets the module's.
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||
}
|
||||
|
||||
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
if len(paths) == 0 {
|
||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||
"manifest of a repository together so the rules between them are checked too")
|
||||
@@ -74,6 +84,15 @@ func moduleCheck(paths []string, out io.Writer) error {
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||
// only the provider's manifest states.
|
||||
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
||||
sort.Strings(identities)
|
||||
for _, p := range identities {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(identities)
|
||||
|
||||
var names []string
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
@@ -109,7 +128,8 @@ func moduleCheck(paths []string, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||
"module not given here reads as unknown\n", len(paths))
|
||||
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
||||
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,433 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
||||
//
|
||||
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||
// while, with a reason, and that is recorded as a hand act.
|
||||
|
||||
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||
var conditionsFrom *conditions.Keeper
|
||||
|
||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||
// does nothing).
|
||||
Changed: statusFrom.nudge,
|
||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||
}
|
||||
|
||||
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
||||
// it was given before it returns.
|
||||
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||
if conditionsFrom != nil {
|
||||
return f(conditionsFrom)
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
k, err := keeperOn(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
k.Close(flushing)
|
||||
}()
|
||||
return f(k)
|
||||
})
|
||||
}
|
||||
|
||||
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
||||
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
||||
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||
if conditionsFrom != nil {
|
||||
return conditionsFrom.Open(ctx)
|
||||
}
|
||||
if _, err := broker.BusAddress(); err != nil {
|
||||
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||
}
|
||||
var out []conditions.Condition
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, err = conditions.Read(reading, store)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// conditionsUsage is how the verb is typed.
|
||||
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
||||
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
||||
"conditions history [--days N] [--key K] [--json]"
|
||||
|
||||
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
||||
func conditionsCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
return listConditions(ctx, args)
|
||||
case "show":
|
||||
return showCondition(ctx, args)
|
||||
case "silence":
|
||||
return silenceCondition(ctx, args)
|
||||
case "history":
|
||||
return conditionHistory(ctx, args)
|
||||
}
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
|
||||
func listConditions(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||
severity := set.String("severity", "", "only urgent, or only warning")
|
||||
machine := set.String("machine", "", "only those about this machine")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
||||
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
||||
}
|
||||
open, err := openConditions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, c := range open {
|
||||
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
||||
(*machine == "" || concerns(c, *machine)) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Condition{}
|
||||
}
|
||||
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if len(open) == 0 {
|
||||
fmt.Println("no open conditions")
|
||||
} else {
|
||||
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, line := range conditionLines(out, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||
func concerns(c conditions.Condition, machine string) bool {
|
||||
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||
return true
|
||||
}
|
||||
for _, part := range strings.Split(c.Key, ".") {
|
||||
if part == machine {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
||||
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||
var out []string
|
||||
for _, c := range list {
|
||||
times := ""
|
||||
if c.Count > 1 {
|
||||
times = fmt.Sprintf(", raised %d times", c.Count)
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
||||
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
||||
if c.SilencedAt(now) {
|
||||
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
||||
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func showCondition(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New("conditions show <key>")
|
||||
}
|
||||
key := rest[0]
|
||||
var c conditions.Condition
|
||||
var found bool
|
||||
if conditionsFrom != nil {
|
||||
c, found, err = conditionsFrom.Get(ctx, key)
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
||||
"history --key %s` what became of it", key, key)
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(c)
|
||||
}
|
||||
now := time.Now()
|
||||
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
if c.Subject.Machine != "" {
|
||||
fmt.Printf(", on %s", c.Subject.Machine)
|
||||
}
|
||||
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||
now.Sub(c.LastObserved).Round(time.Second))
|
||||
if c.Count > 1 {
|
||||
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
}
|
||||
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||
if c.Silenced != nil {
|
||||
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
c.Silenced.By, c.Silenced.Why)
|
||||
}
|
||||
if len(c.Tried) > 0 {
|
||||
fmt.Println("\n tried:")
|
||||
for _, t := range c.Tried {
|
||||
fmt.Printf(" %s %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), t.What, t.Outcome)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n evidence, newest first:")
|
||||
for _, e := range c.Evidence {
|
||||
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolverWords(r string) string {
|
||||
switch r {
|
||||
case conditions.ResolverSelf:
|
||||
return "itself: it clears when observation says it is resolved"
|
||||
case conditions.ResolverOperator:
|
||||
return "the operator: nothing in the mesh will repair it"
|
||||
case conditions.ResolverAgent:
|
||||
return "an agent"
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
||||
// who and why before it is done, its cause the condition's kind unless one is given.
|
||||
func silenceCondition(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
||||
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
||||
acts := addHandActFlags(set)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
||||
}
|
||||
key := rest[0]
|
||||
if err := acts.require("conditions silence"); err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := parseFor(*forFlag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d > conditions.MaxSilence {
|
||||
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
||||
}
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
c, found, err := k.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
||||
}
|
||||
if strings.TrimSpace(*acts.cause) == "" {
|
||||
*acts.cause = c.Kind
|
||||
}
|
||||
*acts.condition = key
|
||||
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
||||
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
||||
"`status` still says it; it clears when observation says it is resolved\n",
|
||||
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// parseFor reads a duration, days included.
|
||||
func parseFor(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
||||
}
|
||||
if days, ok := strings.CutSuffix(s, "d"); ok {
|
||||
n, err := strconv.Atoi(days)
|
||||
if err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("%q is not a number of days", s)
|
||||
}
|
||||
return time.Duration(n) * 24 * time.Hour, nil
|
||||
}
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func conditionHistory(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||
days := set.Int("days", 7, "how many days back, at most 90")
|
||||
key := set.String("key", "", "only this condition")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("conditions history [--days N] [--key K] [--json]")
|
||||
}
|
||||
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
||||
var events []conditions.Event
|
||||
read := func(h conditions.History) error {
|
||||
var err error
|
||||
events, err = h.Since(ctx, since)
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if conditionsFrom != nil {
|
||||
events, err = conditionsFrom.HistorySince(ctx, since)
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
_, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return read(history)
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out []conditions.Event
|
||||
for _, e := range events {
|
||||
if *key == "" || e.Key == *key {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Event{}
|
||||
}
|
||||
return printJSON(map[string]any{"history": out, "days": *days})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for _, e := range out {
|
||||
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
||||
switch e.Change {
|
||||
case conditions.ChangeRaised, conditions.ChangeReopened:
|
||||
line += " — " + e.Summary
|
||||
case conditions.ChangeSeverity:
|
||||
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
||||
case conditions.ChangeResolver:
|
||||
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
||||
default:
|
||||
if e.Why != "" {
|
||||
line += " — " + e.Why
|
||||
}
|
||||
}
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
||||
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
||||
// is not "none open".
|
||||
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
||||
if unread != "" {
|
||||
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
urgent := 0
|
||||
for _, c := range list {
|
||||
if c.Severity == conditions.Urgent {
|
||||
urgent++
|
||||
}
|
||||
}
|
||||
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
||||
for _, line := range conditionLines(list, now) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
||||
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
|
||||
// by what is open.
|
||||
|
||||
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "conditions --json"},
|
||||
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
|
||||
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
|
||||
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
|
||||
"conditions history --json --days 3 --key machine.ace.silent"},
|
||||
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
|
||||
"conditions silence machine.ace.silent --for 2h --why on the train"},
|
||||
{map[string]any{"run": "true"}, "doctor run --json"},
|
||||
{map[string]any{}, "doctor --json"},
|
||||
} {
|
||||
verb := "conditions"
|
||||
if strings.HasPrefix(c.want, "doctor") {
|
||||
verb = "doctor"
|
||||
}
|
||||
argv, err := argvFor(verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
|
||||
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
|
||||
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
|
||||
} {
|
||||
if argv, err := argvFor(c.verb, c.args); err == nil {
|
||||
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
|
||||
t.Error("a silence is not a hand act")
|
||||
}
|
||||
}
|
||||
|
||||
// **A silence through the verb is recorded and bounded**; the condition stays open.
|
||||
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
|
||||
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
|
||||
t.Fatal("silenced without saying why")
|
||||
}
|
||||
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
|
||||
t.Fatal("silenced for more than a week")
|
||||
}
|
||||
said := printed(t, func() error {
|
||||
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
|
||||
})
|
||||
if !strings.Contains(said, "is silenced until") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
c, found, _ := k.Get(ctx, "machine.ace.silent")
|
||||
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
|
||||
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
|
||||
t.Fatalf("%s", listed)
|
||||
}
|
||||
}
|
||||
|
||||
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.Fail = errors.New("the bus is away")
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() || asked.conditionsUnread == "" {
|
||||
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.Fail = nil
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
|
||||
//
|
||||
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
|
||||
// design — every machine's declaration composes and validates, every resolver answers, every seat's
|
||||
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
|
||||
// condition it raises when the invariant does not hold. The serving controller runs the registry every
|
||||
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
|
||||
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
|
||||
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
|
||||
// machine: a controller that stops checking is itself said, through a channel that does not pass
|
||||
// through it.
|
||||
//
|
||||
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
|
||||
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
|
||||
|
||||
// The self-check's clocks.
|
||||
var (
|
||||
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
|
||||
// starts the first run waits, so what the controller hears at its start has arrived.
|
||||
doctorEvery = 5 * time.Minute
|
||||
doctorFirstAfter = time.Minute
|
||||
// probeWithin is each probe's bound.
|
||||
probeWithin = 30 * time.Second
|
||||
)
|
||||
|
||||
// The verdicts a probe can have.
|
||||
const (
|
||||
verdictPass = "pass"
|
||||
verdictFail = "fail"
|
||||
verdictFailedToRun = "failed-to-run"
|
||||
verdictDeferred = "deferred"
|
||||
)
|
||||
|
||||
// probe is one live invariant.
|
||||
type probe struct {
|
||||
ID string
|
||||
Asserts string
|
||||
From string
|
||||
// Kind is the condition kind raised when the invariant does not hold.
|
||||
Kind string
|
||||
Phase int
|
||||
// Deferred says why it is not run yet; empty for one that is.
|
||||
Deferred string
|
||||
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
|
||||
// controller's grant names every one (a test over this registry): a probe whose question the bus
|
||||
// refuses checks nothing (D8, 2026-10-06).
|
||||
Asks []broker.SeatVerb
|
||||
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||
}
|
||||
|
||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
|
||||
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
|
||||
Kind: "archives-unheld", Phase: 1, run: probeArchives},
|
||||
{ID: "D5", Asserts: "exactly one lease holder — the key names this controller at its epoch, and the record " +
|
||||
"holds no other epoch open; no message from a stale epoch refused in the last interval",
|
||||
From: "issue 204", Kind: "lease-split", Phase: 2, run: probeLease},
|
||||
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Phase: 1, run: probeConsumers},
|
||||
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
|
||||
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
|
||||
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
|
||||
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
|
||||
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
|
||||
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
|
||||
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
type probeVerdict struct {
|
||||
ID string `json:"id"`
|
||||
Verdict string `json:"verdict"`
|
||||
Found []string `json:"found,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
}
|
||||
|
||||
// doctorCounts are a run's verdicts, counted.
|
||||
type doctorCounts struct {
|
||||
Passed int `json:"passed"`
|
||||
Failed int `json:"failed"`
|
||||
FailedToRun int `json:"failed-to-run"`
|
||||
Deferred int `json:"deferred"`
|
||||
}
|
||||
|
||||
// doctorRun is one run of the registry, and the body of its heartbeat.
|
||||
type doctorRun struct {
|
||||
Run string `json:"run"`
|
||||
At time.Time `json:"at"`
|
||||
Started time.Time `json:"started"`
|
||||
Took string `json:"took"`
|
||||
IntervalSeconds int `json:"interval-seconds"`
|
||||
Counts doctorCounts `json:"counts"`
|
||||
Probes []probeVerdict `json:"probes"`
|
||||
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
|
||||
Controller string `json:"controller"`
|
||||
Why string `json:"why"`
|
||||
// Unsaid is how many condition transitions this controller could not say, since it started.
|
||||
Unsaid int `json:"unsaid,omitempty"`
|
||||
}
|
||||
|
||||
// doctor is the registry and what its probes need.
|
||||
type doctor struct {
|
||||
open *stores
|
||||
js *broker.JetStream
|
||||
keeper *conditions.Keeper
|
||||
teller conditions.Teller
|
||||
watchdogs *watchdogs
|
||||
host string
|
||||
|
||||
running sync.Mutex
|
||||
mu sync.Mutex
|
||||
last *doctorRun
|
||||
ended time.Time
|
||||
}
|
||||
|
||||
// lastRunEnded is when the last run ended; zero before the first.
|
||||
func (d *doctor) lastRunEnded() time.Time {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.ended
|
||||
}
|
||||
|
||||
// lastRun is the last run's verdict; nil before the first.
|
||||
func (d *doctor) lastRun() *doctorRun {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.last
|
||||
}
|
||||
|
||||
// keep runs the registry on its schedule until ctx ends.
|
||||
func (d *doctor) keep(ctx context.Context) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(doctorFirstAfter):
|
||||
}
|
||||
tick := time.NewTicker(doctorEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
// Only the controller acting checks the mesh on a schedule: one standing by would say a
|
||||
// heartbeat for a self-check that is not the mesh's.
|
||||
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
|
||||
d.runOnce(ctx, "the schedule")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var doctorRuns struct {
|
||||
sync.Mutex
|
||||
n uint64
|
||||
}
|
||||
|
||||
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
|
||||
// at a time: a person's `doctor run` during a scheduled one waits for it.
|
||||
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||
d.running.Lock()
|
||||
defer d.running.Unlock()
|
||||
doctorRuns.Lock()
|
||||
doctorRuns.n++
|
||||
n := doctorRuns.n
|
||||
doctorRuns.Unlock()
|
||||
started := time.Now()
|
||||
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
|
||||
|
||||
type result struct {
|
||||
obs []conditions.Observation
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
results := make([]result, len(probeRegistry))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range probeRegistry {
|
||||
if p.run == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int, p probe) {
|
||||
defer wg.Done()
|
||||
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
|
||||
defer cancel()
|
||||
began := time.Now()
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||
}
|
||||
}()
|
||||
obs, err := p.run(probing, d)
|
||||
done <- result{obs: obs, err: err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
r.took = time.Since(began)
|
||||
results[i] = r
|
||||
case <-probing.Done():
|
||||
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
|
||||
}
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var blind []conditions.Observation
|
||||
for i, p := range probeRegistry {
|
||||
v := probeVerdict{ID: p.ID}
|
||||
r := results[i]
|
||||
switch {
|
||||
case p.run == nil:
|
||||
v.Verdict = verdictDeferred
|
||||
run.Counts.Deferred++
|
||||
case r.err != nil:
|
||||
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
|
||||
run.Counts.FailedToRun++
|
||||
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
|
||||
Token: "failed", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||
Said: firstLine(r.err.Error())})
|
||||
default:
|
||||
if err := d.keeper.Reconcile(ctx, p.ID, kindedAs(r.obs, p.Kind)); err != nil {
|
||||
v.Error = "what it found could not be kept: " + err.Error()
|
||||
}
|
||||
if len(r.obs) == 0 {
|
||||
v.Verdict = verdictPass
|
||||
run.Counts.Passed++
|
||||
} else {
|
||||
v.Verdict = verdictFail
|
||||
run.Counts.Failed++
|
||||
for _, o := range r.obs {
|
||||
v.Found = append(v.Found, o.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.run != nil {
|
||||
v.Took = r.took.Round(time.Millisecond).String()
|
||||
}
|
||||
run.Probes = append(run.Probes, v)
|
||||
}
|
||||
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
|
||||
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
|
||||
}
|
||||
ended := time.Now()
|
||||
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
|
||||
d.mu.Lock()
|
||||
d.last, d.ended = &run, ended
|
||||
d.mu.Unlock()
|
||||
d.sayHeartbeat(ctx, run)
|
||||
return run
|
||||
}
|
||||
|
||||
// sourceDoctor is what raises a probe's own failure to run.
|
||||
const sourceDoctor = "doctor"
|
||||
|
||||
// kindedAs gives each observation of a probe the probe's kind where it named none.
|
||||
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(obs))
|
||||
for _, o := range obs {
|
||||
if o.Kind == "" {
|
||||
o.Kind = kind
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
|
||||
// says it outward: the watcher hears nothing.
|
||||
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
|
||||
if d.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
|
||||
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
|
||||
"will say the self-check is silent: %v\n", run.Run, err)
|
||||
}
|
||||
}
|
||||
|
||||
// doctorFrom is the serving controller's self-check; nil in any other process.
|
||||
var doctorFrom *doctor
|
||||
|
||||
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
|
||||
func doctorCommand(ctx context.Context, args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("doctor [run|probes|signals] [--json]")
|
||||
}
|
||||
answer, err := doctorAnswer(ctx, sub)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(answer)
|
||||
}
|
||||
fmt.Print(doctorText(answer))
|
||||
return nil
|
||||
}
|
||||
|
||||
// doctorAnswer is what the verb answers, as data.
|
||||
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
switch sub {
|
||||
case "":
|
||||
if doctorFrom != nil {
|
||||
if run := doctorFrom.lastRun(); run != nil {
|
||||
return verdictAnswer(*run, time.Now()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||
}
|
||||
run, err := lastHeartbeat(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return verdictAnswer(run, time.Now()), nil
|
||||
case "run":
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
local, closeIt, err := localDoctor(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeIt()
|
||||
d = local
|
||||
}
|
||||
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||
case "probes":
|
||||
return probesAnswer(), nil
|
||||
case "signals":
|
||||
if doctorFrom == nil || doctorFrom.watchdogs == nil {
|
||||
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
|
||||
"hears them: ask it through the mesh-controller seat's doctor verb")
|
||||
}
|
||||
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
|
||||
}
|
||||
|
||||
// verdictAnswer is a run as the verb answers it, with its age.
|
||||
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it"}
|
||||
}
|
||||
|
||||
// probesAnswer is the registry.
|
||||
func probesAnswer() map[string]any {
|
||||
var out []map[string]any
|
||||
for _, p := range probeRegistry {
|
||||
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
|
||||
if p.Deferred != "" {
|
||||
row["deferred"] = p.Deferred
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
|
||||
}
|
||||
|
||||
// signalsAnswer is every row of the signals table with the age of its newest signal.
|
||||
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
|
||||
var rows []map[string]any
|
||||
for _, r := range signalsTable {
|
||||
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
|
||||
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
|
||||
switch {
|
||||
case r.Deferred != "":
|
||||
row["deferred"] = r.Deferred
|
||||
case f == nil:
|
||||
row["newest"] = "not yet looked at"
|
||||
default:
|
||||
if err := r.needs(f); err != nil {
|
||||
row["blind"] = err.Error()
|
||||
} else if newest := r.newest(f); newest.IsZero() {
|
||||
row["newest"] = "none heard"
|
||||
} else {
|
||||
row["newest"] = newest.UTC().Format(time.RFC3339)
|
||||
row["age"] = f.now.Sub(newest).Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
out := map[string]any{"signals": rows, "every": watchEvery.String()}
|
||||
if !ticked.IsZero() {
|
||||
out["looked"] = ticked.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// doctorText is an answer as a person reads it.
|
||||
func doctorText(answer any) string {
|
||||
body, _ := json.Marshal(answer)
|
||||
var b strings.Builder
|
||||
var verdict struct {
|
||||
Run doctorRun `json:"run"`
|
||||
Age string `json:"age"`
|
||||
}
|
||||
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
|
||||
r := verdict.Run
|
||||
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
|
||||
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
|
||||
for _, p := range r.Probes {
|
||||
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
|
||||
for _, f := range p.Found {
|
||||
fmt.Fprintf(&b, " %s\n", f)
|
||||
}
|
||||
if p.Error != "" {
|
||||
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
pretty, _ := json.MarshalIndent(answer, "", " ")
|
||||
return string(pretty) + "\n"
|
||||
}
|
||||
|
||||
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
|
||||
// the serving controller answers `doctor` from.
|
||||
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
|
||||
var run doctorRun
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
|
||||
}
|
||||
return json.Unmarshal(msg.Data, &run)
|
||||
})
|
||||
return run, err
|
||||
}
|
||||
|
||||
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
|
||||
// its own connection, and its own keeper, closed after.
|
||||
func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
k, err := keeperOn(ctx, js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
jsCtx := js.Context()
|
||||
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
|
||||
host: controlHost(ctx, open.inventory)}
|
||||
return d, func() {
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
k.Close(flushing)
|
||||
js.Close()
|
||||
open.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
|
||||
func sortedFound(obs []conditions.Observation) []conditions.Observation {
|
||||
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
|
||||
return obs
|
||||
}
|
||||
|
||||
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
|
||||
type probeAsksKey struct{}
|
||||
|
||||
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
|
||||
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
|
||||
p, ok := ctx.Value(probeAsksKey{}).(probe)
|
||||
if !ok {
|
||||
return "a caller outside the self-check", false
|
||||
}
|
||||
for _, v := range p.Asks {
|
||||
if v.Seat == seat && v.Verb == verb {
|
||||
return p.ID, true
|
||||
}
|
||||
}
|
||||
return p.ID, false
|
||||
}
|
||||
@@ -0,0 +1,425 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/net/dns/dnsmessage"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
|
||||
|
||||
// withProbes runs the test with a registry of its own.
|
||||
func withProbes(t *testing.T, probes ...probe) {
|
||||
t.Helper()
|
||||
before := probeRegistry
|
||||
probeRegistry = probes
|
||||
t.Cleanup(func() { probeRegistry = before })
|
||||
}
|
||||
|
||||
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||
var broken atomic.Bool
|
||||
broken.Store(true)
|
||||
withProbes(t,
|
||||
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return []conditions.Observation{failing}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return nil, errors.New("the store is away")
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
<-ctx.Done()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||
)
|
||||
before := probeWithin
|
||||
probeWithin = 200 * time.Millisecond
|
||||
t.Cleanup(func() { probeWithin = before })
|
||||
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||
run := d.runOnce(t.Context(), "a test")
|
||||
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", run.Counts)
|
||||
}
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key+"="+c.Kind)
|
||||
}
|
||||
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||
"probe.P4.failed=probe-failed"} {
|
||||
if !slices.Contains(keys, want) {
|
||||
t.Errorf("%s is not open: %v", want, keys)
|
||||
}
|
||||
}
|
||||
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||
if len(told.Names) != 1 || told.Names[0] != conditions.HeartbeatEvent {
|
||||
t.Fatalf("said %v", told.Names)
|
||||
}
|
||||
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||
t.Fatal("the run is not the last verdict")
|
||||
}
|
||||
body, _ := json.Marshal(run)
|
||||
var shape map[string]any
|
||||
_ = json.Unmarshal(body, &shape)
|
||||
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
|
||||
// Mended: the next run clears every one of them.
|
||||
broken.Store(false)
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("a passing run left open %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **The registry says what each probe asserts**, and a probe not built says why and when.
|
||||
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, p := range probeRegistry {
|
||||
if seen[p.ID] {
|
||||
t.Errorf("%s twice", p.ID)
|
||||
}
|
||||
seen[p.ID] = true
|
||||
if p.Asserts == "" || p.From == "" || p.Kind == "" {
|
||||
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
|
||||
}
|
||||
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
|
||||
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
|
||||
if !seen[id] {
|
||||
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
|
||||
// stopped is S10 (signals_test.go).
|
||||
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
|
||||
w := &watchdogs{started: time.Now().Add(-time.Hour)}
|
||||
d := &doctor{watchdogs: w, host: "anchor"}
|
||||
got, err := probeWatchdogs(t.Context(), d)
|
||||
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
w.ticked = time.Now()
|
||||
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
|
||||
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
|
||||
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{"rival-one", "rival-two"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
|
||||
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||
answerAs := func(rcode dnsmessage.RCode) string {
|
||||
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
go func() {
|
||||
buf := make([]byte, 1500)
|
||||
for {
|
||||
n, from, err := conn.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var q dnsmessage.Message
|
||||
if q.Unpack(buf[:n]) != nil {
|
||||
continue
|
||||
}
|
||||
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
|
||||
if q.Questions[0].Type == dnsmessage.TypeA {
|
||||
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
|
||||
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
|
||||
} else {
|
||||
reply.RCode = rcode
|
||||
}
|
||||
packed, _ := reply.Pack()
|
||||
_, _ = conn.WriteTo(packed, from)
|
||||
}
|
||||
}()
|
||||
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||
return port
|
||||
}
|
||||
before := resolverPort
|
||||
t.Cleanup(func() { resolverPort = before })
|
||||
|
||||
resolverPort = answerAs(dnsmessage.RCodeSuccess)
|
||||
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
|
||||
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
|
||||
t.Fatalf("%v %v %v", v4, rcode, err)
|
||||
}
|
||||
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
|
||||
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
|
||||
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
|
||||
}
|
||||
resolverPort = answerAs(dnsmessage.RCodeNameError)
|
||||
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
|
||||
t.Fatalf("NXDOMAIN read as %v", rcode)
|
||||
}
|
||||
}
|
||||
|
||||
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||
got, err := p(t.Context(), d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := info.Config
|
||||
cfg.MaxMsgsPerSubject = 3
|
||||
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumers, err := probeConsumers(t.Context(), d)
|
||||
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||
}
|
||||
streams, err := probeStreams(t.Context(), d)
|
||||
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
heard := make(chan *nats.Msg, 16)
|
||||
for _, subject := range broker.BusAdvisories {
|
||||
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
api, _ := jetstream.New(conn)
|
||||
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kinds := map[string]string{}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for len(kinds) < 2 {
|
||||
select {
|
||||
case m := <-heard:
|
||||
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||
kinds[a.Kind] = a.Said
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the bus said only %v", kinds)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||
t.Fatalf("%v", kinds)
|
||||
}
|
||||
}
|
||||
|
||||
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
|
||||
// machine read as behind right after a push sent it the current build — it says the digest-named
|
||||
// directory it runs from, and the mesh holds a commit).
|
||||
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
|
||||
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
|
||||
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
|
||||
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
|
||||
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
|
||||
}}
|
||||
delivered := deliveredVersions(m)
|
||||
if !slices.Equal(delivered, []string{"31045596c83a"}) {
|
||||
t.Fatalf("%v", delivered)
|
||||
}
|
||||
commit := "1545b00a9f0c"
|
||||
for _, c := range []struct {
|
||||
reported string
|
||||
behind bool
|
||||
}{
|
||||
{"31045596c83a", false}, // the live case: current, and was called behind
|
||||
{"0123456789ab", true}, // another delivery
|
||||
{"1545b00a", false}, // placed by hand, stamped with the commit
|
||||
{"", false}, // not said
|
||||
} {
|
||||
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
|
||||
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
|
||||
}
|
||||
}
|
||||
if engineBehind("31045596c83a", nil, commit) {
|
||||
t.Error("behind a mesh that holds no delivered build")
|
||||
}
|
||||
}
|
||||
|
||||
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
|
||||
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
|
||||
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
|
||||
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked := 0
|
||||
for _, p := range probeRegistry {
|
||||
for _, v := range p.Asks {
|
||||
asked++
|
||||
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
|
||||
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
|
||||
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
|
||||
}
|
||||
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
|
||||
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
if asked == 0 {
|
||||
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
|
||||
}
|
||||
// And a probe asking what it did not declare is refused before anything is sent.
|
||||
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
|
||||
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
|
||||
!strings.Contains(err.Error(), "does not declare") {
|
||||
t.Fatalf("an undeclared question was asked: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// subjectMatches is the bus's matching of a permission pattern against a subject.
|
||||
func subjectMatches(pattern, subject string) bool {
|
||||
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
|
||||
for i, tok := range p {
|
||||
if tok == ">" {
|
||||
return len(s) > i
|
||||
}
|
||||
if i >= len(s) || (tok != "*" && tok != s[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(s)
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||
if asked.IsZero() || result.ID == "" {
|
||||
return
|
||||
}
|
||||
subject := result.Module
|
||||
if subject == "" {
|
||||
subject = result.Repository
|
||||
}
|
||||
detail := "built"
|
||||
if result.Failed != "" {
|
||||
detail = "failed: " + firstLine(result.Failed)
|
||||
}
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
case inventory.DurationHeartbeatGap:
|
||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said
|
||||
// it reads one, and to no other: an older node-engine refuses a key it does not know, whole.
|
||||
|
||||
// bodiesDelivery records each send as the mesh does, and keeps the bodies.
|
||||
type bodiesDelivery struct {
|
||||
recordedDelivery
|
||||
bodies map[string][]byte
|
||||
}
|
||||
|
||||
func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
b.bodies[s.node] = body
|
||||
return b.recordedDelivery.declare(ctx, s, body)
|
||||
}
|
||||
|
||||
func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
epoch := uint64(57)
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return epoch, nil }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
|
||||
anchor, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carried := func(node string) (epoch float64, has bool) {
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal(d.bodies[node], &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epoch, has = envelope["epoch"].(float64)
|
||||
return epoch, has
|
||||
}
|
||||
if e, has := carried("anchor"); !has || e != 57 {
|
||||
t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"])
|
||||
}
|
||||
if _, has := carried("laptop"); has {
|
||||
t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"])
|
||||
}
|
||||
|
||||
// A new holder of the lease is not a change of the machine: neither reads as behind.
|
||||
epoch = 58
|
||||
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
sent, err := inv.Outstanding(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if would[node] != sent {
|
||||
t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A process that may not act composes nothing and sends nothing: its number is not taken.
|
||||
func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") {
|
||||
t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused)
|
||||
}
|
||||
anchor, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||
if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 {
|
||||
t.Fatalf("a controller without the lease took sequence %d", seq)
|
||||
}
|
||||
|
||||
// And at the send itself: the gate every declaration passes.
|
||||
gate := link.ActingGate
|
||||
link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { link.ActingGate = gate })
|
||||
if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil ||
|
||||
!strings.Contains(err.Error(), "lost the lease") {
|
||||
t.Fatalf("a declaration was let through the gate: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
|
||||
//
|
||||
// The serving controller hears every report; a clean one about the declaration a machine was last
|
||||
// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is
|
||||
// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
|
||||
// so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log
|
||||
// is read as the count of repairs a healer is wanted for.
|
||||
|
||||
// SecretReplaced is the controller seat's fact that a value given by hand was replaced
|
||||
// (link.KeySecretReplaced). Never the value: neither the old one, which the mesh cannot read,
|
||||
// nor the new one, sealed to the machine as it was made.
|
||||
type SecretReplaced struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
Given time.Time `json:"given"`
|
||||
// Sent are the machines sent so the module starts again on the new value; Unsent says why
|
||||
// they could not be, in which case the next push carries it.
|
||||
Sent []string `json:"sent"`
|
||||
Unsent string `json:"unsent,omitempty"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// givenEvents is where the serving controller states it; nil in a command.
|
||||
var givenEvents link.Bus
|
||||
|
||||
// replacing keeps one replacement per machine at a time: two reports arriving together find the
|
||||
// same rows, and the store's claim makes one of them the replacer, but the sends need not race.
|
||||
var replacing sync.Map
|
||||
|
||||
// startedWell says a report is a machine's clean account of a declaration: everything applied,
|
||||
// nothing failed or refused. Whether it is the declaration last sent is the store's to answer.
|
||||
func startedWell(report link.Report) bool {
|
||||
return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil
|
||||
}
|
||||
|
||||
const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
|
||||
|
||||
// replaceGiven replaces what the report makes due, sends the machines, and says so.
|
||||
func replaceGiven(ctx context.Context, open *stores, report link.Report) {
|
||||
if _, busy := replacing.LoadOrStore(report.Node, true); busy {
|
||||
return
|
||||
}
|
||||
defer replacing.Delete(report.Node)
|
||||
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared)
|
||||
if err != nil {
|
||||
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err)
|
||||
}
|
||||
for _, r := range replaced {
|
||||
said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
|
||||
Sent: r.Machines, Why: givenWhy}
|
||||
log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; sending %s",
|
||||
r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
|
||||
if err := sendTo(ctx, open, r.Machines); err != nil {
|
||||
said.Sent, said.Unsent = nil, err.Error()
|
||||
log.Printf("the new %q of %s is sealed and not yet delivered to %s — the next push carries it: %v",
|
||||
r.Name, r.Module, strings.Join(r.Machines, ", "), err)
|
||||
}
|
||||
stateReplaced(ctx, said)
|
||||
}
|
||||
}
|
||||
|
||||
func stateReplaced(ctx context.Context, said SecretReplaced) {
|
||||
if givenEvents == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(said)
|
||||
if err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
return
|
||||
}
|
||||
stating, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := givenEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySecretReplaced, body); err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A rotation asked through the seat carries why to the command, which records it in the hand-act
|
||||
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
|
||||
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
|
||||
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password --why leaked into logs --cause leaked" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
|
||||
t.Fatalf("without why: %v %v", argv, err)
|
||||
}
|
||||
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
|
||||
t.Fatalf("why beside a provision was passed over: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that
|
||||
// the machine is there is not (novox/hq ADR 0228).
|
||||
func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
report link.Report
|
||||
good bool
|
||||
}{
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true},
|
||||
{link.Report{Node: "anchor"}, false},
|
||||
{link.Report{Node: "anchor", Applied: []string{"x"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false},
|
||||
} {
|
||||
if got := startedWell(c.report); got != c.good {
|
||||
t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
//
|
||||
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||
// (Phase 1).
|
||||
|
||||
// handActFlags are the flags every repairing verb takes.
|
||||
type handActFlags struct {
|
||||
why, cause, condition *string
|
||||
}
|
||||
|
||||
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||
return handActFlags{
|
||||
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||
}
|
||||
}
|
||||
|
||||
// given is whether a reason was given.
|
||||
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||
|
||||
// require refuses an act without a reason, before anything is done.
|
||||
func (f handActFlags) require(verb string) error {
|
||||
if f.given() {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||
}
|
||||
|
||||
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||
// command dials its own.
|
||||
var handActConn *nats.Conn
|
||||
|
||||
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||
func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
|
||||
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
if !f.given() {
|
||||
return
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
act = written
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||
}
|
||||
|
||||
// handActCommand is `hand-act record` and `hand-acts`.
|
||||
func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "record" {
|
||||
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||
if what == "" {
|
||||
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||
}
|
||||
if err := f.require("hand-act record"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||
"act for the same reason will use — it is how a repair done twice is found")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||
}
|
||||
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||
written.Why, written.Cause)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||
}
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
now := time.Now()
|
||||
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeated := link.RepeatedCauses(acts, now)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Println(")")
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
causes := make([]string, 0, len(repeated))
|
||||
for c, n := range repeated {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||
// the log could not be read, which status says rather than reading as none.
|
||||
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||
n := -1
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||
n = len(acts)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return -1, err.Error()
|
||||
}
|
||||
return n, ""
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||
// the verb.
|
||||
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor"}},
|
||||
{"push", map[string]any{}},
|
||||
{"plans", map[string]any{"close": "plan-1"}},
|
||||
{"plans", map[string]any{"stop": "plan-1"}},
|
||||
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||
{"command", map[string]any{"command": "push anchor"}},
|
||||
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if c.verb == "plans" && err == nil {
|
||||
// The seat composes the command line; the command refuses it, before opening anything.
|
||||
err = plansCommand(context.Background(), argv[1:])
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("consumer-reset without why: %v", err)
|
||||
}
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("hand-act record without why: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--cause") {
|
||||
t.Errorf("hand-act record without a cause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||
"plans close plan-1 --why the report will not come"},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
var ds []inventory.Duration
|
||||
for i := 1; i <= 10; i++ {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||
got := summarise(ds)
|
||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
|
||||
|
||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
r.declared = append(r.declared, s.node)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
}
|
||||
|
||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
||||
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
|
||||
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
|
||||
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
|
||||
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
|
||||
//
|
||||
// **The holders on record, and only the sole claimant when there are none** — the same answer the
|
||||
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
|
||||
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
|
||||
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
|
||||
// listed here. A holder off the private network is left out: a resolver named at an address nothing
|
||||
// answers is a lookup that waits out its timeout on every name.
|
||||
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
|
||||
var replicated []catalogue.Seat
|
||||
for _, s := range catalogue.Seats() {
|
||||
if s.Replicated && s.Scope == catalogue.ScopeMesh {
|
||||
replicated = append(replicated, s)
|
||||
}
|
||||
}
|
||||
if len(replicated) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address := map[string]string{}
|
||||
for _, p := range places {
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]map[string]string{}
|
||||
for _, seat := range replicated {
|
||||
var nodes []string
|
||||
for _, h := range recorded {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
|
||||
nodes = append(nodes, h.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
var derived []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||
derived = append(derived, e.On...)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(derived) == 1 {
|
||||
nodes = derived
|
||||
}
|
||||
}
|
||||
at := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
if address[n] != "" {
|
||||
at[overlay.InternalName(n)] = address[n]
|
||||
}
|
||||
}
|
||||
out[seat.Name] = at
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
||||
// one consumer's identity.
|
||||
|
||||
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
||||
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, body string) string {
|
||||
p := filepath.Join(dir, name+".json")
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
objects := write("objects", `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
||||
resolver := write("resolver", `{"module":"resolver","version":"1",
|
||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
||||
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
||||
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
||||
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
||||
}
|
||||
out.Reset()
|
||||
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
||||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
||||
strings.Contains(out.String(), "networkmanager wants") {
|
||||
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
||||
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||
Requires: []string{"wildcard-resolution"}})
|
||||
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||
consumer, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over := consumer.Overflowing()
|
||||
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||
}
|
||||
|
||||
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||
// networkmanager, and no push to the provider could go through.
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||
}
|
||||
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||
}
|
||||
grants, _, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keyless bool
|
||||
for _, g := range grants {
|
||||
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||
}
|
||||
if !keyless {
|
||||
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||
}
|
||||
var granted []string
|
||||
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||
granted = append(granted, c.From)
|
||||
}
|
||||
if strings.Join(granted, ",") != "files" {
|
||||
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||
!strings.Contains(said, "left out of anchor's grants") {
|
||||
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||
}
|
||||
|
||||
// And `status` names it, and does not call the mesh well while it stands.
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||
t.Fatalf("status does not say it:\n%s", shown)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||
doc.Overflowing[0].Bound.Max != 20 {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
}
|
||||
@@ -45,11 +45,11 @@ func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *test
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (int64, error) {
|
||||
allot := func(node string) (order, error) {
|
||||
if node == "anchor" {
|
||||
return 0, context.DeadlineExceeded
|
||||
return order{}, context.DeadlineExceeded
|
||||
}
|
||||
return 7, nil
|
||||
return order{sequence: 7}, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// A command run at a shell (novox/hq to-be 45 §6): under the holder's epoch while a controller holds the
|
||||
// lease, read at the moment it acts; under a lease of its own while none does, given back as it ends.
|
||||
func TestACommandActsUnderTheHoldersEpochOrItsOwn(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
ctx := t.Context()
|
||||
|
||||
// Nobody holds it: the command takes it, and gives it back.
|
||||
cmd := &actor{}
|
||||
own, err := cmd.epoch(ctx)
|
||||
if err != nil || own == 0 {
|
||||
t.Fatalf("a command with nobody holding the lease acts as %d (%v)", own, err)
|
||||
}
|
||||
if h, found, _ := lease.Current(ctx, kv); !found || h.Epoch != own {
|
||||
t.Fatalf("the command's lease is not on the bus: %+v", h)
|
||||
}
|
||||
cmd.release()
|
||||
if _, found, _ := lease.Current(ctx, kv); found {
|
||||
t.Fatal("the command did not give its lease back as it ended")
|
||||
}
|
||||
|
||||
// A controller holds it: a command acts under that epoch.
|
||||
l, err := lease.Open(ctx, mustJetStream(t, url), broker.LeaseBucket, lease.Options{Holder: lease.Holder{Instance: "serving"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := l.TryTake(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
borrower := &actor{}
|
||||
defer borrower.release()
|
||||
if got, err := borrower.epoch(ctx); err != nil || got != held {
|
||||
t.Fatalf("a command acts as %d (%v), want the holder's %d", got, err, held)
|
||||
}
|
||||
// The holder lets go: the command does not go on under an epoch nobody holds.
|
||||
l.Release(ctx)
|
||||
if _, err := borrower.epoch(ctx); err == nil {
|
||||
t.Fatal("a command acted under an epoch nobody holds any more")
|
||||
}
|
||||
}
|
||||
|
||||
// mustJetStream is a connection of its own to the test bus.
|
||||
func mustJetStream(t *testing.T, url string) jetstream.JetStream {
|
||||
t.Helper()
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
|
||||
// serving controllers over one store and one bus. The second waits while the first holds the lease; on
|
||||
// a handover it takes it at a higher epoch, the record says which held what and how each ended; and the
|
||||
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
|
||||
// moment it lost it.
|
||||
//
|
||||
// MESH_TEST_POSTGRES=… MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./cmd/mesh-controller/ -run Controllers
|
||||
|
||||
// aBusForTheLease is the test bus with the controller's lease bucket new.
|
||||
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.KeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) })
|
||||
return url, kv
|
||||
}
|
||||
|
||||
func TestTwoControllersOneActs(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Controller A takes the lease.
|
||||
a := &actor{}
|
||||
aCtx, stopA := context.WithCancel(ctx)
|
||||
defer stopA()
|
||||
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epochA, err := a.epoch(ctx)
|
||||
if err != nil || epochA == 0 {
|
||||
t.Fatalf("A holds no epoch: %d, %v", epochA, err)
|
||||
}
|
||||
|
||||
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
|
||||
b := &actor{}
|
||||
bCtx, stopB := context.WithCancel(ctx)
|
||||
defer stopB()
|
||||
tookB := make(chan (<-chan struct{}), 1)
|
||||
go func() {
|
||||
lost, err := b.serveUnderTheLease(bCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Errorf("B: %v", err)
|
||||
close(tookB)
|
||||
return
|
||||
}
|
||||
tookB <- lost
|
||||
}()
|
||||
select {
|
||||
case <-tookB:
|
||||
t.Fatal("B took the lease while A held it")
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B, waiting, may act: %v", err)
|
||||
}
|
||||
|
||||
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
|
||||
stopA()
|
||||
a.release()
|
||||
var lostB <-chan struct{}
|
||||
select {
|
||||
case lostB = <-tookB:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("B did not take the lease A gave back")
|
||||
}
|
||||
select {
|
||||
case <-lostA:
|
||||
default:
|
||||
t.Fatal("A, having given the lease back, is not told it no longer holds it")
|
||||
}
|
||||
epochB, err := b.epoch(ctx)
|
||||
if err != nil || epochB <= epochA {
|
||||
t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err)
|
||||
}
|
||||
if _, err := a.epoch(ctx); err == nil {
|
||||
t.Fatal("A acts after giving the lease back")
|
||||
}
|
||||
ea, _, _ := inv.EpochOf(ctx, epochA)
|
||||
eb, _, _ := inv.EpochOf(ctx, epochB)
|
||||
if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance {
|
||||
t.Fatalf("the record of the handover reads %+v then %+v", ea, eb)
|
||||
}
|
||||
|
||||
// Something else writes the lease's key — a third controller on a clock that read it as expired:
|
||||
// B's next renewal is refused, and B stops acting at once.
|
||||
if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-lostB:
|
||||
case <-time.After(2 * lease.RenewEvery):
|
||||
t.Fatal("B was not told it lost the lease")
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B acts after losing the lease: %v", err)
|
||||
}
|
||||
// Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition
|
||||
// is not raised.
|
||||
inv.ActsUnder(b.epoch)
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
|
||||
State: inventory.PlanBuilding}
|
||||
if err := inv.SavePlan(ctx, &saved); err == nil {
|
||||
t.Fatal("B wrote a plan after losing the lease")
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
|
||||
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
|
||||
defer keeper.Close(context.Background())
|
||||
if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x",
|
||||
Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil {
|
||||
t.Fatal("B raised a condition after losing the lease")
|
||||
}
|
||||
if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost {
|
||||
t.Fatalf("B's epoch does not say it was lost: %+v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease:
|
||||
// it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can.
|
||||
func TestAControllerTheBusRefusesTheLeaseServesUnleasedAndSaysSo(t *testing.T) {
|
||||
a := &actor{serving: true, unleased: "the bus refused the lease's key"}
|
||||
if epoch, err := a.epoch(context.Background()); err != nil || epoch != 0 {
|
||||
t.Fatalf("an unleased controller answers %d, %v: it acts, claiming no epoch", epoch, err)
|
||||
}
|
||||
if st := a.standing(); st.Unleased == "" || st.Held {
|
||||
t.Fatalf("its standing says %+v", st)
|
||||
}
|
||||
// One that neither holds nor is unleased — still waiting — acts on nothing.
|
||||
waiting := &actor{serving: true}
|
||||
if _, err := waiting.epoch(context.Background()); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("a controller waiting for the lease may act: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -55,6 +55,9 @@ func run() error {
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
||||
// 45 §6), so the next controller takes it at once rather than after its age.
|
||||
defer theLease.release()
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
@@ -145,6 +148,19 @@ func run() error {
|
||||
return seatCommand(ctx, args[1:])
|
||||
case "status":
|
||||
return statusCommand(ctx, args[1:])
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
case "hand-act":
|
||||
return handActCommand(ctx, args[1:])
|
||||
case "hand-acts":
|
||||
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||
case "durations":
|
||||
return durationsCommand(ctx, args[1:])
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
case "conditions":
|
||||
return conditionsCommand(ctx, args[1:])
|
||||
case "doctor":
|
||||
return doctorCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
@@ -194,6 +210,7 @@ func usage() {
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||
@@ -225,19 +242,33 @@ func usage() {
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||
record an act done by hand outside the mesh (to-be 45 §7)
|
||||
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||
conditions [--scope S] [--severity S] [--machine M] [--json]
|
||||
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
||||
conditions show <key> one condition whole, with its evidence
|
||||
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
||||
conditions history [--days N] [--key K] every raising, change and clearing lately
|
||||
doctor [run|probes|signals] [--json]
|
||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||
durations [--kind K] [--days N] [--json]
|
||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||
licence refresh <name> mint a new access token and seal it to every holder
|
||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
||||
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||
pin <node> <provision> <from-node> <module>
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
unpin <node> <provision> put that question back
|
||||
plan <node> [--files|--json] what that node would run, and why
|
||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
||||
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||
that need it; --why records it in the hand-act log
|
||||
version what this binary is
|
||||
|
||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||
@@ -290,6 +321,9 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||
recordBuildDuration(ctx, b.inv, result, asked)
|
||||
switch {
|
||||
case err != nil && result.Failed != "":
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
@@ -316,5 +350,7 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"crypto/ecdh"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -37,6 +40,9 @@ func aMesh(t *testing.T) *stores {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
|
||||
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
|
||||
withConditionsInMemory(t)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -106,3 +112,17 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
||||
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||
}
|
||||
|
||||
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
|
||||
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
|
||||
t.Helper()
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
before := conditionsFrom
|
||||
conditionsFrom = k
|
||||
t.Cleanup(func() {
|
||||
conditionsFrom = before
|
||||
k.Close(context.Background())
|
||||
})
|
||||
return k, store
|
||||
}
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
|
||||
//
|
||||
// The forge's poll announces every merge on the events stream, and the controller acts on what its
|
||||
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
|
||||
// server moved the consumer past it — a fault of consumers with several filters in the server the
|
||||
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
|
||||
// built from that repository stayed behind and were built by hand.
|
||||
//
|
||||
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
|
||||
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
|
||||
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
|
||||
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
|
||||
// would still move something was never acted on — it is said, and acted on now.
|
||||
const (
|
||||
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
|
||||
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
|
||||
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
|
||||
mergeGrace = 10 * time.Minute
|
||||
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
|
||||
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
|
||||
mergeLookBack = 24 * time.Hour
|
||||
// mergeCatchUpEvery is how often the stream is read back.
|
||||
mergeCatchUpEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
// merges is what reads back the forge's announcements; the link server, or a test's list.
|
||||
type merges interface {
|
||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||
}
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := open.inventory.ReadRepositories(ctx)
|
||||
return entries, read, err
|
||||
}
|
||||
failing := ""
|
||||
tick := time.NewTicker(mergeCatchUpEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
watchedMerges.begin()
|
||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||
// says that instead, and leaves what the last one found standing.
|
||||
watchedMerges.end(time.Now(), err)
|
||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("merges the bus may not have handed over are looked for again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
|
||||
// move something is said and acted on, oldest first.
|
||||
//
|
||||
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
|
||||
// because acting on an earlier missed merge of the same repository may have moved what a later one
|
||||
// would have.
|
||||
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
|
||||
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
|
||||
|
||||
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, read, err := catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range all {
|
||||
if now.Sub(a.At) < mergeGrace {
|
||||
continue
|
||||
}
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
moves := wouldMove(a.SourceMoved, entries, read)
|
||||
if len(moves) == 0 {
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moves {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
||||
At: a.At, Modules: names})
|
||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||
isAre(len(names)))
|
||||
if err := act(ctx, a.SourceMoved); err != nil {
|
||||
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
|
||||
a.Owner, a.Repo, a.Commit, err)
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
||||
type missedMerge struct {
|
||||
Owner, Repo, Base, Commit string
|
||||
// At is when the bus took the announcement.
|
||||
At time.Time
|
||||
Modules []string
|
||||
}
|
||||
|
||||
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
||||
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
||||
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
||||
// the fault. The next pass, finding it acted on, clears it.
|
||||
type mergeWatch struct {
|
||||
mu sync.Mutex
|
||||
passed time.Time
|
||||
err error
|
||||
finding []missedMerge
|
||||
missed []missedMerge
|
||||
}
|
||||
|
||||
var watchedMerges = &mergeWatch{}
|
||||
|
||||
func (w *mergeWatch) begin() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = nil
|
||||
}
|
||||
|
||||
func (w *mergeWatch) found(m missedMerge) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = append(w.finding, m)
|
||||
}
|
||||
|
||||
func (w *mergeWatch) end(at time.Time, err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.passed, w.err = at, err
|
||||
if err == nil {
|
||||
w.missed = w.finding
|
||||
}
|
||||
}
|
||||
|
||||
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
||||
// could not read.
|
||||
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// announcedList is the events stream's merges as a test gives them.
|
||||
type announcedList []link.AnnouncedMerge
|
||||
|
||||
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
var out []link.AnnouncedMerge
|
||||
for _, m := range a {
|
||||
if !m.At.Before(since) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
|
||||
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
|
||||
type aCatalogue struct {
|
||||
entries []inventory.Entry
|
||||
acted []string
|
||||
fail error
|
||||
}
|
||||
|
||||
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
return append([]inventory.Entry(nil), c.entries...), nil, nil
|
||||
}
|
||||
|
||||
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
|
||||
return func(_ context.Context, m link.SourceMoved) error {
|
||||
if c.fail != nil {
|
||||
return c.fail
|
||||
}
|
||||
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
|
||||
for _, moved := range wouldMove(m, c.entries, nil) {
|
||||
for i := range c.entries {
|
||||
if c.entries[i].Manifest.Module == moved.Manifest.Module {
|
||||
c.entries[i].Source.Head = m.Commit
|
||||
c.entries[i].Source.Seen = now()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func at(s string) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
|
||||
return link.AnnouncedMerge{
|
||||
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||
MergedAt: mergedAt, Paths: paths,
|
||||
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
|
||||
At: at(onTheBus),
|
||||
}
|
||||
}
|
||||
|
||||
func built(module, repo, path, commit, seen string) inventory.Entry {
|
||||
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
|
||||
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
|
||||
return e
|
||||
}
|
||||
|
||||
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
|
||||
// events stream; the bus never handed it to the controller, which acted on the merges around it and
|
||||
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
|
||||
// move something — so it is said and acted on, once, and only after the controller's own consumer
|
||||
// has had its time with it.
|
||||
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{
|
||||
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
// Acted on when it was announced: looked at after it was merged.
|
||||
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
|
||||
}}
|
||||
stream := announcedList{
|
||||
// Nothing the mesh holds is built from the records repository.
|
||||
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
|
||||
// Acted on: its module was looked at since.
|
||||
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
|
||||
// Never handed over.
|
||||
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
|
||||
"node-tools/internal/console/console.go"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:50:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
pass()
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T22:58:00Z")
|
||||
pass()
|
||||
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
|
||||
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
|
||||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
|
||||
t.Fatalf("the missed merge was not said as one: %q", said)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T23:03:00Z")
|
||||
pass()
|
||||
if len(cat.acted) != 1 || len(said) != 1 {
|
||||
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
|
||||
// that could not be acted on is said and tried again on the next pass.
|
||||
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
|
||||
cat := &aCatalogue{
|
||||
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
|
||||
fail: errors.New("the store is restarting"),
|
||||
}
|
||||
stream := announcedList{
|
||||
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
|
||||
"modules/plex/module.json", "modules/plex/x.ts"),
|
||||
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
|
||||
t.Fatalf("a failed catch-up was not said: %q", said)
|
||||
}
|
||||
cat.fail = nil
|
||||
clock = at("2026-10-05T22:05:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
|
||||
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge older than the look-back is left to the operator: a controller that did not run this
|
||||
// missed it, and acting on it days later would be a surprise rebuild.
|
||||
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
|
||||
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
|
||||
func(string, ...any) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
|
||||
}
|
||||
}
|
||||
@@ -59,8 +59,19 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||
if args[0] == "check" {
|
||||
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
||||
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
||||
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
||||
"judge each module's identity on a machine name this many characters long")
|
||||
given, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *longest < 1 {
|
||||
return errors.New("--longest-machine-name is a length, at least 1")
|
||||
}
|
||||
var paths []string
|
||||
for _, a := range args[1:] {
|
||||
for _, a := range given {
|
||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||
under, err := manifestsUnder(a)
|
||||
if err != nil {
|
||||
@@ -71,7 +82,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
paths = append(paths, a)
|
||||
}
|
||||
return moduleCheck(paths, os.Stdout)
|
||||
return moduleCheckFor(paths, *longest, os.Stdout)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -661,7 +672,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
// durable subscription nobody reads.
|
||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -387,7 +388,7 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
return busAccounts(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||
return consumerReset(args[1:])
|
||||
return consumerReset(ctx, args[1:])
|
||||
}
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||
@@ -414,10 +415,21 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||
func consumerReset(args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
||||
func consumerReset(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
args, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||
}
|
||||
if err := why.require("broker consumer-reset"); err != nil {
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "broker consumer-reset", args)
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -505,6 +517,26 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
fmt.Printf(" public domain %s\n", domain)
|
||||
}
|
||||
|
||||
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
|
||||
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
|
||||
// is something not working now and they are a description. Unreadable is said, not passed over.
|
||||
open, err := openConditions(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
|
||||
}
|
||||
var here []conditions.Condition
|
||||
for _, c := range open {
|
||||
if concerns(c, name) {
|
||||
here = append(here, c)
|
||||
}
|
||||
}
|
||||
if len(here) > 0 {
|
||||
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
|
||||
for _, line := range conditionLines(here, time.Now()) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+58
-17
@@ -399,7 +399,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
@@ -464,6 +464,11 @@ func reportLeftOut(node string, declared sendable) {
|
||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||
node, m, declared.leftOutWhy[m])
|
||||
}
|
||||
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||
// 0225): the machine is sent everything else, and the consumer is named.
|
||||
for _, o := range declared.withheld {
|
||||
fmt.Printf("%s: %s\n", node, o)
|
||||
}
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
@@ -471,7 +476,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||
inv := open.inventory
|
||||
grants, err := grantsFor(ctx, open, node)
|
||||
grants, withheld, err := grantsFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
@@ -707,6 +712,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
|
||||
// lists every holder of the mesh's resolver.
|
||||
replicas, err := replicatedHolders(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
||||
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
||||
// before it had an address on the private network. A machine joins through the tunnel now, and
|
||||
@@ -783,11 +795,11 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines, Zones: zones,
|
||||
Machines: machines, Zones: zones, Holders: replicas,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||
BusUsers: busUsers,
|
||||
BusUsers: busUsers, Withheld: withheld,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
@@ -923,28 +935,34 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
|
||||
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
||||
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
||||
// the mesh hands over something it cannot itself use.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
||||
//
|
||||
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
|
||||
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
|
||||
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
|
||||
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, []catalogue.Overflow, error) {
|
||||
inv := open.inventory
|
||||
issued, err := inv.SecretsFrom(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
||||
// the mesh names machines.
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
||||
// from a record beside it. A provider told to create a password and not what to create it for
|
||||
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
||||
out := make([]catalogue.Grant, 0, len(issued))
|
||||
var withheld []catalogue.Overflow
|
||||
for _, s := range issued {
|
||||
plan, settings, err := planFor(ctx, open, s.Consumer)
|
||||
switch {
|
||||
@@ -957,11 +975,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
||||
// nothing — and withholding a grant on that reading takes a consumer's access away
|
||||
// (novox/hq 04-ISSUES/152).
|
||||
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
return nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
}
|
||||
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
// A port in there is the consumer's software port until this. The consumer is on another
|
||||
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
||||
@@ -969,7 +987,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
||||
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
||||
from := s.ConsumerModule
|
||||
@@ -980,9 +998,10 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
from = ""
|
||||
}
|
||||
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
||||
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
||||
// remedy a short slug rather than a login a provider silently shortened.
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
|
||||
// this provision, as the consumer's resolution states it from the provider's offer (ADR
|
||||
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
|
||||
// provider silently shortened — and rather than this whole machine refused for it.
|
||||
slug := ""
|
||||
for _, mm := range plan.Modules {
|
||||
if mm.Module == s.ConsumerModule {
|
||||
@@ -991,15 +1010,32 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
}
|
||||
}
|
||||
if from != "" {
|
||||
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
||||
return nil, err
|
||||
bound := boundOfGrant(plan, s, node)
|
||||
source := catalogue.IdentitySource(slug, s.ConsumerModule)
|
||||
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
|
||||
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
|
||||
Consumer: s.Consumer, Module: s.ConsumerModule,
|
||||
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, catalogue.Grant{
|
||||
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
||||
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
||||
}
|
||||
return out, nil
|
||||
return out, withheld, nil
|
||||
}
|
||||
|
||||
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
|
||||
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
|
||||
// than to none: what the provider keeps of it is not known here.
|
||||
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
|
||||
for _, n := range consumer.Needs {
|
||||
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
|
||||
return n.Identity
|
||||
}
|
||||
}
|
||||
return catalogue.DefaultIdentityBound
|
||||
}
|
||||
|
||||
// listensLines is what a person is told about what this module would open, and why — the same
|
||||
@@ -1075,6 +1111,11 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
left := plan.LeftOut(settings, record.Adopted)
|
||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||
}
|
||||
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||
for _, o := range plan.Overflowing() {
|
||||
fmt.Printf("%s: %s\n", args[0], o)
|
||||
}
|
||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||
// stored before its definition moved from under it.
|
||||
for _, m := range plan.Modules {
|
||||
|
||||
@@ -280,7 +280,7 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
}
|
||||
}
|
||||
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.State != inventory.PlanBuilding {
|
||||
@@ -323,7 +323,7 @@ func joinAPlan(ctx context.Context, open *stores, module string) (bool, string,
|
||||
askModule(ctx, &p, module, byName)
|
||||
s := p.Modules[module]
|
||||
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if s.State != "asked" {
|
||||
@@ -393,7 +393,7 @@ func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||
if err := open.inventory.SavePlan(ctx, *p); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
|
||||
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
|
||||
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,901 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"github.com/novox/mesh-host/validate"
|
||||
"golang.org/x/net/dns/dnsmessage"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// The probes of the self-check's registry (doctor.go), one function each. A probe answers what is
|
||||
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
||||
// look" (ADR 0227 rule 4).
|
||||
|
||||
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
||||
// (mesh-host's `validate`, the package the host runs) takes it.
|
||||
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
open := d.open
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
// The private network every declaration is composed with. Not computable is not "could not
|
||||
// look": it is the finding — no machine's declaration composes without it — and the machines that
|
||||
// do not resolve, which are usually why, are named below.
|
||||
gens, gensErr := generators(ctx, open)
|
||||
if gensErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "private-network",
|
||||
Token: "uncomposable", Severity: conditions.Urgent,
|
||||
Summary: "the private network cannot be computed, so no machine's declaration composes",
|
||||
Said: firstLine(gensErr.Error())})
|
||||
}
|
||||
for _, n := range nodes {
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil && !unresolvable(err) {
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
var problems []string
|
||||
if err == nil && gensErr == nil {
|
||||
var declared sendable
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
||||
// With the order it was last sent, so the validator reads the envelope a machine is sent
|
||||
// — its epoch included (novox/hq to-be 45 §6).
|
||||
var serr error
|
||||
if declared.Sequence, serr = open.inventory.Sequence(ctx, n.ID); serr == nil {
|
||||
declared.Epoch, serr = open.inventory.SentEpoch(ctx, n.ID)
|
||||
}
|
||||
if serr != nil {
|
||||
return nil, serr // the store, not the machine: the probe could not run
|
||||
}
|
||||
var body []byte
|
||||
if body, err = declared.Body(); err == nil {
|
||||
problems = validate.Declaration(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "uncomposable",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("nothing can be sent to %s: its declaration does not compose — %s", n.Name,
|
||||
oneLine(err.Error())),
|
||||
Said: oneLine(err.Error())})
|
||||
case len(problems) > 0:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "refused",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
||||
n.Name, len(problems), problems[0]),
|
||||
Said: strings.Join(problems, "; ")})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
||||
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
||||
// takes as final (issue 262).
|
||||
func probeResolvers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
holders, err := replicatedHolders(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolvers := holders["mesh-dns-resolver"]
|
||||
if len(resolvers) == 0 {
|
||||
return nil, nil // the mesh holds no resolver of its own: nothing is asked of one
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
suffix := overlay.Suffix()
|
||||
var out []conditions.Observation
|
||||
for holder, at := range resolvers {
|
||||
var wrong []string
|
||||
for _, p := range places {
|
||||
if p.Address == "" {
|
||||
continue
|
||||
}
|
||||
name := p.Name + "." + suffix
|
||||
v4, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeA)
|
||||
switch {
|
||||
case err != nil:
|
||||
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "A", name, err))
|
||||
continue
|
||||
case rcode != dnsmessage.RCodeSuccess:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (A)", holder, rcode, name))
|
||||
case !slices.Contains(v4, p.Address):
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (A), not %s", holder, v4, name, p.Address))
|
||||
}
|
||||
v6, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeAAAA)
|
||||
switch {
|
||||
case err != nil:
|
||||
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "AAAA", name, err))
|
||||
case rcode != dnsmessage.RCodeSuccess:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (AAAA), not NODATA: a musl machine "+
|
||||
"takes that as no such name", holder, rcode, name))
|
||||
case len(v6) > 0:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (AAAA); the mesh has no IPv6 addresses", holder, v6, name))
|
||||
}
|
||||
}
|
||||
if len(wrong) > 0 {
|
||||
node := strings.TrimSuffix(holder, "."+suffix)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver." + node,
|
||||
Token: "wrong", Machine: node, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s",
|
||||
node, wrong[0]),
|
||||
Said: strings.Join(wrong, "; ")})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// resolverPort is where a resolver answers; a variable so a test can stand one up.
|
||||
var resolverPort = "53"
|
||||
|
||||
// askResolver asks one resolver one question over UDP, and answers the addresses and the code.
|
||||
func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]string, dnsmessage.RCode, error) {
|
||||
q, err := dnsmessage.NewName(strings.TrimSuffix(name, ".") + ".")
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
msg := dnsmessage.Message{Header: dnsmessage.Header{ID: uint16(time.Now().UnixNano()), RecursionDesired: true},
|
||||
Questions: []dnsmessage.Question{{Name: q, Type: kind, Class: dnsmessage.ClassINET}}}
|
||||
packed, err := msg.Pack()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
dialer := net.Dialer{Timeout: 3 * time.Second}
|
||||
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(at, resolverPort))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
|
||||
if _, err := conn.Write(packed); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
buf := make([]byte, 1500)
|
||||
n, err := conn.Read(buf)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("no answer from %s: %w", at, err)
|
||||
}
|
||||
var answer dnsmessage.Message
|
||||
if err := answer.Unpack(buf[:n]); err != nil {
|
||||
return nil, 0, fmt.Errorf("an answer from %s that cannot be read: %w", at, err)
|
||||
}
|
||||
var addresses []string
|
||||
for _, a := range answer.Answers {
|
||||
switch r := a.Body.(type) {
|
||||
case *dnsmessage.AResource:
|
||||
addresses = append(addresses, net.IP(r.A[:]).String())
|
||||
case *dnsmessage.AAAAResource:
|
||||
addresses = append(addresses, net.IP(r.AAAA[:]).String())
|
||||
}
|
||||
}
|
||||
return addresses, answer.RCode, nil
|
||||
}
|
||||
|
||||
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
||||
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
||||
// S1's, and is not asked about here.
|
||||
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recorded, err := d.open.inventory.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
expected := map[string]map[string]bool{} // seat → machine
|
||||
add := func(seat, node string) {
|
||||
if expected[seat] == nil {
|
||||
expected[seat] = map[string]bool{}
|
||||
}
|
||||
expected[seat][node] = true
|
||||
}
|
||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
||||
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
||||
}
|
||||
onRecord := false
|
||||
for _, h := range recorded {
|
||||
if h.Claim == s.Name && heard[h.Node] {
|
||||
add(s.Name, h.Node)
|
||||
onRecord = true
|
||||
}
|
||||
}
|
||||
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name != s.Name {
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
||||
add(s.Name, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(expected) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
answering, err := discoverHolders(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for seat, nodes := range expected {
|
||||
for node := range nodes {
|
||||
if answering[seat][node] {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: seat + "." + node,
|
||||
Token: "silent", Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s's holder on %s does not answer the bus: its verbs reach nothing there", seat, node),
|
||||
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
||||
func heardMachines(d *doctor) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if d.watchdogs == nil {
|
||||
return out
|
||||
}
|
||||
f := d.watchdogs.lastFacts()
|
||||
if f == nil {
|
||||
return out
|
||||
}
|
||||
for _, m := range f.machines {
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// discoveryPatience is how long the discovery's answers are waited for after the last arrived, and at
|
||||
// the most (ADR 0197: a large runtime's answer arrives last).
|
||||
const (
|
||||
discoveryQuiet = 1500 * time.Millisecond
|
||||
discoveryPatience = 8 * time.Second
|
||||
)
|
||||
|
||||
// discoverHolders asks the bus's discovery who serves what, and answers seat → machine for every
|
||||
// endpoint a seat's verb is served on.
|
||||
func discoverHolders(ctx context.Context, conn *nats.Conn) (map[string]map[string]bool, error) {
|
||||
inbox := conn.NewRespInbox()
|
||||
sub, err := conn.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||
}
|
||||
out := map[string]map[string]bool{}
|
||||
deadline := time.Now().Add(discoveryPatience)
|
||||
for time.Now().Before(deadline) {
|
||||
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
break
|
||||
}
|
||||
var info micro.Info
|
||||
if json.Unmarshal(msg.Data, &info) != nil {
|
||||
continue
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
seat, node := e.Metadata["seat"], e.Metadata["node"]
|
||||
if seat == "" {
|
||||
continue
|
||||
}
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
if out[seat] == nil {
|
||||
out[seat] = map[string]bool{}
|
||||
}
|
||||
out[seat][node] = true
|
||||
}
|
||||
// A holder that announces itself as its seat, by name and machine.
|
||||
if out[info.Name] == nil {
|
||||
out[info.Name] = map[string]bool{}
|
||||
}
|
||||
out[info.Name][info.ID] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeArchives is D4: every archive the mesh keeps is held by its manifest in the artifact store.
|
||||
func probeArchives(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if store == "" {
|
||||
return nil, errors.New("the mesh keeps archives and has no artifact store on its network to ask")
|
||||
}
|
||||
var report collectionReport
|
||||
if unasked, stopped := askHeld(ctx, artifacts.Store{Address: store}, kept, &report); stopped != "" {
|
||||
return nil, fmt.Errorf("%d kept archive(s) could not be asked about: %s", unasked, stopped)
|
||||
}
|
||||
var out []conditions.Observation
|
||||
if n := len(report.Unheld); n > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-unheld",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%d of %d kept archive(s) are not held by a manifest: the store's collector would "+
|
||||
"delete them", n, len(kept)),
|
||||
Said: "first: " + report.Unheld[0]})
|
||||
}
|
||||
if n := len(report.Missing); n > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-missing",
|
||||
Kind: "archives-missing", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%d kept archive(s) are not in the artifact store at all", n),
|
||||
Said: "first: " + report.Missing[0]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// consumerFarBehind is how far a durable consumer may be from its stream's head (D6).
|
||||
const consumerFarBehind = 1000
|
||||
|
||||
// probeConsumers is D6: every durable consumer the mesh expects exists, as the controller defines it,
|
||||
// and is near its stream's head.
|
||||
func probeConsumers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
_, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var out []conditions.Observation
|
||||
for _, c := range consumers {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
info, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||
who := c.Stream + "." + c.Name
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "missing",
|
||||
Kind: "consumer-lost", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(c)),
|
||||
Said: "no consumer " + c.Name + " on " + c.Stream})
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||
}
|
||||
if differs := consumerDiffers(c, info.Config); differs != "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "redefined",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is not as the controller defines it: %s", consumerWords(c), differs),
|
||||
Said: differs})
|
||||
}
|
||||
if c.Stream != "NODES" && info.NumPending > consumerFarBehind {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(c), info.NumPending),
|
||||
Said: fmt.Sprintf("%d pending, %d handed out and not settled", info.NumPending, info.NumAckPending)})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// consumerWords is a consumer as the mesh says it, with its name.
|
||||
func consumerWords(c broker.Consumer) string {
|
||||
return fmt.Sprintf("%s (%s on %s)", link.ConsumerInWords(c.Stream, c.Name), c.Name, c.Stream)
|
||||
}
|
||||
|
||||
// consumerDiffers is what about a consumer on the bus is not as defined; empty when nothing is. The
|
||||
// delivery subject is not compared: one kept as it was while a holder is bound is the assertion's
|
||||
// stated choice (novox/hq issue 156).
|
||||
func consumerDiffers(want broker.Consumer, have nats.ConsumerConfig) string {
|
||||
var differs []string
|
||||
haveFilters := append([]string(nil), have.FilterSubjects...)
|
||||
if have.FilterSubject != "" {
|
||||
haveFilters = append(haveFilters, have.FilterSubject)
|
||||
}
|
||||
wantFilters := append([]string(nil), want.Filters...)
|
||||
sort.Strings(haveFilters)
|
||||
sort.Strings(wantFilters)
|
||||
if !slices.Equal(haveFilters, wantFilters) {
|
||||
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
||||
}
|
||||
if have.AckPolicy != nats.AckExplicitPolicy {
|
||||
differs = append(differs, "acknowledges "+have.AckPolicy.String()+", defined explicit")
|
||||
}
|
||||
if wantMax := want.MaxDeliver; wantMax != 0 && have.MaxDeliver != wantMax {
|
||||
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, wantMax))
|
||||
}
|
||||
if wantWait := time.Duration(want.AckWaitSeconds) * time.Second; wantWait != 0 && have.AckWait != wantWait {
|
||||
differs = append(differs, fmt.Sprintf("waits %s for an acknowledgement, defined %s", have.AckWait, wantWait))
|
||||
}
|
||||
if want.MaxAckPending != 0 && have.MaxAckPending != want.MaxAckPending {
|
||||
differs = append(differs, fmt.Sprintf("hands out %d at once, defined %d", have.MaxAckPending, want.MaxAckPending))
|
||||
}
|
||||
if wantPush, havePush := want.Push || want.Queue != "", have.DeliverSubject != ""; wantPush != havePush {
|
||||
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// probeStreams is D7: every stream the controller defines exists as defined, and its own buckets.
|
||||
func probeStreams(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
streams, _, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var out []conditions.Observation
|
||||
for _, s := range streams {
|
||||
info, err := js.StreamInfo(s.Name, nats.Context(ctx))
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrStreamNotFound):
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-missing",
|
||||
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||
Said: "no stream " + s.Name})
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||
}
|
||||
if differs := streamDiffers(s, info.Config); differs != "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-redefined",
|
||||
Kind: "stream-wrong", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the stream %s is not as the controller defines it: %s", s.Name, differs),
|
||||
Said: differs})
|
||||
}
|
||||
}
|
||||
for _, bucket := range broker.ControllerBuckets() {
|
||||
if _, err := js.StreamInfo("KV_"+bucket, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: bucket, Token: "bucket-missing",
|
||||
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller's bucket %s is not on the bus: what it keeps there is not kept", bucket),
|
||||
Said: "no bucket " + bucket})
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s cannot be read: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// streamDiffers is what about a stream on the bus is not as defined; empty when nothing is.
|
||||
func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||
var differs []string
|
||||
haveSubjects := append([]string(nil), have.Subjects...)
|
||||
wantSubjects := append([]string(nil), want.Subjects...)
|
||||
sort.Strings(haveSubjects)
|
||||
sort.Strings(wantSubjects)
|
||||
if !slices.Equal(haveSubjects, wantSubjects) {
|
||||
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
||||
}
|
||||
retention, perSubject := nats.LimitsPolicy, int64(want.MaxMsgsPerSubject)
|
||||
switch want.Retention {
|
||||
case broker.RetentionWorkQueue:
|
||||
retention = nats.WorkQueuePolicy
|
||||
case broker.RetentionLastPerSubject:
|
||||
perSubject = 1
|
||||
}
|
||||
if have.Retention != retention {
|
||||
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
||||
}
|
||||
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
||||
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// ipLike finds the addresses in a ban list, whatever shape its holder answers in.
|
||||
var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
||||
|
||||
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are
|
||||
// every machine's private address and the address its endpoint names; the ban lists are asked of each
|
||||
// machine's holder of `node-intrusion-prevention` that is heard from.
|
||||
func probeBans(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owned := map[string]string{} // address → whose
|
||||
for _, p := range places {
|
||||
if p.Address != "" {
|
||||
owned[p.Address] = p.Name + "'s private address"
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(p.Endpoint); err == nil && host != "" {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
owned[ip.String()] = p.Name + "'s endpoint"
|
||||
} else if addrs, err := net.DefaultResolver.LookupHost(ctx, host); err == nil {
|
||||
for _, a := range addrs {
|
||||
owned[a] = p.Name + "'s endpoint (" + host + ")"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var holders []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name == "node-intrusion-prevention" {
|
||||
for _, node := range e.On {
|
||||
if heard[node] && !slices.Contains(holders, node) {
|
||||
holders = append(holders, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(holders)
|
||||
var out []conditions.Observation
|
||||
// Every machine asked at once: one after another, four holders that each wait their bound
|
||||
// outlast the probe's thirty seconds, and the probe says nothing about any of them.
|
||||
answers := make([]json.RawMessage, len(holders))
|
||||
errs := make([]error, len(holders))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range holders {
|
||||
wg.Add(1)
|
||||
go func(i int, node string) {
|
||||
defer wg.Done()
|
||||
answers[i], errs[i] = askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
|
||||
}(i, node)
|
||||
}
|
||||
wg.Wait()
|
||||
var unasked []string
|
||||
for i, node := range holders {
|
||||
answer, err := answers[i], errs[i]
|
||||
if err != nil {
|
||||
unasked = append(unasked, err.Error())
|
||||
continue
|
||||
}
|
||||
var banned []string
|
||||
for _, ip := range ipLike.FindAllString(string(answer), -1) {
|
||||
if whose, ours := owned[ip]; ours && !slices.Contains(banned, ip+" ("+whose+")") {
|
||||
banned = append(banned, ip+" ("+whose+")")
|
||||
}
|
||||
}
|
||||
if len(banned) > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "bans-the-mesh",
|
||||
Machine: node, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es): %s — the mesh is locked out "+
|
||||
"of itself there (ADR 0186)", node, len(banned), strings.Join(banned, ", ")),
|
||||
Said: strings.Join(banned, ", ")})
|
||||
}
|
||||
}
|
||||
if len(unasked) > 0 {
|
||||
return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeStatus is D9: `status` answers in full within ten seconds, from a summary composed lately.
|
||||
func probeStatus(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
started := time.Now()
|
||||
stale := ""
|
||||
if statusFrom != nil {
|
||||
answer, err := statusFrom.answer(ctx)
|
||||
if err != nil {
|
||||
stale = err.Error()
|
||||
} else if m, ok := answer.(map[string]any); ok {
|
||||
if failed, _ := m["lastAttemptFailed"].(string); failed != "" {
|
||||
stale = "its last composition failed: " + failed
|
||||
}
|
||||
if composed, err := time.Parse(time.RFC3339, fmt.Sprint(m["composed"])); err == nil &&
|
||||
time.Since(composed) > 3*statusEvery+statusComposeWithin {
|
||||
stale = fmt.Sprintf("it answers a summary composed %s ago", time.Since(composed).Round(time.Second))
|
||||
}
|
||||
}
|
||||
} else if _, err := theThreeQuestions(ctx, d.open); err != nil {
|
||||
stale = err.Error()
|
||||
}
|
||||
took := time.Since(started)
|
||||
var out []conditions.Observation
|
||||
if took > 10*time.Second || stale != "" {
|
||||
why := stale
|
||||
if why == "" {
|
||||
why = fmt.Sprintf("it took %s", took.Round(time.Millisecond))
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller", Token: "status-slow",
|
||||
Machine: d.host, Severity: conditions.Warning,
|
||||
Summary: "status does not answer in full within ten seconds: " + why, Said: why})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeCoreBuilds is D10: every machine runs the node-engine and the node tools the mesh holds, or a
|
||||
// plan is rolling one of them out. The node-engine says its build in each report; the node tools' is
|
||||
// the build the machine was last sent, once it reported applying that send.
|
||||
func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hostVersions := deliveredVersions(shelf[hostModule])
|
||||
rolling := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
for m := range p.Modules {
|
||||
rolling[m] = true
|
||||
}
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applied := map[string]bool{}
|
||||
for _, r := range reports {
|
||||
applied[r.Node] = r.Current
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var out []conditions.Observation
|
||||
for _, n := range nodes {
|
||||
if !heard[n.Name] {
|
||||
continue // a machine not heard from is S1's
|
||||
}
|
||||
var behind []string
|
||||
// **A node-engine says its version as the directory it was delivered into** — its archive's
|
||||
// digest, twelve characters (catalogue.versionOf, ADR 0141) — not the commit it was built
|
||||
// from. Compared as a commit, every machine read as behind right after a push sent it the
|
||||
// current one (2026-10-06). An engine placed by hand reports its link-time stamp instead,
|
||||
// which a commit can match.
|
||||
if !rolling[hostModule] && engineBehind(n.HostVersion, hostVersions, current[hostModule].Commit) {
|
||||
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)",
|
||||
n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit)))
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tools := current[broker.RuntimeModule].Commit
|
||||
if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] {
|
||||
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case !known:
|
||||
case !sameCommit(sent[broker.RuntimeModule], tools):
|
||||
behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s",
|
||||
short(orNotKnown(sent[broker.RuntimeModule])), short(tools)))
|
||||
case !applied[n.Name]:
|
||||
behind = append(behind, "the node tools it was last sent, not yet reported applied")
|
||||
}
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind",
|
||||
Machine: n.Name, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s runs %s, and no plan is rolling them out", n.Name, strings.Join(behind, "; ")),
|
||||
Said: strings.Join(behind, "; ")})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
||||
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
||||
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
||||
func deliveredVersions(m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, r := range m.Resources {
|
||||
path, _ := r["path"].(string)
|
||||
before, version, found := strings.Cut(path, "/versions/")
|
||||
if !found || before == "" || version == "" || strings.Contains(version, "/") || strings.Contains(version, "$") {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(out, version) {
|
||||
out = append(out, version)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// engineBehind says a node-engine's reported version is not the build the mesh holds: neither the
|
||||
// directory that build is delivered as, nor (for an engine placed by hand) its commit. A machine that
|
||||
// has not said, or a mesh that holds no delivered build, is not behind anything.
|
||||
func engineBehind(reported string, delivered []string, commit string) bool {
|
||||
if reported == "" || len(delivered) == 0 {
|
||||
return false
|
||||
}
|
||||
return !slices.Contains(delivered, reported) && !sameCommit(reported, commit)
|
||||
}
|
||||
|
||||
// hostModule is the node-engine's module.
|
||||
const hostModule = "mesh-host"
|
||||
|
||||
// sameCommit says two commits are one, either written short.
|
||||
func sameCommit(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
|
||||
}
|
||||
|
||||
func orNotKnown(s string) string {
|
||||
if s == "" {
|
||||
return "(not known)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// probeWatchdogs is DW: the watchdogs ran within three of their intervals. The doctor and the
|
||||
// watchdogs watch each other: S10 is the other half.
|
||||
func probeWatchdogs(_ context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.watchdogs == nil {
|
||||
return nil, nil // a self-check run outside the serving controller has none to watch
|
||||
}
|
||||
ticked := d.watchdogs.lastTick()
|
||||
since := ticked
|
||||
if since.IsZero() {
|
||||
since = d.watchdogs.started
|
||||
}
|
||||
if time.Since(since) <= 3*watchEvery {
|
||||
return nil, nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "watchdogs", Token: "silent",
|
||||
Machine: d.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the watchdogs of the signals table have not run since %s: no late signal is being said",
|
||||
since.UTC().Format("2006-01-02 15:04 MST")),
|
||||
Said: fmt.Sprintf("no tick for %s", time.Since(since).Round(time.Second))}}, nil
|
||||
}
|
||||
|
||||
// askSeatTool asks one machine's holder of a node seat a verb and answers its result; the holder's
|
||||
// own refusal is an error.
|
||||
func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string) (json.RawMessage, error) {
|
||||
if who, declared := declaredBy(ctx, seat, verb); !declared {
|
||||
return nil, fmt.Errorf("%s asks %s.%s, which it does not declare in the probe registry — and so the "+
|
||||
"controller is not granted it", who, seat, verb)
|
||||
}
|
||||
answer, err := link.AskSeatTool(ctx, conn, seat, verb, node, map[string]any{}, 10*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s's %s.%s refused: %s", node, seat, verb, answer.Error)
|
||||
}
|
||||
return answer.Result, nil
|
||||
}
|
||||
|
||||
// oneLine is a message of several lines said on one, its runs of space made one.
|
||||
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
// probeLease is D5 (novox/hq to-be 45 §4, §6): exactly one lease holder — the key on the bus names this
|
||||
// controller at the epoch it acts under, and the mesh's record has that epoch and no other open — and no
|
||||
// message from a stale epoch refused in the last interval.
|
||||
func probeLease(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
st := theLease.standing()
|
||||
var out []conditions.Observation
|
||||
split := func(token, summary, said string) {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: token,
|
||||
Machine: d.host, Severity: conditions.Urgent, Summary: summary, Said: said})
|
||||
}
|
||||
if st.Unleased != "" {
|
||||
split("unheld", "this controller acts without the lease, so nothing keeps another from acting beside it: "+
|
||||
st.Unleased, st.Unleased)
|
||||
return out, nil
|
||||
}
|
||||
api, err := jetstream.New(d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.LeaseBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the lease bucket cannot be read: %w", err)
|
||||
}
|
||||
holder, found, err := lease.Current(ctx, kv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the lease cannot be read: %w", err)
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
split("split", fmt.Sprintf("nobody holds the lease on the bus, and this controller acts as epoch %d", st.Epoch),
|
||||
"the lease's key is absent")
|
||||
case holder.Instance != instance || holder.Epoch != st.Epoch:
|
||||
split("split", fmt.Sprintf("the lease on the bus names %s at epoch %d, and this controller (%s) acts as "+
|
||||
"epoch %d: two controllers believe they may act", holder.Instance, holder.Epoch, instance, st.Epoch),
|
||||
fmt.Sprintf("held by %s, epoch %d", holder.Instance, holder.Epoch))
|
||||
}
|
||||
// The record: one epoch open, this one.
|
||||
epochs, err := d.open.inventory.EpochsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the epochs the mesh issued cannot be read: %w", err)
|
||||
}
|
||||
var open []string
|
||||
for _, e := range epochs {
|
||||
if e.Ended == nil && e.Epoch != st.Epoch {
|
||||
open = append(open, fmt.Sprintf("epoch %d (%s)", e.Epoch, e.Instance))
|
||||
}
|
||||
}
|
||||
if len(open) > 0 {
|
||||
split("open-epochs", fmt.Sprintf("the mesh's record holds %s open beside this controller's epoch %d: a "+
|
||||
"holder that neither gave the lease back nor was found expired", strings.Join(open, ", "), st.Epoch),
|
||||
strings.Join(open, ", "))
|
||||
}
|
||||
// And no message from a stale epoch in the last interval.
|
||||
for _, w := range link.StaleRefusals.Within(time.Now().Add(-doctorEvery)) {
|
||||
if w.Epoch <= 0 || uint64(w.Epoch) >= st.Epoch {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: fmt.Sprintf("controller.epoch-%d", w.Epoch),
|
||||
Token: "stale-epoch", Machine: firstOf(w.Receivers), Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%d declaration(s) from epoch %d — older than this controller's %d — reached %s in the "+
|
||||
"last %s and were refused: a controller that lost the lease is still sending", w.Count, w.Epoch, st.Epoch,
|
||||
strings.Join(w.Receivers, ", "), doctorEvery),
|
||||
Said: fmt.Sprintf("%d refused, the last at %s", w.Count, w.Last.UTC().Format(time.RFC3339))})
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
+195
-59
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
@@ -62,7 +63,7 @@ func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.En
|
||||
return link.ConnectNats(js, enroller, listener), nil
|
||||
}
|
||||
|
||||
func serve(ctx context.Context) error {
|
||||
func serve(ctx context.Context) (err error) {
|
||||
// The one process whose log is read over time, so the one that says each change to a node's
|
||||
// unmet seat dependencies once (novox/hq ADR 0207).
|
||||
logUnheldChanges = true
|
||||
@@ -103,9 +104,47 @@ func serve(ctx context.Context) error {
|
||||
// being live is refused, because a mesh half on each is one where a declaration goes out on one
|
||||
// and the report comes back on the other, and every component logs success while it happens.
|
||||
|
||||
// **The lease, before anything that acts** (novox/hq to-be 45 §6): asserting the bus's objects is the
|
||||
// controller's to do, and so is everything after. A controller starting while another holds it waits
|
||||
// here, said; one that loses it stops: every act's gate closes at once, and ctx ends so the process
|
||||
// exits and is started again as a candidate.
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lost, err := theLease.serveUnderTheLease(ctx, inv, busAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Given back before the store closes, so the epoch is recorded as given back rather than found
|
||||
// expired by the next holder.
|
||||
defer theLease.release()
|
||||
ctx, stopActing := context.WithCancel(ctx)
|
||||
defer stopActing()
|
||||
go func() {
|
||||
select {
|
||||
case <-lost:
|
||||
stopActing()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
select {
|
||||
case <-lost:
|
||||
if err == nil {
|
||||
err = errors.New("the controller lease was lost; this controller stopped acting and exits, to " +
|
||||
"be started again as a candidate")
|
||||
}
|
||||
default:
|
||||
}
|
||||
}()
|
||||
|
||||
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
||||
OnNATS: true}
|
||||
server, err := connectLink(ctx, inv, work, work)
|
||||
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
|
||||
// something new is one thing that moves it, so the listener nudges it.
|
||||
statusFrom = newStatusSummary(composeStatus(open))
|
||||
server, err := connectLink(ctx, inv, work, nudgingListener{Enrolment: work, summary: statusFrom, open: open})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -121,17 +160,27 @@ func serve(ctx context.Context) error {
|
||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||
go planTicker(ctx, open)
|
||||
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||
go forgettingOldDurations(ctx, inv)
|
||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||
// above; this is the other half — the control plane is the only one of the three that knows
|
||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||
if err := server.Follows(following{open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the merges the bus announced and never handed over, read back on a timer and acted on late
|
||||
// rather than never (novox/hq issue 266).
|
||||
go catchingUpOnMerges(ctx, open, server)
|
||||
// And a catalogue that has just started, asking for what it missed. The same type answers
|
||||
// both: what a build meant and what the builds were are two questions about one record.
|
||||
if err := server.Answers(following{open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||
// 0224): a provider's journal must not be the only place that says so.
|
||||
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
|
||||
// from the store's row, so what the seat declares is what is answered.
|
||||
@@ -150,6 +199,33 @@ func serve(ctx context.Context) error {
|
||||
if !isNATS {
|
||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||
}
|
||||
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||
handActConn = bus.Conn
|
||||
// And says when it replaced a value given by hand (novox/hq ADR 0228).
|
||||
givenEvents = bus
|
||||
// Composed now and kept current, before the verb that answers from it is served.
|
||||
go statusFrom.keep(ctx)
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
||||
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
||||
// And every call is kept on the bus, so a restart of this process keeps what came of each
|
||||
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
|
||||
// answering no calls at all would be worse than answering them without the record.
|
||||
said := log.New(os.Stdout, "", log.LstdFlags)
|
||||
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
|
||||
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
|
||||
} else if err := link.Calls.Durably(ctx, keeper, instance, said); err != nil {
|
||||
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
|
||||
}
|
||||
// What is wrong, kept and said (novox/hq to-be 45 §2): the condition store, the watchdogs of the
|
||||
// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
|
||||
// is said and the controller serves on: status then says the conditions cannot be read, and is
|
||||
// not well — louder than not serving at all, and the push that repairs the bus still runs.
|
||||
if stopWatching := watchTheMesh(ctx, open, server, bus); stopWatching != nil {
|
||||
defer stopWatching()
|
||||
}
|
||||
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -219,7 +295,12 @@ func declare(ctx context.Context, args []string) error {
|
||||
// is still what the machine was last told, and status must not read it as current for the one
|
||||
// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
|
||||
// the mesh did not compose it, so a push that does not name this machine treats it as held.
|
||||
if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
|
||||
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
||||
var carried struct {
|
||||
Epoch uint64 `json:"epoch"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &carried)
|
||||
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||
@@ -250,6 +331,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
||||
wait := set.Duration("wait", 0,
|
||||
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
||||
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
|
||||
// recorded when given at a shell — see handacts.go for why a shell is not refused.
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -264,6 +348,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
||||
"other asks which machines need one")
|
||||
}
|
||||
recorded := append([]string(nil), args...)
|
||||
if *behind {
|
||||
recorded = append(recorded, "--behind")
|
||||
}
|
||||
why.record(ctx, "push", recorded)
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -317,7 +406,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
if len(needsOne) == 0 {
|
||||
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
|
||||
// must never look the same.
|
||||
fmt.Println("every machine is doing what it was told")
|
||||
fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -358,6 +447,18 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
asked = append(asked, n.Name)
|
||||
}
|
||||
// **A push that named no machine says so, first.** Through the console a machine the caller
|
||||
// meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as
|
||||
// `push --behind`, and every machine behind was pushed by someone who thought they had pushed one.
|
||||
// Its whole-mesh reach is the first line of the answer, with the machines it is about to send.
|
||||
if len(args) == 0 {
|
||||
which := "every machine"
|
||||
if *behind {
|
||||
which = "every machine that is behind"
|
||||
}
|
||||
fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n",
|
||||
which, len(asked), strings.Join(asked, ", "))
|
||||
}
|
||||
|
||||
// **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's
|
||||
// user list, and a module's new grants are refused by the bus until that list says them. Among
|
||||
@@ -425,7 +526,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
release()
|
||||
fmt.Printf("\n%d node(s) told\n", len(sending))
|
||||
told := make([]string, 0, len(sending))
|
||||
for _, r := range sending {
|
||||
told = append(told, r.node)
|
||||
}
|
||||
fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", "))
|
||||
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
|
||||
|
||||
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
||||
@@ -524,7 +629,7 @@ type readyNode struct {
|
||||
//
|
||||
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
||||
// across two machines that must agree — and dropped here, where it never did.
|
||||
func composeEach(names []string, allot func(node string) (int64, error),
|
||||
func composeEach(names []string, allot func(node string) (order, error),
|
||||
compose func(node string) (sendable, error)) ([]readyNode, []string) {
|
||||
|
||||
var sending []readyNode
|
||||
@@ -538,7 +643,7 @@ func composeEach(names []string, allot func(node string) (int64, error),
|
||||
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
|
||||
// two machines was undone two seconds later by exactly that. Taken here, before the first
|
||||
// read, what was composed earlier is numbered lower whatever order the sends happen in.
|
||||
seq, err := allot(name)
|
||||
numbered, err := allot(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
@@ -548,7 +653,7 @@ func composeEach(names []string, allot func(node string) (int64, error),
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence = seq
|
||||
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||
if len(declared.Resources) == 0 {
|
||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||
@@ -716,6 +821,13 @@ type overTheBus struct {
|
||||
}
|
||||
|
||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
|
||||
// when they cannot be, and never what holds the send back (assertOnSend).
|
||||
if bus, ok := b.server.Bus().(link.OverNATS); ok {
|
||||
js := broker.OnConn(bus.Conn)
|
||||
js.Note = func(format string, args ...any) { fmt.Printf(b.indent+" "+format+"\n", args...) }
|
||||
_ = assertOnSend(ctx, b.open.inventory, js, b.indent)
|
||||
}
|
||||
return issueMemberships(ctx, b.open, b.server, sending)
|
||||
}
|
||||
|
||||
@@ -725,7 +837,7 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -878,7 +990,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
|
||||
seq, err := allot(ctx, inv, name)
|
||||
numbered, err := allot(ctx, inv, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
@@ -894,7 +1006,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence = seq
|
||||
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||
reportLeftOut(name, declared)
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
@@ -1013,6 +1125,20 @@ func digestOf(body []byte) string {
|
||||
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
||||
func wouldSend(ctx context.Context, open *stores,
|
||||
nodes []inventory.Node) (map[string]string, error) {
|
||||
return wouldSendFrom(ctx, open, nodes, nil)
|
||||
}
|
||||
|
||||
// planned is one machine's plan as planFor answered it, for a caller that already asked.
|
||||
type planned struct {
|
||||
plan catalogue.Resolution
|
||||
settings catalogue.SettingsBy
|
||||
}
|
||||
|
||||
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
|
||||
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
|
||||
// to-be 45 Phase 0). A machine absent from plans is worked out here.
|
||||
func wouldSendFrom(ctx context.Context, open *stores,
|
||||
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
|
||||
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
@@ -1020,9 +1146,12 @@ func wouldSend(ctx context.Context, open *stores,
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
known, have := plans[n.Name]
|
||||
plan, settings := known.plan, known.settings
|
||||
if !have {
|
||||
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
@@ -1034,6 +1163,11 @@ func wouldSend(ctx context.Context, open *stores,
|
||||
if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And the epoch it was last sent under, for the same reason: a new holder of the lease is not a
|
||||
// change of the machine (novox/hq to-be 45 §6).
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1081,35 +1215,22 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
}
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
// The mesh's own streams and consumers, the seats' work queues and their workers, and how every
|
||||
// machine hears its declaration — one derivation, which the self-check reads as well (D6, D7).
|
||||
names, err := assertBusObjects(ctx, inv, js)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
if err := broker.Raise(js, names); err != nil {
|
||||
return err
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||
// consumer nothing had created (2026-09-28).
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
|
||||
return err
|
||||
}
|
||||
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
|
||||
// whether it was cancelled the moment it took it.
|
||||
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
|
||||
// by hand, kept where a restart of this process does not take them.
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
||||
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
||||
// nothing declares any more is said and kept — what it holds is data.
|
||||
@@ -1125,25 +1246,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||
}
|
||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
records, err := inv.BusRecords(ctx)
|
||||
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||
hearing, err := moduleConsumerCount(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hearing := 0
|
||||
for _, c := range broker.ConsumersOf(users) {
|
||||
if err := js.EnsureConsumer(c.Consumer); err != nil {
|
||||
return fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
|
||||
}
|
||||
hearing++
|
||||
}
|
||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||
return nil
|
||||
@@ -1186,17 +1293,46 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
|
||||
|
||||
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
|
||||
// allotting is allot over one inventory, in the shape composeEach takes.
|
||||
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (int64, error) {
|
||||
return func(node string) (int64, error) { return allot(ctx, inv, node) }
|
||||
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (order, error) {
|
||||
return func(node string) (order, error) { return allot(ctx, inv, node) }
|
||||
}
|
||||
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries.
|
||||
func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, error) {
|
||||
// epochForActs is the lease's gate as a composition asks it; a variable so a test can act under an epoch
|
||||
// without a bus.
|
||||
var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }
|
||||
|
||||
// order is what a declaration carries of its writer's order (link/order.go): its sequence, and the
|
||||
// epoch of the lease it is composed under — zero for a machine that has not said it reads one.
|
||||
type order struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
}
|
||||
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
||||
// lease: a process that may not act takes none, and composes nothing (novox/hq to-be 45 §6).
|
||||
func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, error) {
|
||||
epoch, err := epochForActs(ctx)
|
||||
if err != nil {
|
||||
return order{}, fmt.Errorf("nothing was composed for %s: %w", node, err)
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return order{}, err
|
||||
}
|
||||
return inv.NextSequence(ctx, record.ID)
|
||||
if epoch > 0 {
|
||||
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
if !reads {
|
||||
epoch = 0
|
||||
}
|
||||
}
|
||||
seq, err := inv.NextSequence(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
return order{sequence: seq, epoch: epoch}, nil
|
||||
}
|
||||
|
||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||
@@ -1211,7 +1347,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, e
|
||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||
builds map[string]string) (string, error) {
|
||||
builds map[string]string, epoch uint64) (string, error) {
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
record, err := inv.NodeByName(kept, node)
|
||||
@@ -1219,7 +1355,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
||||
return "", err
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
|
||||
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
|
||||
@@ -76,7 +76,7 @@ func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||
}
|
||||
|
||||
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||
func numbered() func(string) (int64, error) {
|
||||
func numbered() func(string) (order, error) {
|
||||
var n int64
|
||||
return func(string) (int64, error) { n++; return n, nil }
|
||||
return func(string) (order, error) { n++; return order{sequence: n}, nil }
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.KilledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
||||
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
||||
@@ -433,7 +433,7 @@ func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
|
||||
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -631,21 +631,21 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
if err := open.inventory.SavePlan(ctx, stopped); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &stopped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A newer plan holding a refuses it: sending the older build would put it back.
|
||||
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
|
||||
t.Fatalf("retried under a newer plan: %v", err)
|
||||
}
|
||||
newer.State = inventory.PlanSuperseded
|
||||
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -683,7 +683,7 @@ func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
||||
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -78,6 +79,24 @@ type meshStatus struct {
|
||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||
// dependency is met. Reported, not refused, until the switch.
|
||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
|
||||
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
|
||||
// HandActsUnread says why when it could not be read, rather than reading as none.
|
||||
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
|
||||
HandActsUnread string `json:"handActsUnread,omitempty"`
|
||||
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
|
||||
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
|
||||
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
|
||||
Conditions []conditions.Condition `json:"conditions"`
|
||||
ConditionsUnread string `json:"conditionsUnread,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
|
||||
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
|
||||
// provider says so. A document without it called the mesh well while the identity provider
|
||||
// refused every consumer for a day (04-ISSUES/179).
|
||||
Failing []conditions.Condition `json:"failing,omitempty"`
|
||||
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
|
||||
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
|
||||
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
@@ -210,6 +229,13 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
|
||||
if out.Conditions == nil {
|
||||
out.Conditions = []conditions.Condition{}
|
||||
}
|
||||
out.Failing = providerStandings(asked.conditions)
|
||||
out.Overflowing = asked.overflowing
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -441,7 +441,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
||||
state.BuiltAt = &now
|
||||
state.Commit = commit
|
||||
}
|
||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
continue
|
||||
}
|
||||
@@ -500,7 +500,7 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||
// the state is left as it was and the step is tried again on the next tick.
|
||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
}
|
||||
break
|
||||
@@ -508,7 +508,7 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
if p.State == inventory.PlanFailed {
|
||||
sayUnsent(p, rollsOut)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
break
|
||||
}
|
||||
@@ -986,10 +986,18 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
||||
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||
// Refused before anything is opened: a repair by hand says why.
|
||||
if err := why.require("plans " + positionals[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1061,13 +1069,14 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if !p.Open() {
|
||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
||||
|
||||
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
|
||||
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||
// would bury the ones that matter under a list nobody can act on.
|
||||
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// rotateCommand replaces a credential and moves both ends together.
|
||||
@@ -33,12 +35,16 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||
// One consuming module rather than every module on the machine. A machine runs many consumers
|
||||
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
|
||||
// issue 268) is no reason to restart every other one on the machine.
|
||||
module := set.String("module", "", "only this consuming module's credential")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("rotate <provision> [--consumer <machine>]")
|
||||
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
|
||||
}
|
||||
provision := positionals[0]
|
||||
|
||||
@@ -53,6 +59,12 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
holders = ofModule(holders, *module)
|
||||
if len(holders) == 0 && *module != "" {
|
||||
return fmt.Errorf(
|
||||
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
|
||||
"says what a machine holds", *module, onMachine(*only), provision)
|
||||
}
|
||||
if len(holders) == 0 {
|
||||
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||
// and a rotation somebody believes happened is worse than one they know did not.
|
||||
@@ -116,6 +128,27 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ofModule is the holders whose consuming module is this one; all of them when none is named.
|
||||
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
|
||||
if module == "" {
|
||||
return holders
|
||||
}
|
||||
var out []inventory.Holder
|
||||
for _, h := range holders {
|
||||
if h.ConsumerModule == module {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func onMachine(machine string) string {
|
||||
if machine == "" {
|
||||
return ""
|
||||
}
|
||||
return " on " + machine
|
||||
}
|
||||
|
||||
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
||||
func asLocal(local string) string {
|
||||
if local == "" {
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
|
||||
// 268): a module that leaked its database password is no reason to restart every other consumer.
|
||||
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
|
||||
holders := []inventory.Holder{
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
|
||||
}
|
||||
got := ofModule(holders, "letta")
|
||||
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
|
||||
t.Fatalf("narrowed to letta: %+v", got)
|
||||
}
|
||||
if len(ofModule(holders, "")) != 3 {
|
||||
t.Fatal("no module named narrowed anyway")
|
||||
}
|
||||
if len(ofModule(holders, "absent")) != 0 {
|
||||
t.Fatal("a module holding nothing matched")
|
||||
}
|
||||
}
|
||||
@@ -29,11 +29,13 @@ type seatHolder struct {
|
||||
|
||||
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
|
||||
type seatRow struct {
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
||||
rows := make([]seatRow, 0, len(seats))
|
||||
for _, s := range seats {
|
||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||
Holders: []seatHolder{}}
|
||||
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||
seen := map[seatHolder]bool{}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
if len(args) == 3 && args[1] == "--to" {
|
||||
return handOver(ctx, args[0], args[2])
|
||||
return handOver(ctx, args[0], args[2], false)
|
||||
}
|
||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
|
||||
if len(args) == 3 && args[1] == "--add" {
|
||||
return handOver(ctx, args[0], args[2], true)
|
||||
}
|
||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
|
||||
"seat <name> --add <node>/<module>")
|
||||
}
|
||||
|
||||
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
|
||||
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
|
||||
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
|
||||
// and refusing to record a handover to a module the node has not started would make the handover
|
||||
// impossible to do before the switch instead of as the switch.
|
||||
func handOver(ctx context.Context, seatName, to string) error {
|
||||
//
|
||||
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
|
||||
// records the named assignment as a further holder and leaves every holder on record as it is. A
|
||||
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
|
||||
// the one named, as it always did.
|
||||
func handOver(ctx context.Context, seatName, to string, adding bool) error {
|
||||
nodeName, module, ok := strings.Cut(to, "/")
|
||||
if !ok || nodeName == "" || module == "" {
|
||||
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
||||
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
if !known {
|
||||
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
|
||||
}
|
||||
if adding && !seat.Replicated {
|
||||
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
|
||||
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
|
||||
}
|
||||
var was string
|
||||
var held []string
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
for _, h := range holdings {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
|
||||
was = h.Node
|
||||
held = append(held, h.Node)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
} else if err := catalogue.CanHold(*m, seat); err != nil {
|
||||
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
|
||||
}
|
||||
if adding {
|
||||
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
|
||||
strings.Join(held, ", "))
|
||||
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
|
||||
return nil
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -9,9 +9,11 @@ import (
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -36,19 +38,194 @@ type verbAnswer struct {
|
||||
|
||||
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
|
||||
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
|
||||
//
|
||||
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
|
||||
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
|
||||
// the verb declares but did not use for the command line it composed — given beside another that
|
||||
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
|
||||
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
|
||||
// not take that" would have stopped it before anything was sent.
|
||||
func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
str := func(key string) string {
|
||||
v, _ := args[key].(string)
|
||||
return strings.TrimSpace(v)
|
||||
a, err := readArguments(verb, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
need := func(keys ...string) error {
|
||||
for _, k := range keys {
|
||||
if str(k) == "" {
|
||||
return fmt.Errorf("%s needs %q", verb, k)
|
||||
argv, err := a.commandLine()
|
||||
if len(a.misread) > 0 {
|
||||
// The table and the command line disagree: the verb reads an argument no caller can see
|
||||
// in its schema, so no caller could ever pass it.
|
||||
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
|
||||
"table and its command lines disagree", verb, quoteAll(a.misread))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unused := a.unused(); len(unused) > 0 {
|
||||
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
|
||||
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
|
||||
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
|
||||
// command line was composed from.
|
||||
type verbArguments struct {
|
||||
verb string
|
||||
given map[string]string
|
||||
used map[string]bool
|
||||
declared map[string]bool
|
||||
misread []string // arguments the command line read that the schema does not declare: a bug here
|
||||
}
|
||||
|
||||
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
|
||||
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
|
||||
// wrote.
|
||||
func controllerVerb(name string) (catalogue.Verb, bool) {
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == name {
|
||||
return v, true
|
||||
}
|
||||
}
|
||||
return catalogue.Verb{}, false
|
||||
}
|
||||
|
||||
// declaredArguments are a schema's properties, and which of them are switches.
|
||||
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
|
||||
switches = map[string]bool{}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
for name, p := range props {
|
||||
names = append(names, name)
|
||||
desc, _ := p.(map[string]any)
|
||||
switch enum := desc["enum"].(type) {
|
||||
case []string:
|
||||
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
||||
case []any:
|
||||
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, switches
|
||||
}
|
||||
|
||||
// readArguments refuses what the verb does not take, before anything is composed.
|
||||
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
|
||||
v, known := controllerVerb(verb)
|
||||
if !known {
|
||||
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
declared := map[string]bool{}
|
||||
for _, n := range names {
|
||||
declared[n] = true
|
||||
}
|
||||
takes := "none"
|
||||
if len(names) > 0 {
|
||||
takes = quoteAll(names)
|
||||
}
|
||||
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
|
||||
keys := make([]string, 0, len(args))
|
||||
for k := range args {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if !declared[k] {
|
||||
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
|
||||
}
|
||||
var value string
|
||||
switch x := args[k].(type) {
|
||||
case nil:
|
||||
continue
|
||||
case string:
|
||||
value = strings.TrimSpace(x)
|
||||
case bool:
|
||||
if !switches[k] {
|
||||
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
|
||||
}
|
||||
value = fmt.Sprint(x)
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
|
||||
}
|
||||
if switches[k] {
|
||||
switch value {
|
||||
case "true":
|
||||
case "false", "":
|
||||
continue // said and off: the same as not given, and nothing passed over
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
if value != "" {
|
||||
a.given[k] = value
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// str is one argument's value, marked as used.
|
||||
func (a *verbArguments) str(key string) string {
|
||||
if !a.declared[key] {
|
||||
a.misread = append(a.misread, key)
|
||||
}
|
||||
a.used[key] = true
|
||||
return a.given[key]
|
||||
}
|
||||
|
||||
// on is a switch, marked as used.
|
||||
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
|
||||
|
||||
// need refuses a call missing a required argument, in the verb's own words.
|
||||
func (a *verbArguments) need(keys ...string) error {
|
||||
for _, k := range keys {
|
||||
if a.str(k) == "" {
|
||||
return fmt.Errorf("%s needs %q", a.verb, k)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unused are the arguments given that the command line was not composed from.
|
||||
func (a *verbArguments) unused() []string {
|
||||
var out []string
|
||||
for k := range a.given {
|
||||
if !a.used[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *verbArguments) usedGiven() []string {
|
||||
var out []string
|
||||
for k := range a.given {
|
||||
if a.used[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
if len(out) == 0 {
|
||||
return []string{"nothing"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func quoteAll(xs []string) string {
|
||||
q := make([]string, len(xs))
|
||||
for i, x := range xs {
|
||||
if x == "nothing" {
|
||||
q[i] = x
|
||||
continue
|
||||
}
|
||||
q[i] = fmt.Sprintf("%q", x)
|
||||
}
|
||||
return strings.Join(q, ", ")
|
||||
}
|
||||
|
||||
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
|
||||
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
|
||||
func (a *verbArguments) commandLine() ([]string, error) {
|
||||
verb, str, on, need := a.verb, a.str, a.on, a.need
|
||||
switch verb {
|
||||
case "command":
|
||||
// The generic verb: the command line as given, split as a shell would split it, with
|
||||
@@ -64,7 +241,15 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if len(argv) == 0 {
|
||||
return nil, errors.New("command names no command")
|
||||
}
|
||||
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||
// it says why, as it would through its own verb.
|
||||
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||
}
|
||||
return argv, nil
|
||||
case "tools":
|
||||
return nil, errors.New("tools is answered from the records, not by a command")
|
||||
case "status":
|
||||
return []string{"status", "--json"}, nil
|
||||
case "nodes":
|
||||
@@ -82,10 +267,14 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if id := str("log"); id != "" {
|
||||
return []string{"builds", "--log", id}, nil
|
||||
}
|
||||
if m := str("module"); m != "" {
|
||||
return []string{"builds", m}, nil
|
||||
argv := []string{"builds"}
|
||||
if n := str("limit"); n != "" {
|
||||
argv = append(argv, "-n", n)
|
||||
}
|
||||
return []string{"builds"}, nil
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, m)
|
||||
}
|
||||
return argv, nil
|
||||
case "plans":
|
||||
if r := str("repository"); r != "" {
|
||||
argv := []string{"plans", "--what-if", r}
|
||||
@@ -97,19 +286,30 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if id := str("stop"); id != "" {
|
||||
return []string{"plans", "stop", id}, nil
|
||||
}
|
||||
if id := str("close"); id != "" {
|
||||
return []string{"plans", "close", id}, nil
|
||||
}
|
||||
if id := str("retry"); id != "" {
|
||||
return []string{"plans", "retry", id}, nil
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if id := str(act); id != "" {
|
||||
argv := []string{"plans", act, id}
|
||||
if act == "retry" {
|
||||
return argv, nil
|
||||
}
|
||||
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
}
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"plans", id}, nil
|
||||
}
|
||||
return []string{"plans"}, nil
|
||||
argv := []string{"plans"}
|
||||
if n := str("limit"); n != "" {
|
||||
argv = append(argv, "-n", n)
|
||||
}
|
||||
return argv, nil
|
||||
// The build queue (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return []string{"queue"}, nil
|
||||
@@ -119,7 +319,7 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
}
|
||||
return []string{verb, str("id")}, nil
|
||||
case "clear":
|
||||
if str("dead") == "true" {
|
||||
if on("dead") {
|
||||
return []string{"clear", "--dead"}, nil
|
||||
}
|
||||
return []string{"clear"}, nil
|
||||
@@ -133,10 +333,10 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"replay", str("id")}
|
||||
if str("register") == "true" {
|
||||
if on("register") {
|
||||
argv = append(argv, "--register")
|
||||
}
|
||||
if str("older") == "true" {
|
||||
if on("older") {
|
||||
argv = append(argv, "--older")
|
||||
}
|
||||
return argv, nil
|
||||
@@ -149,6 +349,9 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if on("files") {
|
||||
return []string{"plan", str("node"), "--files"}, nil
|
||||
}
|
||||
return []string{"plan", str("node"), "--json"}, nil
|
||||
case "assign", "unassign":
|
||||
if err := need("node", "module"); err != nil {
|
||||
@@ -171,23 +374,126 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||
if n := str("node"); n != "" {
|
||||
return []string{"push", n, "--wait", "0"}, nil
|
||||
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
||||
if err := need("why"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
||||
}
|
||||
return []string{"push", "--behind", "--wait", "0"}, nil
|
||||
why := []string{"--why", str("why")}
|
||||
if c := str("cause"); c != "" {
|
||||
why = append(why, "--cause", c)
|
||||
}
|
||||
if n := str("node"); n != "" {
|
||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||
// a machine and asking for every machine behind are two requests, and guessing one
|
||||
// would push a machine nobody named, or not push one somebody did.
|
||||
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
||||
}
|
||||
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||
// first, so a caller who meant one machine reads that it was not one.
|
||||
on("behind")
|
||||
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
||||
case "hand-act":
|
||||
if err := need("what", "why", "cause"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
|
||||
if c := str("condition"); c != "" {
|
||||
argv = append(argv, "--condition", c)
|
||||
}
|
||||
return argv, nil
|
||||
case "hand-acts":
|
||||
argv := []string{"hand-acts", "--json"}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "durations":
|
||||
argv := []string{"durations", "--json"}
|
||||
if k := str("kind"); k != "" {
|
||||
argv = append(argv, "--kind", k)
|
||||
}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "conditions":
|
||||
// One verb, four shapes, as `plans` (novox/hq to-be 45 §2): a silence, one condition, the
|
||||
// history, or the open ones filtered.
|
||||
if key := str("silence"); key != "" {
|
||||
if err := need("for", "why"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"conditions", "silence", key, "--for", str("for"), "--why", str("why")}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if on("history") {
|
||||
argv := []string{"conditions", "history", "--json"}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
if k := str("key"); k != "" {
|
||||
argv = append(argv, "--key", k)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if k := str("key"); k != "" {
|
||||
return []string{"conditions", "show", k, "--json"}, nil
|
||||
}
|
||||
argv := []string{"conditions", "--json"}
|
||||
for _, filter := range []string{"scope", "severity", "machine"} {
|
||||
if v := str(filter); v != "" {
|
||||
argv = append(argv, "--"+filter, v)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
case "doctor":
|
||||
which := 0
|
||||
argv := []string{"doctor"}
|
||||
for _, sub := range []string{"run", "probes", "signals"} {
|
||||
if on(sub) {
|
||||
which++
|
||||
argv = append(argv, sub)
|
||||
}
|
||||
}
|
||||
if which > 1 {
|
||||
return nil, errors.New("doctor answers one of run, probes or signals at a time")
|
||||
}
|
||||
return append(argv, "--json"), nil
|
||||
case "rotate":
|
||||
if p := str("provision"); p != "" {
|
||||
argv := []string{"rotate", p}
|
||||
if c := str("consumer"); c != "" {
|
||||
argv = append(argv, "--consumer", c)
|
||||
}
|
||||
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
|
||||
// and secret stay the other shape's, and are refused as passed over.
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, "--module", m)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if str("node") != "" && str("module") != "" && str("secret") != "" {
|
||||
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
||||
_, node := a.given["node"]
|
||||
_, module := a.given["module"]
|
||||
_, secret := a.given["secret"]
|
||||
if node || module || secret {
|
||||
if err := need("node", "module", "secret"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
|
||||
}
|
||||
argv := []string{"secret", "rotate", str("node"), str("module"), str("secret")}
|
||||
// Why, recorded in the hand-act log (novox/hq ADR 0228); a cause only beside a why.
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
// Half of either shape: the command says its usage, which names both shapes, and that is
|
||||
// the answer the caller needs.
|
||||
// Neither shape: the command says its usage, which names both, and that is the answer the
|
||||
// caller needs.
|
||||
return []string{"rotate"}, nil
|
||||
case "settings":
|
||||
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
||||
@@ -195,15 +501,16 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if err := need("module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"settings", "set", str("module")}
|
||||
switch {
|
||||
case str("clear") == "true":
|
||||
var argv []string
|
||||
if on("clear") {
|
||||
argv = []string{"settings", "clear", str("module")}
|
||||
case str("values") != "":
|
||||
argv = append(argv, str("values"))
|
||||
} else {
|
||||
argv = []string{"settings", "set", str("module")}
|
||||
// Neither values nor clear: the command says its usage, which names both.
|
||||
if v := str("values"); v != "" {
|
||||
argv = append(argv, v)
|
||||
}
|
||||
}
|
||||
// Neither values nor clear: the command says its usage, which names both, and that is the
|
||||
// answer the caller needs — the same as `rotate` given half of either shape.
|
||||
if n := str("node"); n != "" {
|
||||
argv = append(argv, "--node", n)
|
||||
}
|
||||
@@ -235,11 +542,35 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
||||
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
|
||||
verb, catalogue.ControllerSeatName)
|
||||
}
|
||||
|
||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||
"hand-acts": true, "durations": true, "conditions": true, "doctor": true}
|
||||
|
||||
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||
func repairingCommand(argv []string) string {
|
||||
switch {
|
||||
case argv[0] == "push":
|
||||
return "push"
|
||||
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close"):
|
||||
return "plans " + argv[1]
|
||||
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||
return "broker consumer-reset"
|
||||
case argv[0] == "hand-act":
|
||||
return "hand-act record"
|
||||
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
||||
return "conditions silence"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isWhyFlag(word string) bool {
|
||||
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
||||
}
|
||||
|
||||
// runVerb runs this binary with the given command line and gathers what it said.
|
||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
@@ -251,6 +582,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||
// from a shell in this container would have, because it is that.
|
||||
cmd.Env = os.Environ()
|
||||
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||
caller := link.CallerIn(ctx)
|
||||
if caller == "" {
|
||||
caller = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||
@@ -287,8 +624,35 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
handlers := map[string]link.ToolHandler{}
|
||||
for _, v := range seat.Serves {
|
||||
verb := v.Name
|
||||
if verb == "tools" {
|
||||
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
|
||||
if inProcess[verb] {
|
||||
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
||||
args := map[string]any{}
|
||||
if len(bytes.TrimSpace(raw)) > 0 {
|
||||
if err := json.Unmarshal(raw, &args); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
|
||||
}
|
||||
}
|
||||
// Refused like any verb's: what a verb does not take is not ignored.
|
||||
a, err := readArguments(verb, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verb == "calls" {
|
||||
return callsAnswer(link.Calls, a.given["call"])
|
||||
}
|
||||
if verb == "doctor" {
|
||||
// From the serving controller, which runs the self-check and hears the signals
|
||||
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
|
||||
argv, err := a.commandLine()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sub := ""
|
||||
if len(argv) > 2 {
|
||||
sub = argv[1]
|
||||
}
|
||||
return doctorAnswer(ctx, sub)
|
||||
}
|
||||
return seatTools(), nil
|
||||
}
|
||||
continue
|
||||
@@ -327,12 +691,85 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verb == "status" && statusFrom != nil {
|
||||
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||
return statusFrom.answer(ctx)
|
||||
}
|
||||
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||
// Whatever it did, `status` is composed again once it has.
|
||||
defer statusFrom.nudge()
|
||||
}
|
||||
if answersFirst(argv) {
|
||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||
link.Acknowledge(ctx)
|
||||
}
|
||||
return runVerb(ctx, argv)
|
||||
}
|
||||
}
|
||||
return handlers, behind, nil
|
||||
}
|
||||
|
||||
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||
func actsOnAPlan(args map[string]any) bool {
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||
// the records, `calls` from what this process served.
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
|
||||
|
||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
|
||||
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
|
||||
func answersFirst(argv []string) bool {
|
||||
return len(argv) > 0 && argv[0] == "push"
|
||||
}
|
||||
|
||||
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
||||
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
|
||||
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
|
||||
// the reason said beside it.
|
||||
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||
if id != "" {
|
||||
c, ok, err := log.Get(id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
||||
"bus could not be read: %w", id, err)
|
||||
}
|
||||
if !ok {
|
||||
if log.IsDurable() {
|
||||
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
|
||||
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||
}
|
||||
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
||||
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
recent, err := log.Recent()
|
||||
for i := range recent {
|
||||
recent[i].Answer = nil
|
||||
}
|
||||
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
|
||||
if log.IsDurable() {
|
||||
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
|
||||
broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||
} else {
|
||||
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
|
||||
}
|
||||
if err != nil {
|
||||
answer["unread"] = err.Error()
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
|
||||
func seatTools() map[string]any {
|
||||
@@ -353,9 +790,10 @@ func seatTools() map[string]any {
|
||||
}
|
||||
|
||||
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
|
||||
// verb runnable rather than that it happens to want the arguments the check guessed.
|
||||
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
|
||||
// more, since an argument a verb does not declare is refused.
|
||||
func sampleArguments(v catalogue.Verb) map[string]any {
|
||||
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
|
||||
sample := map[string]any{}
|
||||
switch required := v.Input["required"].(type) {
|
||||
case []string:
|
||||
for _, k := range required {
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The schemas the controller serves, verb by verb, as the console receives them: from the
|
||||
// announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244).
|
||||
func servedSchemas(t *testing.T) map[string]catalogue.Verb {
|
||||
t.Helper()
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(behind) != 0 {
|
||||
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
|
||||
}
|
||||
served := map[string]catalogue.Verb{}
|
||||
for _, e := range seatAnnouncement(handlers).Endpoints {
|
||||
var input map[string]any
|
||||
if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil {
|
||||
t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err)
|
||||
}
|
||||
served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input}
|
||||
}
|
||||
if len(served) != len(catalogue.ControllerVerbs) {
|
||||
t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs))
|
||||
}
|
||||
return served
|
||||
}
|
||||
|
||||
// subsetsOf is every subset of the names, the empty one included.
|
||||
func subsetsOf(names []string) [][]string {
|
||||
var out [][]string
|
||||
for mask := 0; mask < 1<<len(names); mask++ {
|
||||
var s []string
|
||||
for i, n := range names {
|
||||
if mask&(1<<i) != 0 {
|
||||
s = append(s, n)
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func argumentsFor(subset []string, switches map[string]bool) map[string]any {
|
||||
args := map[string]any{}
|
||||
for _, n := range subset {
|
||||
if switches[n] {
|
||||
args[n] = "true"
|
||||
} else {
|
||||
args[n] = "x-" + n
|
||||
}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// saidOutright are the switches that say the default aloud, so the line is the same without them —
|
||||
// on purpose, and only these.
|
||||
var saidOutright = map[string]string{
|
||||
"push": "behind", // the whole mesh is what no machine means; behind lets a caller say they meant it
|
||||
}
|
||||
|
||||
// **No argument a caller gives is passed over** (novox/hq issue 244). For every verb served and
|
||||
// every combination of the arguments its schema declares, the verb either refuses the call, or
|
||||
// composes a command line that each given argument changed: taking any one away changes the line
|
||||
// or makes it refused. An argument the line is the same without is one the verb ignored — the
|
||||
// shape of the push that named a machine and pushed every machine behind.
|
||||
func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
if inProcess[name] {
|
||||
continue
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
for _, subset := range subsetsOf(names) {
|
||||
args := argumentsFor(subset, switches)
|
||||
argv, err := argvFor(name, args)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "does not declare") {
|
||||
t.Errorf("%s %v: %v", name, args, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, dropped := range subset {
|
||||
fewer := map[string]any{}
|
||||
for k, val := range args {
|
||||
if k != dropped {
|
||||
fewer[k] = val
|
||||
}
|
||||
}
|
||||
without, err := argvFor(name, fewer)
|
||||
if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped {
|
||||
t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with
|
||||
// what it requires and one argument more; and `node` in particular, for every verb whose schema
|
||||
// does not take a machine.
|
||||
func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
names, _ := declaredArguments(v)
|
||||
strangers := []string{"no-such-argument"}
|
||||
if !contains(names, "node") {
|
||||
strangers = append(strangers, "node")
|
||||
}
|
||||
for _, stranger := range strangers {
|
||||
args := sampleArguments(v)
|
||||
args[stranger] = "x"
|
||||
_, err := argvFor(name, args)
|
||||
if inProcess[name] {
|
||||
_, err = readArguments(name, args)
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
|
||||
t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil {
|
||||
t.Error("a switch took a word that is neither true nor false, and would have read it as false")
|
||||
}
|
||||
if _, err := argvFor("node", map[string]any{"node": 7}); err == nil {
|
||||
t.Error("a number was taken as a machine's name, or as no machine")
|
||||
}
|
||||
if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" {
|
||||
t.Errorf("a switch given as JSON true: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
||||
// naming one beside behind is refused rather than one of the two guessed.
|
||||
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
|
||||
t.Fatalf("a named push: %v %v", argv, err)
|
||||
}
|
||||
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
||||
argv, err := argvFor("push", args)
|
||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
|
||||
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil ||
|
||||
!strings.Contains(err.Error(), `"behind"`) {
|
||||
t.Fatalf("a named push with behind was taken: %v", err)
|
||||
}
|
||||
if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) {
|
||||
t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// commandFlags is every flag set this package's commands parse, by the name the set is made with,
|
||||
// and the flags defined on it — read from the source, so a flag added to a command is seen here
|
||||
// without anyone remembering to.
|
||||
func commandFlags(t *testing.T) map[string][]string {
|
||||
t.Helper()
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fset := token.NewFileSet()
|
||||
out := map[string][]string{}
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
file, err := parser.ParseFile(fset, f, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, decl := range file.Decls {
|
||||
fn, ok := decl.(*ast.FuncDecl)
|
||||
if !ok || fn.Body == nil {
|
||||
continue
|
||||
}
|
||||
sets := map[string]string{} // variable → the set's name
|
||||
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
||||
if assign, ok := n.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 && len(assign.Rhs) == 1 {
|
||||
if call, ok := assign.Rhs[0].(*ast.CallExpr); ok && isSelector(call.Fun, "flag", "NewFlagSet") {
|
||||
if id, ok := assign.Lhs[0].(*ast.Ident); ok {
|
||||
if name, ok := stringLit(call.Args[0]); ok {
|
||||
sets[id.Name] = name
|
||||
out[name] = append(out[name], []string{}...)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
recv, ok := sel.X.(*ast.Ident)
|
||||
if !ok || sets[recv.Name] == "" {
|
||||
return true
|
||||
}
|
||||
arg := 0
|
||||
switch sel.Sel.Name {
|
||||
case "Bool", "String", "Int", "Int64", "Uint", "Uint64", "Float64", "Duration", "Func", "BoolFunc", "TextVar":
|
||||
case "BoolVar", "StringVar", "IntVar", "Int64Var", "UintVar", "Uint64Var", "Float64Var", "DurationVar", "Var":
|
||||
arg = 1
|
||||
default:
|
||||
return true
|
||||
}
|
||||
if arg < len(call.Args) {
|
||||
if flagName, ok := stringLit(call.Args[arg]); ok {
|
||||
out[sets[recv.Name]] = append(out[sets[recv.Name]], flagName)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isSelector(e ast.Expr, pkg, name string) bool {
|
||||
sel, ok := e.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
id, ok := sel.X.(*ast.Ident)
|
||||
return ok && id.Name == pkg && sel.Sel.Name == name
|
||||
}
|
||||
|
||||
func stringLit(e ast.Expr) (string, bool) {
|
||||
lit, ok := e.(*ast.BasicLit)
|
||||
if !ok || lit.Kind != token.STRING {
|
||||
return "", false
|
||||
}
|
||||
s, err := strconv.Unquote(lit.Value)
|
||||
return s, err == nil
|
||||
}
|
||||
|
||||
// accountedFlags are the flags of a verb's command that are not an argument of the same name:
|
||||
// carried by an argument named otherwise ("=argument"), or set by the verb itself, or withheld from
|
||||
// the named verb on purpose — each with why. `command` reaches every flag of the binary regardless.
|
||||
var accountedFlags = map[string]map[string]string{
|
||||
"status": {"json": "set by the verb: the answer is data"},
|
||||
"seats": {"json": "set by the verb: the answer is data"},
|
||||
"queue": {"json": "not set: the verb answers the table a person reads"},
|
||||
"plan": {"json": "set by the verb unless files is asked"},
|
||||
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
|
||||
"build": {
|
||||
"wait": "set by the verb to 0: the id follows the build (issue 176)",
|
||||
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
|
||||
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
|
||||
"behind": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||
"on": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||
},
|
||||
"builds": {"n": "=limit"},
|
||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||
"durations": {
|
||||
"json": "set by the verb: the answer is data",
|
||||
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||
},
|
||||
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||
"conditions": {"json": "set by the verb: the answer is data"},
|
||||
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||
}
|
||||
|
||||
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
||||
// from the source, so a flag added to a command — or a verb added whose command takes flags —
|
||||
// fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And
|
||||
// a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody
|
||||
// reads.
|
||||
func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
|
||||
flags := commandFlags(t)
|
||||
if len(flags["push"]) == 0 || len(flags["plan"]) == 0 {
|
||||
t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags)
|
||||
}
|
||||
reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
|
||||
served := servedSchemas(t)
|
||||
for name, v := range served {
|
||||
if inProcess[name] || name == "command" {
|
||||
continue
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
for _, subset := range subsetsOf(names) {
|
||||
argv, err := argvFor(name, argumentsFor(subset, switches))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// The flag set is named by the longest run of leading words that names one.
|
||||
var words []string
|
||||
for _, w := range argv {
|
||||
if strings.HasPrefix(w, "-") {
|
||||
break
|
||||
}
|
||||
words = append(words, w)
|
||||
}
|
||||
for n := len(words); n > 0; n-- {
|
||||
if _, has := flags[strings.Join(words[:n], " ")]; has {
|
||||
if reached[name] == nil {
|
||||
reached[name] = map[string]bool{}
|
||||
}
|
||||
reached[name][strings.Join(words[:n], " ")] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
used := map[string]map[string]bool{}
|
||||
verbs := make([]string, 0, len(reached))
|
||||
for verb := range reached {
|
||||
verbs = append(verbs, verb)
|
||||
}
|
||||
sort.Strings(verbs)
|
||||
for _, verb := range verbs {
|
||||
declared, _ := declaredArguments(served[verb])
|
||||
for set := range reached[verb] {
|
||||
if used[set] == nil {
|
||||
used[set] = map[string]bool{}
|
||||
}
|
||||
for _, flagName := range flags[set] {
|
||||
why, accounted := accountedFlags[set][flagName]
|
||||
switch {
|
||||
case accounted && strings.HasPrefix(why, "="):
|
||||
used[set][flagName] = true
|
||||
if !contains(declared, strings.TrimPrefix(why, "=")) {
|
||||
t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare",
|
||||
verb, flagName, set, strings.TrimPrefix(why, "="))
|
||||
}
|
||||
case accounted:
|
||||
used[set][flagName] = true
|
||||
case contains(declared, flagName):
|
||||
default:
|
||||
t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+
|
||||
"or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for set, fs := range accountedFlags {
|
||||
for flagName := range fs {
|
||||
if !used[set][flagName] {
|
||||
t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every verb's required arguments are properties of its schema: a schema that requires what it
|
||||
// does not describe is the uncallable verb of issue 244 from the other side.
|
||||
func TestEveryRequiredArgumentIsDescribed(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
names, _ := declaredArguments(v)
|
||||
for k := range sampleArguments(v) {
|
||||
if !contains(names, k) {
|
||||
t.Errorf("%s requires %q and does not describe it", name, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,29 +10,6 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
|
||||
// schema names and no other (novox/hq ADR 0154, ADR 0035).
|
||||
func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == "tools" {
|
||||
continue
|
||||
}
|
||||
args := map[string]any{}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
for name := range props {
|
||||
args[name] = "x"
|
||||
}
|
||||
argv, err := argvFor(v.Name, args)
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", v.Name, err)
|
||||
continue
|
||||
}
|
||||
if argv[0] == "" {
|
||||
t.Errorf("%s: empty command", v.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `builds` given a build's id reads that build's log from the bus rather than listing builds
|
||||
// (novox/hq ADR 0157).
|
||||
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
|
||||
@@ -66,13 +43,22 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
||||
t.Fatalf("a pair credential: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "module": "letta"})
|
||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace --module letta" {
|
||||
t.Fatalf("one consuming module's pair credential: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
||||
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
||||
t.Fatalf("an own secret: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
|
||||
if strings.Join(argv, " ") != "rotate" {
|
||||
t.Fatalf("half an own secret falls to the command's usage: %v", argv)
|
||||
if _, err := argvFor("rotate", map[string]any{"node": "ace"}); err == nil || !strings.Contains(err.Error(), `"module"`) {
|
||||
t.Fatalf("half an own secret is refused, naming what it lacks: %v", err)
|
||||
}
|
||||
if argv, _ := argvFor("rotate", nil); strings.Join(argv, " ") != "rotate" {
|
||||
t.Fatalf("neither shape falls to the command's usage: %v", argv)
|
||||
}
|
||||
if _, err := argvFor("rotate", map[string]any{"provision": "p", "node": "ace", "module": "m", "secret": "s"}); err == nil {
|
||||
t.Fatal("both shapes at once were taken, and one of them passed over")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,8 +107,8 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0" {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
|
||||
t.Fatalf("push waits: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||
|
||||
@@ -93,18 +93,31 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
|
||||
return nil
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, node, module, name, value); err != nil {
|
||||
untilStart, err := open.inventory.AcceptGivenSecret(ctx, node, module, name, value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
|
||||
// machine and the mesh cannot read it again.
|
||||
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
|
||||
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
|
||||
if untilStart {
|
||||
// Accepted, and said what it is for (novox/hq ADR 0228): a value given by hand adopts
|
||||
// something that already holds it, and lives only until the module has started on it.
|
||||
fmt.Printf(" it lives until %s next starts well under the mesh on it, and is then replaced with a value\n"+
|
||||
" the mesh makes, sealed and sent (ADR 0228): a value given by hand is for adopting something\n"+
|
||||
" already running that holds it\n", module)
|
||||
if record, err := open.inventory.NodeByName(ctx, node); err == nil && !record.Adopted {
|
||||
fmt.Printf(" %s is not an adopted machine: if %s is installed fresh there, it needs no given value —\n"+
|
||||
" the mesh makes one at the first push\n", node, module)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
|
||||
}
|
||||
fmt.Printf(" run `push %s` to send it\n", node)
|
||||
return nil
|
||||
}
|
||||
|
||||
const secretUsage = "secret rotate <node> <module> <name>\n" +
|
||||
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
||||
"secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
|
||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||
"secret export [--out <file>]"
|
||||
@@ -367,9 +380,20 @@ func valueFor(node, module, name, from string) (string, error) {
|
||||
// secretRotate makes a module's own secret anew and sends the machine, so the module starts again on
|
||||
// the new value (novox/hq ADR 0114, issue 180). A pair credential rotates with `rotate <provision>`;
|
||||
// this is the secret with one party. Said in the log with who asked and when, never the value.
|
||||
//
|
||||
// **A value given to the mesh rotates the same way** (novox/hq ADR 0228): what a module reads at start
|
||||
// is held by nobody else, so the old value is not needed to replace it. Refused for a value an
|
||||
// outside party issued, which no value of the mesh's would replace. Why it was rotated is recorded in
|
||||
// the hand-act log when given — a rotation asked by a person is an act by hand, and a leak is a cause
|
||||
// worth counting.
|
||||
func secretRotate(ctx context.Context, args []string) error {
|
||||
rest, _ := split(args)
|
||||
if len(rest) != 3 {
|
||||
rest, flags := split(args)
|
||||
set := flag.NewFlagSet("secret rotate", flag.ContinueOnError)
|
||||
why := addHandActFlags(set)
|
||||
if err := set.Parse(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 3 || set.NArg() != 0 {
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
node, module, name := rest[0], rest[1], rest[2]
|
||||
@@ -378,6 +402,10 @@ func secretRotate(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
origin, given, err := open.inventory.OwnSecretOrigin(ctx, node, module, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := open.inventory.RotateModuleSecret(ctx, node, module, name); err != nil {
|
||||
var refused inventory.ErrNotRotatable
|
||||
if errors.As(err, &refused) {
|
||||
@@ -385,8 +413,13 @@ func secretRotate(ctx context.Context, args []string) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "secret rotate", []string{node, module, name})
|
||||
fmt.Printf("rotated %q of %s on %s at %s, asked by %s; the value is sealed and not shown\n",
|
||||
name, module, node, time.Now().UTC().Format(time.RFC3339), whoAsked())
|
||||
if origin == inventory.OriginAccepted {
|
||||
fmt.Printf(" it replaces the value given to the mesh on %s: the mesh made this one, so `secret rotate` "+
|
||||
"replaces it again whenever asked (ADR 0228)\n", given.UTC().Format("2006-01-02"))
|
||||
}
|
||||
// A shared credential (ADR 0158) has as many holders as the provision has consumers, and all
|
||||
// of them are sent in one act, so no machine is left reading a value the provider no longer takes.
|
||||
machines, err := open.inventory.SharedHolders(ctx, node, module, name)
|
||||
|
||||
@@ -22,6 +22,11 @@ type sendable struct {
|
||||
// under the node's hold just before the body is made (novox/hq 04-ISSUES/107). Zero is not sent
|
||||
// at all, which a host reads as "no order claimed" — the shape of every declaration before this.
|
||||
Sequence int64
|
||||
// Epoch is the controller lease's epoch it was composed under (novox/hq to-be 45 §6): a machine that
|
||||
// heard a later epoch refuses it. Zero is not sent at all — every machine whose node-engine has not
|
||||
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
||||
// (link/order.go, the contract).
|
||||
Epoch uint64
|
||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||
Adoption *adoptionEnvelope
|
||||
@@ -43,6 +48,9 @@ type sendable struct {
|
||||
LeftOut []string
|
||||
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||
leftOutWhy map[string]string
|
||||
// withheld is every consumer this machine's grants leave out, because its identity overflows the
|
||||
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
|
||||
withheld []catalogue.Overflow
|
||||
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||
// Composed only on the send path; nil records that it is not known.
|
||||
@@ -67,6 +75,9 @@ func (s sendable) Body() ([]byte, error) {
|
||||
if s.Sequence > 0 {
|
||||
envelope["sequence"] = s.Sequence
|
||||
}
|
||||
if s.Epoch > 0 {
|
||||
envelope["epoch"] = s.Epoch
|
||||
}
|
||||
if len(s.LeftOut) > 0 {
|
||||
envelope["left_out"] = s.LeftOut
|
||||
}
|
||||
|
||||
@@ -0,0 +1,585 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The signals table (novox/hq to-be 45 §3, ADR 0227 rule 5), compiled in.
|
||||
//
|
||||
// **Every signal the core expects has a watchdog; its absence is a condition.** A row says what is
|
||||
// expected, from whom, after what, within what bound, and what is raised when it does not come. One
|
||||
// table, read three ways: by the watchdog loop (watchdogs.go), which runs every row's watch over the
|
||||
// facts it gathered; by `doctor signals`, which says the age of every row's newest signal; and by the
|
||||
// test generated from it (signals_test.go), which suppresses each signal in turn and asserts its
|
||||
// condition — so a row added without a watch, or a watch that does not fire, fails the build.
|
||||
//
|
||||
// A row not watched yet says why, and in which phase it will be: a watchdog of a signal nothing emits
|
||||
// would be a condition that can only cry wolf. Bounds marked provisional are set from what Phase 0
|
||||
// measured (`durations`) and corrected in Phase 1's first live week; a corrected bound is a change to
|
||||
// this table, reviewed like code.
|
||||
|
||||
// signalRow is one row of the table.
|
||||
type signalRow struct {
|
||||
Row string
|
||||
Signal string
|
||||
Emitter string
|
||||
Trigger string
|
||||
Bound string
|
||||
Kind string
|
||||
Severity conditions.Severity
|
||||
// Phase is the phase of to-be 45 its watchdog is built in.
|
||||
Phase int
|
||||
// Deferred says why it is not watched yet; empty for a row that is.
|
||||
Deferred string
|
||||
// needs is the part of the facts the row reads: an error there is the row blind, which is
|
||||
// itself said (probe-failed) rather than read as nothing wrong.
|
||||
needs func(f *signalFacts) error
|
||||
// watch is what is wrong now, from the facts.
|
||||
watch func(f *signalFacts) []conditions.Observation
|
||||
// newest is the time of the newest signal of this row, for `doctor signals`; zero when none.
|
||||
newest func(f *signalFacts) time.Time
|
||||
}
|
||||
|
||||
// The bounds, provisional where to-be 45 says so.
|
||||
const (
|
||||
// heartbeatEvery is the interval a node-engine or node tools that say none have always used.
|
||||
heartbeatEvery = 60 * time.Second
|
||||
// heartbeatsMissed is how many intervals may pass in silence (S1, S11).
|
||||
heartbeatsMissed = 3
|
||||
// controlNodeUrgentAfter is how long the control node may be silent before it is urgent (S1).
|
||||
controlNodeUrgentAfter = 30 * time.Minute
|
||||
// reportAtLeast is the least a machine is given to report a send (S2).
|
||||
reportAtLeast = 2 * time.Minute
|
||||
// tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at.
|
||||
tierAtLeast = planWaitBound
|
||||
// loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4).
|
||||
loopDeafAfter = 2 * time.Minute
|
||||
// askAtLeast is the least a build ask is given, and askDefault the bound while nothing is
|
||||
// measured (S6): the build seat declares no timeout of its own.
|
||||
askAtLeast = 20 * time.Minute
|
||||
askDefault = time.Hour
|
||||
// callDefault is the bound of a verb that declares none (S7); push's and build's are longer.
|
||||
callDefault = 10 * time.Minute
|
||||
// advisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9).
|
||||
advisoryQuiet = time.Hour
|
||||
// staleRefusalsAllowed in staleRefusalsWithin are what S13 lets pass from one writer.
|
||||
staleRefusalsAllowed = 5
|
||||
staleRefusalsWithin = 5 * time.Minute
|
||||
// leaseBound is how long the lease may go unrenewed (S12): the key's age.
|
||||
leaseBound = broker.LeaseTTL
|
||||
)
|
||||
|
||||
// callBounds are the verbs that may run longer than callDefault, and how long (S7).
|
||||
var callBounds = map[string]time.Duration{
|
||||
"push": 30 * time.Minute, "rotate": 30 * time.Minute, "assign": 15 * time.Minute,
|
||||
"unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute,
|
||||
}
|
||||
|
||||
// callBound is a verb's bound.
|
||||
func callBound(verb string) time.Duration {
|
||||
if b, ok := callBounds[verb]; ok {
|
||||
return b
|
||||
}
|
||||
return callDefault
|
||||
}
|
||||
|
||||
// signalsTable is the table, in to-be 45's order.
|
||||
var signalsTable = []signalRow{
|
||||
{Row: "S1", Signal: "machine heartbeat", Emitter: "node-engine", Trigger: "its interval",
|
||||
Bound: "3 × the interval it says (60 s when it says none), provisional; not raised while the machine " +
|
||||
"said it is asleep or shutting down (ADR 0211); urgent after 30 min for a control node",
|
||||
Kind: "silent", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchHeartbeats,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard })
|
||||
}},
|
||||
{Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent",
|
||||
Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep",
|
||||
Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt })
|
||||
}},
|
||||
{Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered",
|
||||
Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional; not while the " +
|
||||
"build seat is paused under it",
|
||||
Kind: "stalled", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.plans, func(p planFacts) time.Time { return p.entered })
|
||||
}},
|
||||
{Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller",
|
||||
Trigger: "while its consumers have pending messages", Bound: "2 min",
|
||||
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
|
||||
newest: func(f *signalFacts) time.Time { return f.loop.took }},
|
||||
{Row: "S5", Signal: "a merge announced becomes a plan, or nothing reads it", Emitter: "announcer → controller",
|
||||
Trigger: "each merge", Bound: "10 min (the catch-up pass of issue 266)",
|
||||
Kind: "merge-not-acted", Severity: conditions.Urgent, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.mergesErr }, watch: watchMerges,
|
||||
newest: func(f *signalFacts) time.Time { return f.mergesPassed }},
|
||||
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
|
||||
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
|
||||
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
|
||||
Kind: "ask-lost", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.asksErr }, watch: watchAsks,
|
||||
newest: func(f *signalFacts) time.Time { return newestOf(f.asks, func(a askFacts) time.Time { return a.since }) }},
|
||||
{Row: "S7", Signal: "a call running → finished", Emitter: "controller", Trigger: "each call",
|
||||
Bound: "the verb's bound: push, rotate and command 30 min, assign and unassign 15 min, doctor 3 min, " +
|
||||
"any other 10 min",
|
||||
Kind: "call-hung", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(*signalFacts) error { return nil }, watch: watchCalls,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.calls, func(c link.Call) time.Time { return c.Started })
|
||||
}},
|
||||
{Row: "S8", Signal: "a provider's failing word repeated", Emitter: "provider",
|
||||
Trigger: "every 15 min while failing (ADR 0224)", Bound: "30 min",
|
||||
Kind: kindProviderSilent, Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.standingsErr }, watch: watchProviders,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
||||
}},
|
||||
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
||||
Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
|
||||
"once it exists again or the mesh no longer expects it",
|
||||
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
||||
}},
|
||||
{Row: "S10", Signal: "the self-check's heartbeat", Emitter: "controller's doctor", Trigger: "every run",
|
||||
Bound: "2 × its interval; watched from a second machine by mesh-watcher, and here as well",
|
||||
Kind: "self-check-silent", Severity: conditions.Urgent, Phase: 1,
|
||||
needs: func(*signalFacts) error { return nil }, watch: watchSelfCheck,
|
||||
newest: func(f *signalFacts) time.Time { return f.selfCheck.last }},
|
||||
{Row: "S11", Signal: "node tools heartbeat", Emitter: "node tools", Trigger: "its interval",
|
||||
Bound: "3 × the interval it says (60 s when it says none), provisional; only where node-tools is assigned",
|
||||
Kind: "tools-silent", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchTools,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
|
||||
}},
|
||||
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
|
||||
Bound: "15 s (the key's age); a holder that lost the lease, or stopped renewing and was taken over, and a " +
|
||||
"lease bucket found raised again from nothing, are said for an hour after; a controller serving without " +
|
||||
"the lease, for as long as it does",
|
||||
Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
|
||||
needs: func(f *signalFacts) error { return f.leaseErr }, watch: watchLease,
|
||||
newest: func(f *signalFacts) time.Time { return f.lease.renewed }},
|
||||
{Row: "S13", Signal: "stale refusals", Emitter: "every receiver (rule 2)", Trigger: "each refusal",
|
||||
Bound: "more than 5 from one writer in 5 min: a controller epoch, a controller that claimed none, or a " +
|
||||
"machine's node-engine whose accounts the controller refused",
|
||||
Kind: "stale-writer", Severity: conditions.Warning, Phase: 2,
|
||||
needs: func(*signalFacts) error { return nil }, watch: watchStaleRefusals,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.staleRefusals, func(w link.WriterRefusals) time.Time { return w.Last })
|
||||
}},
|
||||
{Row: "S14", Signal: "facts snapshot exported", Emitter: "controller", Trigger: "daily",
|
||||
Bound: "2 days", Kind: "facts-stale", Severity: conditions.Warning, Phase: 5,
|
||||
Deferred: "the facts snapshot is built in Phase 5 (to-be 45 §9): nothing exports one yet"},
|
||||
{Row: "S15", Signal: "a hand act with a cause already recorded", Emitter: "hand-act log",
|
||||
Trigger: "each act", Bound: "the second within 14 days", Kind: "healer-wanted",
|
||||
Severity: conditions.Warning, Phase: 3,
|
||||
Deferred: "Phase 3 (to-be 45 §10): `hand-acts` lists repeated causes today; the condition comes with the healers"},
|
||||
}
|
||||
|
||||
// newestOf is the newest time among things.
|
||||
func newestOf[T any](list []T, at func(T) time.Time) time.Time {
|
||||
var newest time.Time
|
||||
for _, x := range list {
|
||||
if t := at(x); t.After(newest) {
|
||||
newest = t
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
// ago is a duration as the summaries say it.
|
||||
func ago(d time.Duration) string {
|
||||
if d < time.Minute {
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
return d.Round(time.Minute).String()
|
||||
}
|
||||
|
||||
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
||||
func heartbeatBound(every time.Duration) time.Duration {
|
||||
if every <= 0 {
|
||||
every = heartbeatEvery
|
||||
}
|
||||
return heartbeatsMissed * every
|
||||
}
|
||||
|
||||
func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.machines {
|
||||
if m.lastHeard.IsZero() || m.asleep() {
|
||||
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
||||
continue
|
||||
}
|
||||
bound := heartbeatBound(m.every)
|
||||
silent := f.now.Sub(m.lastHeard)
|
||||
if silent <= bound {
|
||||
continue
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if m.control && silent > controlNodeUrgentAfter {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "silent",
|
||||
Machine: m.name, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", m.name,
|
||||
m.lastHeard.UTC().Format("2006-01-02 15:04 MST"), bound),
|
||||
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchReports(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.machines {
|
||||
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||
continue
|
||||
}
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
continue // silent: S1 says it, and a silent machine reports nothing
|
||||
}
|
||||
bound := max(reportAtLeast, 3*m.lastApply)
|
||||
waited := f.now.Sub(m.sentAt)
|
||||
if waited <= bound {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name,
|
||||
Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)",
|
||||
m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||
Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchPlans(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, p := range f.plans {
|
||||
if p.paused {
|
||||
continue
|
||||
}
|
||||
in := f.now.Sub(p.entered)
|
||||
if in <= p.bound {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
|
||||
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
|
||||
ago(p.bound), p.waiting),
|
||||
Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchLoop(f *signalFacts) []conditions.Observation {
|
||||
if f.loop.pending == 0 {
|
||||
return nil
|
||||
}
|
||||
since := f.loop.took
|
||||
if since.IsZero() {
|
||||
since = f.started
|
||||
}
|
||||
if f.now.Sub(since) <= loopDeafAfter {
|
||||
return nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "controller", Kind: "controller-deaf",
|
||||
Token: "deaf", Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller's event loop has taken nothing for %s while its consumers hold %d "+
|
||||
"message(s): reports, builds and merges are not being acted on", ago(f.now.Sub(since)), f.loop.pending),
|
||||
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
|
||||
}
|
||||
|
||||
func watchMerges(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.merges {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, ID: m.Repo + "." + short(m.Commit),
|
||||
Kind: "merge-not-acted", Token: "not-acted", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s/%s merged into %s (%s) was never handed to the controller by the bus; "+
|
||||
"acted on late by the catch-up", m.Owner, m.Repo, m.Base, short(m.Commit)),
|
||||
Said: fmt.Sprintf("announced %s, %s behind it", m.At.UTC().Format(time.RFC3339), readableList(m.Modules))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchAsks(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, a := range f.asks {
|
||||
var said string
|
||||
switch {
|
||||
case a.state == link.AskDead:
|
||||
said = fmt.Sprintf("the %s queue handed it out as often as it may and it was never settled", a.seat)
|
||||
case a.state == link.AskInFlight && f.now.Sub(a.since) > a.bound:
|
||||
said = fmt.Sprintf("in flight on %s for %s (bound %s)", orSomewhere(a.on), ago(f.now.Sub(a.since)), ago(a.bound))
|
||||
default:
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBuild, ID: a.id, Kind: "ask-lost",
|
||||
Token: "lost", Machine: a.on, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the build %s of %s has no outcome: %s", a.id, a.what, said), Said: said})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orSomewhere(node string) string {
|
||||
if node == "" {
|
||||
return "a machine that did not say which"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func watchCalls(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, c := range f.calls {
|
||||
bound := callBound(c.Verb)
|
||||
running := f.now.Sub(c.Started)
|
||||
if running <= bound {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCall, ID: c.ID, Kind: "call-hung",
|
||||
Token: "hung", Machine: f.host, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s.%s (call %s) has been running since %s, past its bound of %s", c.Seat, c.Verb,
|
||||
c.ID, c.Started.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||
Said: fmt.Sprintf("running %s, asked by %s", ago(running), orSomebody(c.Caller))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orSomebody(caller string) string {
|
||||
if caller == "" {
|
||||
return "a caller the bus did not name"
|
||||
}
|
||||
return caller
|
||||
}
|
||||
|
||||
func watchProviders(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, c := range f.standings {
|
||||
quiet := f.now.Sub(c.LastObserved)
|
||||
if quiet <= providerSaysAgainWithin {
|
||||
continue
|
||||
}
|
||||
module, node, consumer, ok := providerOf(c)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, ID: module + "." + node + "." + consumer,
|
||||
Token: "silent", Kind: kindProviderSilent, Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped "+
|
||||
"saying anything, so its last word is all the mesh has", module, node, consumer,
|
||||
c.LastObserved.UTC().Format("2006-01-02 15:04 MST")),
|
||||
Said: fmt.Sprintf("not said again for %s", ago(quiet))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, a := range f.advisories {
|
||||
if f.now.Sub(a.Last) > advisoryQuiet {
|
||||
continue
|
||||
}
|
||||
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||
}
|
||||
severity := conditions.Warning
|
||||
times := ""
|
||||
if a.Count > 1 {
|
||||
times = fmt.Sprintf(" (%d times since %s)", a.Count, a.First.UTC().Format("15:04 MST"))
|
||||
}
|
||||
machine := ""
|
||||
if a.ID == "controller" {
|
||||
machine = f.host
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
|
||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchSelfCheck(f *signalFacts) []conditions.Observation {
|
||||
every := f.selfCheck.every
|
||||
if every <= 0 {
|
||||
every = doctorEvery
|
||||
}
|
||||
since := f.selfCheck.last
|
||||
if since.IsZero() {
|
||||
since = f.started
|
||||
}
|
||||
if f.now.Sub(since) <= 2*every {
|
||||
return nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "doctor", Kind: "self-check-silent",
|
||||
Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's "+
|
||||
"invariants are not being checked", since.UTC().Format("2006-01-02 15:04 MST"), every),
|
||||
Said: fmt.Sprintf("no run for %s", ago(f.now.Sub(since)))}}
|
||||
}
|
||||
|
||||
func watchTools(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.machines {
|
||||
if !m.tools || m.asleep() {
|
||||
continue
|
||||
}
|
||||
bound := heartbeatBound(m.toolsEvery)
|
||||
since := m.toolsHeard
|
||||
if since.IsZero() {
|
||||
// Not heard since this controller started: silent since then at the most.
|
||||
since = f.toolsHeardFrom
|
||||
}
|
||||
silent := f.now.Sub(since)
|
||||
if silent <= bound {
|
||||
continue
|
||||
}
|
||||
heard := "not since this controller started at " + since.UTC().Format("2006-01-02 15:04 MST")
|
||||
if !m.toolsHeard.IsZero() {
|
||||
heard = "since " + m.toolsHeard.UTC().Format("2006-01-02 15:04 MST")
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "tools-silent",
|
||||
Machine: m.name, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the node tools on %s have not said they are there %s (bound %s): nothing can "+
|
||||
"ask that machine anything", m.name, heard, bound),
|
||||
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchStaleRefusals(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, w := range f.staleRefusals {
|
||||
if w.Count <= staleRefusalsAllowed {
|
||||
continue
|
||||
}
|
||||
scope, id, machine := conditions.ScopeCore, "controller.unnamed", ""
|
||||
named := w.Writer
|
||||
switch {
|
||||
case w.Epoch > 0:
|
||||
id = fmt.Sprintf("controller.epoch-%d", w.Epoch)
|
||||
if e, ok := f.epochs[w.Epoch]; ok {
|
||||
named = fmt.Sprintf("the controller of epoch %d (%s%s)", w.Epoch, e.Instance, endedWords(e))
|
||||
}
|
||||
case w.Writer == link.WriterNodeEngine(firstOf(w.Receivers)) || strings.HasPrefix(w.Writer, "the node-engine on "):
|
||||
node := strings.TrimPrefix(w.Writer, "the node-engine on ")
|
||||
scope, id, machine = conditions.ScopeMachine, node, node
|
||||
}
|
||||
if machine == "" && len(w.Receivers) > 0 {
|
||||
machine = w.Receivers[0]
|
||||
}
|
||||
var also []string
|
||||
for _, r := range w.Receivers {
|
||||
if r != machine && r != "controller" {
|
||||
also = append(also, r)
|
||||
}
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: scope, ID: id, Kind: "stale-writer", Token: "stale-writer",
|
||||
Machine: machine, Also: also, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s was refused %d time(s) in %s as older than what its receivers hold (%s): a "+
|
||||
"writer is sending what the mesh has moved past", named, w.Count, staleRefusalsWithin,
|
||||
strings.Join(w.Receivers, ", ")),
|
||||
Said: fmt.Sprintf("%d stale refusals in %s, the last at %s", w.Count, staleRefusalsWithin,
|
||||
w.Last.UTC().Format(time.RFC3339))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// firstOf is a list's first, empty for none.
|
||||
func firstOf(list []string) string {
|
||||
if len(list) == 0 {
|
||||
return ""
|
||||
}
|
||||
return list[0]
|
||||
}
|
||||
|
||||
// endedWords is how an epoch ended, for a sentence naming it; nothing while it is held.
|
||||
func endedWords(e inventory.Epoch) string {
|
||||
if e.Ended == nil {
|
||||
return ", still holding the lease"
|
||||
}
|
||||
return fmt.Sprintf(", %s at %s", e.How, e.Ended.UTC().Format("15:04:05 MST"))
|
||||
}
|
||||
|
||||
// watchLease is S12: the lease held and renewed by this controller, and every holder that lost it.
|
||||
func watchLease(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
l := f.lease
|
||||
if l.unleased != "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "unleased",
|
||||
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: "the controller serves WITHOUT the lease: nothing keeps a second controller from acting " +
|
||||
"beside it, and its declarations carry no epoch. A bus whose user list is older than this " +
|
||||
"controller does not grant it the lease's bucket: a push of the machine holding the bus sends " +
|
||||
"the list that does, and the controller takes the lease within five seconds — " + l.unleased,
|
||||
Said: l.unleased})
|
||||
} else if l.held && !l.renewed.IsZero() && f.now.Sub(l.renewed) > leaseBound {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "late",
|
||||
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller has not renewed its lease (epoch %d) since %s, past the key's age "+
|
||||
"of %s: another controller may take it", l.epoch, l.renewed.UTC().Format(time.RFC3339), leaseBound),
|
||||
Said: fmt.Sprintf("not renewed for %s", ago(f.now.Sub(l.renewed)))})
|
||||
}
|
||||
if !l.reset.IsZero() && f.now.Sub(l.reset) <= advisoryQuiet {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "reset",
|
||||
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: "the controller lease's bucket was raised again from nothing: " + l.resetSaid,
|
||||
Said: l.resetSaid})
|
||||
}
|
||||
var lost []string
|
||||
newest := inventory.Epoch{}
|
||||
for _, e := range l.ended {
|
||||
if e.Ended == nil || e.How == inventory.EpochReleased || f.now.Sub(*e.Ended) > advisoryQuiet {
|
||||
continue
|
||||
}
|
||||
lost = append(lost, fmt.Sprintf("epoch %d (%s) %s at %s", e.Epoch, e.Instance, e.How,
|
||||
e.Ended.UTC().Format("15:04:05 MST")))
|
||||
if newest.Ended == nil || e.Ended.After(*newest.Ended) {
|
||||
newest = e
|
||||
}
|
||||
}
|
||||
if len(lost) > 0 {
|
||||
how := "lost it: its renewal was refused or could not be made"
|
||||
if newest.How == inventory.EpochExpired {
|
||||
how = "stopped renewing it without giving it back, and was taken over"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
|
||||
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
|
||||
newest.Instance, how, l.epoch),
|
||||
Said: strings.Join(lost, "; ")})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchedRows are the rows a watchdog runs for.
|
||||
func watchedRows() []signalRow {
|
||||
var out []signalRow
|
||||
for _, r := range signalsTable {
|
||||
if r.watch != nil {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// kindsOf is a row's condition kinds, one or several.
|
||||
func kindsOf(r signalRow) []string {
|
||||
var out []string
|
||||
for _, k := range strings.Split(r.Kind, ",") {
|
||||
out = append(out, strings.TrimSpace(k))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
|
||||
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
|
||||
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
|
||||
// and the condition clears. A row that is not watched says why. A row added to the table without a
|
||||
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
|
||||
|
||||
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
|
||||
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
|
||||
// running, the self-check a minute old.
|
||||
func calm(now time.Time) *signalFacts {
|
||||
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
||||
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
||||
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
||||
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
||||
mergesPassed: now.Add(-time.Minute),
|
||||
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
||||
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
|
||||
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
|
||||
}
|
||||
}
|
||||
|
||||
// refusedBy is n refusals of one writer, the last a moment ago.
|
||||
func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals {
|
||||
return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n,
|
||||
Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}}
|
||||
}
|
||||
|
||||
// suppression is one row's signal held back: inside its bound, and past it.
|
||||
type suppression struct{ inside, past func(f *signalFacts) }
|
||||
|
||||
// suppressions are every watched row's, by row.
|
||||
var suppressions = map[string]suppression{
|
||||
"S1": {
|
||||
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
|
||||
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
|
||||
},
|
||||
"S2": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
|
||||
},
|
||||
},
|
||||
"S3": {
|
||||
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
|
||||
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
|
||||
},
|
||||
"S4": {
|
||||
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
||||
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
||||
},
|
||||
"S5": {
|
||||
inside: func(f *signalFacts) {},
|
||||
past: func(f *signalFacts) {
|
||||
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
||||
},
|
||||
},
|
||||
"S6": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
|
||||
},
|
||||
},
|
||||
"S7": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
|
||||
},
|
||||
},
|
||||
"S8": {
|
||||
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
||||
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
||||
},
|
||||
"S9": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||
},
|
||||
},
|
||||
"S10": {
|
||||
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
|
||||
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
|
||||
},
|
||||
"S11": {
|
||||
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
|
||||
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
|
||||
},
|
||||
"S12": {
|
||||
inside: func(f *signalFacts) { f.lease.renewed = f.now.Add(-15 * time.Second) },
|
||||
past: func(f *signalFacts) { f.lease.renewed = f.now.Add(-16 * time.Second) },
|
||||
},
|
||||
"S13": {
|
||||
inside: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 5) },
|
||||
past: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 6) },
|
||||
},
|
||||
}
|
||||
|
||||
// standingSaid is a provider's failing word last said at a moment.
|
||||
func standingSaid(at time.Time) conditions.Condition {
|
||||
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
|
||||
}
|
||||
|
||||
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||
seen := map[string]bool{}
|
||||
for _, row := range signalsTable {
|
||||
t.Run(row.Row, func(t *testing.T) {
|
||||
if seen[row.Row] {
|
||||
t.Fatalf("%s is in the table twice", row.Row)
|
||||
}
|
||||
seen[row.Row] = true
|
||||
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
|
||||
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
|
||||
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
|
||||
}
|
||||
if row.Deferred != "" {
|
||||
if row.watch != nil || row.Phase <= 1 {
|
||||
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
|
||||
}
|
||||
if _, has := suppressions[row.Row]; has {
|
||||
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
|
||||
}
|
||||
return
|
||||
}
|
||||
if row.watch == nil || row.needs == nil || row.newest == nil {
|
||||
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
|
||||
}
|
||||
s, ok := suppressions[row.Row]
|
||||
if !ok {
|
||||
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
|
||||
}
|
||||
if got := row.watch(calm(now)); len(got) != 0 {
|
||||
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
|
||||
}
|
||||
inside := calm(now)
|
||||
s.inside(inside)
|
||||
if got := row.watch(inside); len(got) != 0 {
|
||||
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
|
||||
}
|
||||
past := calm(now)
|
||||
s.past(past)
|
||||
got := row.watch(past)
|
||||
if len(got) == 0 {
|
||||
t.Fatalf("%s raised nothing past its bound", row.Row)
|
||||
}
|
||||
for _, o := range got {
|
||||
if !slices.Contains(kindsOf(row), o.Kind) {
|
||||
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
|
||||
}
|
||||
if o.Severity != row.Severity {
|
||||
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
|
||||
}
|
||||
if strings.TrimSpace(o.Summary) == "" {
|
||||
t.Errorf("%s raised a condition that says nothing", row.Row)
|
||||
}
|
||||
}
|
||||
|
||||
// Through the store: raised past the bound, cleared when the signal returns.
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
||||
Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
w.see(t.Context(), past)
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != len(got) {
|
||||
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
|
||||
}
|
||||
w.see(t.Context(), calm(now))
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
|
||||
}
|
||||
})
|
||||
}
|
||||
for name := range suppressions {
|
||||
if !seen[name] {
|
||||
t.Errorf("a suppression for %s, which the table does not have", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
|
||||
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
|
||||
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
|
||||
got := watchHeartbeats(f)
|
||||
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
|
||||
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
|
||||
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
|
||||
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
|
||||
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
|
||||
t.Fatalf("a sleeping machine raised %+v", got)
|
||||
}
|
||||
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
|
||||
if got := watchHeartbeats(f); len(got) != 1 {
|
||||
t.Fatalf("a woken machine silent past its bound raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
|
||||
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
|
||||
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
|
||||
now := time.Now()
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
silent := calm(now)
|
||||
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
|
||||
w.see(t.Context(), silent)
|
||||
blind := calm(now)
|
||||
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
|
||||
w.see(t.Context(), blind)
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
|
||||
if !slices.Contains(keys, want) {
|
||||
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
|
||||
}
|
||||
}
|
||||
w.see(t.Context(), calm(now))
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("seeing again left open %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
|
||||
// every machine silent.
|
||||
func TestAControllerStandingBySaysNothing(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
|
||||
w.tick(t.Context())
|
||||
if w.lastTick().IsZero() {
|
||||
t.Fatal("a tick standing by was not counted")
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("%+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **A stale writer is named** (novox/hq to-be 45 §3, S13): by the controller instance that held the epoch
|
||||
// its refused declarations claimed, and how that epoch ended — the question issue 204 could not answer.
|
||||
func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
ended := now.Add(-time.Minute)
|
||||
f.staleRefusals = refusedBy(now, 41, 6)
|
||||
f.epochs[41] = inventory.Epoch{Epoch: 41, Instance: "controller@anchor pid 7 since 2026-10-06T10:00:00Z",
|
||||
Host: "anchor", Ended: &ended, How: inventory.EpochExpired}
|
||||
got := watchStaleRefusals(f)
|
||||
if len(got) != 1 || got[0].Key() != "core.controller.epoch-41.stale-writer" ||
|
||||
!strings.Contains(got[0].Summary, "pid 7") || !strings.Contains(got[0].Summary, "expired") ||
|
||||
got[0].Machine != "anchor" || !slices.Equal(got[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the stale writer is not named: %+v", got)
|
||||
}
|
||||
// An account the controller refused names the machine whose node-engine sent it.
|
||||
f.staleRefusals = []link.WriterRefusals{{Writer: link.WriterNodeEngine("laptop"), Count: 6,
|
||||
Receivers: []string{"controller"}, Last: now}}
|
||||
got = watchStaleRefusals(f)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.stale-writer" || got[0].Machine != "laptop" {
|
||||
t.Fatalf("a node-engine sending older accounts is not named: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
|
||||
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
expired := now.Add(-59 * time.Minute)
|
||||
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
|
||||
Ended: &expired, How: inventory.EpochExpired}}
|
||||
got := watchLease(f)
|
||||
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
|
||||
got[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a holder that stopped renewing was not said: %+v", got)
|
||||
}
|
||||
long := now.Add(-61 * time.Minute)
|
||||
f.lease.ended[0].Ended = &long
|
||||
if got := watchLease(f); len(got) != 0 {
|
||||
t.Fatalf("a loss an hour old is still said: %+v", got)
|
||||
}
|
||||
f.lease.ended[0].How, f.lease.ended[0].Ended = inventory.EpochReleased, &expired
|
||||
if got := watchLease(f); len(got) != 0 {
|
||||
t.Fatalf("a lease given back is said as lost: %+v", got)
|
||||
}
|
||||
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
|
||||
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
|
||||
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
|
||||
}
|
||||
f.lease = leaseFacts{unleased: "the bus refused the key"}
|
||||
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.unleased" {
|
||||
t.Fatalf("serving without the lease was not said: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -134,19 +135,28 @@ func seatBase(world catalogue.World, seatName string) (string, error) {
|
||||
// seatBases is the clone base of every seat a recipe's context may name, for a build request
|
||||
// (novox/hq ADR 0155). A seat nobody holds is left out rather than refused here: the build may not
|
||||
// name it at all, and if it does the builder refuses with the seat's name.
|
||||
//
|
||||
// **What cannot be read is said, not passed over as no seats** (novox/hq to-be 45 Phase 2, the
|
||||
// empty-on-error lint): the build still goes ahead — one that names no seat needs none — and one that
|
||||
// does is refused naming it, but the reason is the store, and that is said here where it is known.
|
||||
func seatBases(ctx context.Context) map[string]string {
|
||||
unread := func(what string, err error) map[string]string {
|
||||
fmt.Fprintf(os.Stderr, "could not read %s, so a build naming a seat's clone base will be told that "+
|
||||
"seat is not held: %v\n", what, err)
|
||||
return nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
return unread("the mesh's store", err)
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
return unread("the catalogue", err)
|
||||
}
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return nil
|
||||
return unread("who holds which seat", err)
|
||||
}
|
||||
bases := map[string]string{}
|
||||
for _, seatName := range []string{gitSeat} {
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224) —
|
||||
// **the condition store's first kind** (to-be 45 §2, ADR 0227).
|
||||
//
|
||||
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
||||
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
||||
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
||||
// provider announces a consumer it has failed for minutes; the controller keeps it until the provider
|
||||
// says it recovered; and `status`, its JSON and `node show` name it, breaking "all well". Unchanged in
|
||||
// what it says and when; kept as a condition, `provider.<module>.<node>.<consumer>.failing`, rather
|
||||
// than a row of its own, so it is said outward like every other fault and silenced like one.
|
||||
|
||||
// Kinds of the provider standing.
|
||||
const (
|
||||
kindProviderFailing = "provider-failing"
|
||||
kindProviderSilent = "provider-silent"
|
||||
// sourceProvisioner is what raised a standing: the provider's own event.
|
||||
sourceProvisioner = "provisioner.failing"
|
||||
)
|
||||
|
||||
// providerSaysAgainWithin is how long a failing word stays current without being said again: twice
|
||||
// the quarter of an hour a provider repeats it at (ADR 0224). Past it, S8.
|
||||
const providerSaysAgainWithin = 30 * time.Minute
|
||||
|
||||
// standings keeps what providers say, as conditions.
|
||||
type standings struct {
|
||||
keeper func() *conditions.Keeper
|
||||
}
|
||||
|
||||
// standingObservation is a provider's failing word as an observation: the provider, its machine and
|
||||
// the consumer name it, so the same consumer failed again is the same condition.
|
||||
func standingObservation(st link.Standing) conditions.Observation {
|
||||
whom := st.Consumer
|
||||
if st.Node != "" {
|
||||
whom += " on " + st.Node
|
||||
}
|
||||
summary := fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode,
|
||||
whom, orUnclassed(st.Class), st.Attempts, st.Since.UTC().Format("2006-01-02 15:04 MST"))
|
||||
said := orUnclassed(st.Class)
|
||||
if e := firstLine(st.Error); e != "" {
|
||||
said += ": " + e
|
||||
}
|
||||
if st.Provider != "" {
|
||||
said += fmt.Sprintf(" (provision %s, %d attempts)", st.Provider, st.Attempts)
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: st.Module + "." + st.ProviderNode + "." + st.Consumer,
|
||||
Token: "failing", Kind: kindProviderFailing, Machine: st.ProviderNode, Also: alsoOn(st.Node, st.ProviderNode),
|
||||
Severity: conditions.Warning, Summary: summary, Said: said, Source: sourceProvisioner}
|
||||
}
|
||||
|
||||
// Stood keeps a provider's newest word: failing raises or observes its condition, recovered clears
|
||||
// it. An error is the store away, and the link holds the message to be asked again — a recovery is
|
||||
// said once, and dropping it would leave a consumer named failing that is fine.
|
||||
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
||||
k := s.keeper()
|
||||
if k == nil {
|
||||
return false, fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
|
||||
}
|
||||
o := standingObservation(st)
|
||||
if !st.Failing {
|
||||
why := "the provider says it recovered"
|
||||
if st.Why != "" {
|
||||
why += ": " + st.Why
|
||||
}
|
||||
cleared, err := k.Clear(ctx, o.Key(), why)
|
||||
return cleared, storeAway(err)
|
||||
}
|
||||
_, err := k.Observe(ctx, o)
|
||||
return false, storeAway(err)
|
||||
}
|
||||
|
||||
// storeAway reads the condition store failing as the bus being away for the moment: the link holds the
|
||||
// message and asks again, as it does for a store restarting (ADR 0083), rather than taking it unkept.
|
||||
func storeAway(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%v: %w", err, link.ErrTryAgain)
|
||||
}
|
||||
|
||||
// providerStandings is every open provider-failing condition, from what is open.
|
||||
func providerStandings(open []conditions.Condition) []conditions.Condition {
|
||||
var out []conditions.Condition
|
||||
for _, c := range open {
|
||||
if c.Kind == kindProviderFailing {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// providerOf reads a standing's provider module and machine back from its key.
|
||||
func providerOf(c conditions.Condition) (module, node, consumer string, ok bool) {
|
||||
parts := strings.Split(c.Key, ".")
|
||||
if len(parts) != 5 || parts[0] != conditions.ScopeProvider {
|
||||
return "", "", "", false
|
||||
}
|
||||
return parts[1], parts[2], parts[3], true
|
||||
}
|
||||
|
||||
// unassignedProviders clears the standing of every provider no longer assigned where it ran.
|
||||
//
|
||||
// **A provider no longer assigned is not asked about** (ADR 0224 §4): nothing runs there to fail
|
||||
// anybody, and nothing there will ever say it recovered. The observation that resolves it is the
|
||||
// assignment. Assigned again, its first failure raises it again.
|
||||
func unassignedProviders(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper,
|
||||
open []conditions.Condition) error {
|
||||
assigned := map[string]map[string]bool{}
|
||||
for _, c := range providerStandings(open) {
|
||||
module, node, _, ok := providerOf(c)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
on, asked := assigned[node]
|
||||
if !asked {
|
||||
modules, err := inv.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
if errors.Is(err, inventory.ErrNoSuchNode) {
|
||||
modules = nil // a machine the mesh no longer knows runs nothing
|
||||
} else {
|
||||
return fmt.Errorf("what %s is assigned cannot be read: %w", node, err)
|
||||
}
|
||||
}
|
||||
on = map[string]bool{}
|
||||
for _, m := range modules {
|
||||
on[m] = true
|
||||
}
|
||||
assigned[node] = on
|
||||
}
|
||||
if !on[module] {
|
||||
if _, err := k.Clear(ctx, c.Key, module+" is no longer assigned to "+node+
|
||||
": nothing runs there to fail anybody"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func orUnclassed(class string) string {
|
||||
if class == "" {
|
||||
return "failing"
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// alsoOn is a consumer's machine, when it is not the provider's.
|
||||
func alsoOn(consumerNode, providerNode string) []string {
|
||||
if consumerNode == "" || consumerNode == providerNode {
|
||||
return nil
|
||||
}
|
||||
return []string{consumerNode}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224), kept as
|
||||
// the condition store's first kind (to-be 45 §2). On 2026-10-05 the identity provider refused every
|
||||
// consumer for a day and status called the mesh well (04-ISSUES/179): this is that day, told to the
|
||||
// controller the way the provider now tells it.
|
||||
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
|
||||
Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
|
||||
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||
since := time.Now().Add(-23 * time.Hour)
|
||||
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
||||
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
||||
Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
|
||||
if _, err := kept.Stood(ctx, failing); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing",
|
||||
"idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("status does not say %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
if strings.Contains(said, "all doing what they were told") {
|
||||
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
||||
}
|
||||
if !strings.HasPrefix(said, "1 open condition(s)") {
|
||||
t.Fatalf("status does not lead with what is open:\n%s", said)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Conditions []conditions.Condition `json:"conditions"`
|
||||
Failing []conditions.Condition `json:"failing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 ||
|
||||
!strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||
}
|
||||
}
|
||||
|
||||
// Recovered: gone, and the mesh may be well again as far as this is concerned.
|
||||
failing.Failing = false
|
||||
if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
|
||||
t.Fatalf("%v %v", cleared, err)
|
||||
}
|
||||
asked, err = theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.conditions) != 0 {
|
||||
t.Fatalf("a recovered consumer is still named: %+v", asked.conditions)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
||||
// cleared on the next look, said as resolved by the assignment.
|
||||
func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||
if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := conditionsFrom.Open(ctx)
|
||||
if err != nil || len(all) != 1 {
|
||||
t.Fatalf("%+v %v", all, err)
|
||||
}
|
||||
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if all, _ := conditionsFrom.Open(ctx); len(all) != 0 {
|
||||
t.Fatalf("%+v", all)
|
||||
}
|
||||
}
|
||||
|
||||
// **The store away holds a recovery** (ADR 0224 §3): a standing that cannot be kept is asked again.
|
||||
func TestAStandingTheStoreCannotKeepIsAskedAgain(t *testing.T) {
|
||||
kept := standings{keeper: func() *conditions.Keeper { return nil }}
|
||||
if _, err := kept.Stood(t.Context(), link.Standing{Module: "idp", ProviderNode: "anchor", Consumer: "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), link.ErrTryAgain.Error()) {
|
||||
t.Fatalf("answered %v", err)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
@@ -81,8 +82,12 @@ func printStatus(asked answers) error {
|
||||
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||
behind, sources := asked.behind, asked.sources
|
||||
|
||||
// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
|
||||
// first, silenced ones with when their silence ends.
|
||||
printConditions(asked.conditions, asked.conditionsUnread, time.Now())
|
||||
|
||||
if len(asked.refused) > 0 {
|
||||
// First, above everything else. A machine that cannot be worked out is not running an old
|
||||
// First of what follows. A machine that cannot be worked out is not running an old
|
||||
// declaration — it has no declaration, and nothing below this line is about it.
|
||||
var names []string
|
||||
for name := range asked.refused {
|
||||
@@ -298,6 +303,29 @@ func printStatus(asked answers) error {
|
||||
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||
}
|
||||
|
||||
if len(asked.overflowing) > 0 {
|
||||
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
|
||||
// out of its provider's grants, so the module holds a login nothing created; the provider's
|
||||
// machine is sent everything else rather than refused for one consumer elsewhere.
|
||||
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
|
||||
len(asked.overflowing))
|
||||
for _, o := range asked.overflowing {
|
||||
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
||||
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
||||
}
|
||||
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
||||
}
|
||||
|
||||
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
||||
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
|
||||
switch {
|
||||
case asked.handActsUnread != "":
|
||||
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
|
||||
asked.handActsUnread)
|
||||
case asked.handActs != nil && *asked.handActs > 0:
|
||||
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
|
||||
}
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||
@@ -307,8 +335,9 @@ func printStatus(asked answers) error {
|
||||
|
||||
if asked.well() {
|
||||
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||
// and getting here means every question was asked and answered.
|
||||
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
|
||||
// and getting here means every question was asked and answered. **No open conditions first**
|
||||
// (novox/hq to-be 45 §2): it is what the sentence means now, silenced ones included.
|
||||
fmt.Printf("no open conditions; %d machine(s), all doing what they were told, all heard from, running what "+
|
||||
"the mesh would send them, and every module current with its source\n", len(nodes))
|
||||
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
||||
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
||||
@@ -406,15 +435,27 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||
// the private network is built from and should say nothing else; a machine that does not
|
||||
// resolve is already in refused, and is passed over here.
|
||||
plans := map[string]planned{}
|
||||
for _, n := range out.nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
continue
|
||||
}
|
||||
return answers{}, err
|
||||
}
|
||||
plans[n.Name] = planned{plan, settings}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||
// resolution, as the provider's composition judges it.
|
||||
out.overflowing = append(out.overflowing, plan.Overflowing()...)
|
||||
}
|
||||
// And every open condition (novox/hq to-be 45 §2): what the watchdogs, the self-check and the
|
||||
// providers' own words say is wrong — a provider failing a consumer among them (ADR 0224). Kept on
|
||||
// the bus; a process that cannot read them says so, and the mesh is then not called well.
|
||||
if out.conditions, err = openConditions(ctx); err != nil {
|
||||
out.conditionsUnread = err.Error()
|
||||
}
|
||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||
if err != nil {
|
||||
@@ -422,6 +463,16 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
}
|
||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||
// to read the log from; a process with none has no log to count.
|
||||
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||
n, unread := handActsThisWeek(ctx)
|
||||
if unread != "" {
|
||||
out.handActsUnread = unread
|
||||
} else {
|
||||
out.handActs = &n
|
||||
}
|
||||
}
|
||||
|
||||
// And which machines are not running what the mesh would send them. The same question as a
|
||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||
@@ -433,7 +484,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||
// reason is kept and reported as data; every question that does not depend on it is still
|
||||
// answered.
|
||||
would, err := wouldSend(ctx, open, out.nodes)
|
||||
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
|
||||
if err != nil {
|
||||
out.network = err.Error()
|
||||
would = map[string]string{}
|
||||
@@ -528,7 +579,8 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||
len(a.conditions) == 0 && a.conditionsUnread == ""
|
||||
}
|
||||
|
||||
// hostSplit is which machines report which host version, for every version more than one machine
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
|
||||
// §4's D9 and §8's health of the controller).
|
||||
//
|
||||
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
|
||||
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
|
||||
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
|
||||
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
|
||||
// controller composes it in the background — at its start, after anything that changes what it says
|
||||
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
|
||||
// the last composition at once, saying when it was composed and how long that took. A caller who
|
||||
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
|
||||
|
||||
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
|
||||
// long a nudge waits for the next, so a push answered by four machines is composed once.
|
||||
var (
|
||||
statusEvery = time.Minute
|
||||
statusSettle = 2 * time.Second
|
||||
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
|
||||
// good; the attempt is said as failed, and the last summary stands with its age.
|
||||
statusComposeWithin = 2 * time.Minute
|
||||
)
|
||||
|
||||
// statusSummary is the last composed `status --json`, and when.
|
||||
type statusSummary struct {
|
||||
compose func(context.Context) ([]byte, error)
|
||||
// every, settle and within are the clocks above, read once when it is made.
|
||||
every, settle, within time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
body []byte
|
||||
composedAt time.Time
|
||||
took time.Duration
|
||||
failed string
|
||||
failedAt time.Time
|
||||
started time.Time
|
||||
first chan struct{} // closed when the first attempt ends, either way
|
||||
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
|
||||
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
|
||||
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
|
||||
}
|
||||
|
||||
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
|
||||
// composed when asked, as at a shell.
|
||||
var statusFrom *statusSummary
|
||||
|
||||
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
|
||||
func (s *statusSummary) nudge() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.nudged <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
|
||||
func (s *statusSummary) keep(ctx context.Context) {
|
||||
once := sync.Once{}
|
||||
for {
|
||||
s.composeOnce(ctx)
|
||||
once.Do(func() { close(s.first) })
|
||||
timer := time.NewTimer(s.every)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
case <-s.nudged:
|
||||
timer.Stop()
|
||||
// Let what else is arriving arrive, then compose once for all of it.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(s.settle):
|
||||
}
|
||||
select {
|
||||
case <-s.nudged:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statusSummary) composeOnce(ctx context.Context) {
|
||||
start := time.Now()
|
||||
asking, cancel := context.WithTimeout(ctx, s.within)
|
||||
body, err := s.compose(asking)
|
||||
cancel()
|
||||
took := time.Since(start)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err != nil {
|
||||
s.failed, s.failedAt = err.Error(), time.Now()
|
||||
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
|
||||
"with its age\n", took.Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
|
||||
}
|
||||
|
||||
// answer is what the `status` verb answers: the last summary at once, the same document `status
|
||||
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
|
||||
// but never past the caller's window; a controller that has none says so and why, rather than
|
||||
// answering an empty mesh as a well one.
|
||||
func (s *statusSummary) answer(ctx context.Context) (any, error) {
|
||||
wait := time.NewTimer(link.AnswerWithin - time.Second)
|
||||
defer wait.Stop()
|
||||
select {
|
||||
case <-s.first:
|
||||
case <-wait.C:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.body == nil {
|
||||
why := "its first composition has not finished"
|
||||
if s.failed != "" {
|
||||
why = "it could not be composed: " + s.failed
|
||||
}
|
||||
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
|
||||
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
|
||||
}
|
||||
var parsed any
|
||||
_ = json.Unmarshal(s.body, &parsed)
|
||||
out := map[string]any{
|
||||
"output": string(s.body), "ok": true, "answer": parsed,
|
||||
"composed": s.composedAt.UTC().Format(time.RFC3339),
|
||||
"age": time.Since(s.composedAt).Round(time.Second).String(),
|
||||
"composedIn": s.took.Round(time.Millisecond).String(),
|
||||
"note": "composed by the serving controller at its start, after each report, build or act, and " +
|
||||
"every minute; answered at once from the last composition",
|
||||
}
|
||||
if s.failed != "" && s.failedAt.After(s.composedAt) {
|
||||
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
|
||||
s.failedAt.UTC().Format(time.RFC3339), s.failed)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// composeStatus is `status --json`, composed in this process against its stores.
|
||||
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||
return func(ctx context.Context) ([]byte, error) {
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return statusAsJSON(asked)
|
||||
}
|
||||
}
|
||||
|
||||
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
|
||||
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
|
||||
// controller composing for ever.
|
||||
var readingVerbs = map[string]bool{
|
||||
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||
"doctor": true, "conditions": true,
|
||||
}
|
||||
|
||||
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||
type nudgingListener struct {
|
||||
link.Enrolment
|
||||
summary *statusSummary
|
||||
// open is the serving controller's stores, for replacing a given value after a module's first
|
||||
// good start (novox/hq ADR 0228).
|
||||
open *stores
|
||||
}
|
||||
|
||||
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||
// A declaration refused as older than the one the machine holds is counted by its writer — the
|
||||
// controller epoch it claimed (novox/hq to-be 45 §6, S13) — and so is what the machine's own count
|
||||
// says it refused beyond the refusals heard.
|
||||
now := time.Now()
|
||||
if report.StaleRefusalOf() {
|
||||
link.StaleRefusals.Refused(link.Refusal{Writer: link.WriterEpoch(report.Epoch), Epoch: report.Epoch,
|
||||
Receiver: report.Node, At: now})
|
||||
}
|
||||
if report.Ordered() {
|
||||
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
||||
}
|
||||
news, err := l.Enrolment.Heard(ctx, report)
|
||||
if news {
|
||||
l.summary.nudge()
|
||||
}
|
||||
if err == nil && l.open != nil && startedWell(report) {
|
||||
// Off the report's path: replacing a given value sends the machine, and a report waits for
|
||||
// nothing it caused (novox/hq ADR 0228).
|
||||
go replaceGiven(context.WithoutCancel(ctx), l.open, report)
|
||||
}
|
||||
return news, err
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// quickly shortens the summary's clocks for one test.
|
||||
func quickly(t *testing.T) {
|
||||
t.Helper()
|
||||
every, settle := statusEvery, statusSettle
|
||||
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
|
||||
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
|
||||
}
|
||||
|
||||
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
|
||||
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
|
||||
// verb answered "still running"; from the summary it answers at once, in full, saying when.
|
||||
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
|
||||
quickly(t)
|
||||
var composed atomic.Int32
|
||||
slow := make(chan struct{})
|
||||
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
|
||||
if composed.Add(1) > 1 {
|
||||
<-slow // every composition after the first outlasts any caller
|
||||
}
|
||||
return []byte(`{"wrong":[],"machines":4}`), nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
defer close(slow)
|
||||
go s.keep(ctx)
|
||||
for i := 0; i < 5; i++ {
|
||||
s.nudge() // a composition is under way and does not finish
|
||||
start := time.Now()
|
||||
got, err := s.answer(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if took := time.Since(start); took > time.Second {
|
||||
t.Fatalf("answer %d took %s", i+1, took)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
|
||||
t.Fatalf("answer %d was not in full: %v", i+1, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The first composition is waited for, never past the caller's window; a controller with none yet
|
||||
// says so rather than answering an empty mesh as a well one.
|
||||
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
|
||||
quickly(t)
|
||||
was := link.AnswerWithin
|
||||
link.AnswerWithin = 1100 * time.Millisecond
|
||||
t.Cleanup(func() { link.AnswerWithin = was })
|
||||
never := make(chan struct{})
|
||||
defer close(never)
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
start := time.Now()
|
||||
_, err := s.answer(ctx)
|
||||
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
|
||||
t.Fatalf("answered %v", err)
|
||||
}
|
||||
if took := time.Since(start); took > link.AnswerWithin {
|
||||
t.Fatalf("waited %s, past the caller's window", took)
|
||||
}
|
||||
}
|
||||
|
||||
// A nudge composes it again, once for several close together; a failed composition leaves the last
|
||||
// summary standing and says it is the last that could be composed.
|
||||
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
|
||||
quickly(t)
|
||||
var composed atomic.Int32
|
||||
fail := atomic.Bool{}
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) {
|
||||
n := composed.Add(1)
|
||||
if fail.Load() {
|
||||
return nil, errors.New("the store did not answer")
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"n": n})
|
||||
return body, nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
if _, err := s.answer(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.nudge()
|
||||
s.nudge()
|
||||
s.nudge()
|
||||
waitFor(t, func() bool { return composed.Load() == 2 })
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if n := composed.Load(); n != 2 {
|
||||
t.Fatalf("three nudges together composed %d times after the first", n-1)
|
||||
}
|
||||
fail.Store(true)
|
||||
s.nudge()
|
||||
waitFor(t, func() bool { return composed.Load() == 3 })
|
||||
waitFor(t, func() bool {
|
||||
got, err := s.answer(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
|
||||
})
|
||||
}
|
||||
|
||||
// The seat's `status` answers from the summary when this process keeps one.
|
||||
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
|
||||
quickly(t)
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
statusFrom = s
|
||||
t.Cleanup(func() { statusFrom = nil })
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
|
||||
t.Fatalf("answered %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, ok func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for !ok() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("never happened")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,8 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
||||
inv.Close()
|
||||
return nil, err
|
||||
}
|
||||
// A plan is written only under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
inv.ActsUnder(theLease.epoch)
|
||||
// Load the seat set from the store, so the control plane reads the set as data rather than as
|
||||
// the slice it was compiled with (novox/hq ADR 0122). A store not yet seeded — or one whose
|
||||
// seat table a migration has not reached — returns nothing, and UseSeats leaves the compiled
|
||||
|
||||
@@ -72,24 +72,28 @@ func TestAPersonClosesAStuckPlan(t *testing.T) {
|
||||
stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
|
||||
Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
|
||||
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, &stuck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a plan was closed by hand without saying why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
|
||||
!strings.Contains(closed.Note, "its report will not come") {
|
||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
|
||||
t.Fatal("a plan already closed was closed again")
|
||||
}
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
|
||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -266,6 +267,11 @@ func notNow(err error) error {
|
||||
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
||||
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
||||
// what the bus did not hand over (novox/hq issue 266). Each judges against what the other wrote.
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
@@ -276,34 +282,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
return notNow(err)
|
||||
}
|
||||
|
||||
// Two kinds of module are affected by one merge, and they are affected differently.
|
||||
//
|
||||
// A module **built from** this repository and branch has moved: the mesh records the new commit
|
||||
// as what its source now has, and only what the merge actually changed is rebuilt. A module that
|
||||
// only **packages source from** it has not moved — its own source is somewhere else, at the
|
||||
// commit it already records — so it is rebuilt and its record left alone. Writing this commit as
|
||||
// its source would make it permanently behind a repository its manifest does not come from.
|
||||
var from, packaging []inventory.Entry
|
||||
already := 0
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case sourceIs(e.Source, m):
|
||||
if e.Source.BuiltFrom == m.Commit {
|
||||
already++
|
||||
continue
|
||||
}
|
||||
// **A merge older than the last look at the source is history, not a move.** The forge
|
||||
// announces what it finds merged, and an old merge surfacing late would otherwise move
|
||||
// the recorded head backwards and rebuild everything built from that repository, once
|
||||
// per old merge (2026-09-28).
|
||||
if isHistory(m.MergedAt, e.Source.Seen) {
|
||||
continue
|
||||
}
|
||||
from = append(from, e)
|
||||
case readsFrom(read[e.Manifest.Module], m):
|
||||
packaging = append(packaging, e)
|
||||
}
|
||||
}
|
||||
from, packaging, already := mergeCandidates(m, entries, read)
|
||||
if len(from) == 0 && len(packaging) == 0 {
|
||||
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
||||
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
|
||||
@@ -404,13 +383,13 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
||||
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
||||
}
|
||||
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
|
||||
// both open, which the next merge settles, rather than neither.
|
||||
for _, old := range superseded {
|
||||
if err := inv.SavePlan(ctx, old); err != nil {
|
||||
if err := inv.SavePlan(ctx, &old); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
fmt.Printf(" %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
|
||||
@@ -432,12 +411,63 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
if err := askTier(ctx, inv, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||
var actingOnMerges sync.Mutex
|
||||
|
||||
// mergeCandidates is what one merge could move, judged against what the catalogue holds.
|
||||
//
|
||||
// Two kinds of module are affected by one merge, and they are affected differently.
|
||||
//
|
||||
// A module **built from** this repository and branch has moved: the mesh records the new commit as
|
||||
// what its source now has, and only what the merge actually changed is rebuilt. A module that only
|
||||
// **packages source from** it has not moved — its own source is somewhere else, at the commit it
|
||||
// already records — so it is rebuilt and its record left alone. Writing this commit as its source
|
||||
// would make it permanently behind a repository its manifest does not come from. `already` counts
|
||||
// the modules built from this repository that are already built from this very commit.
|
||||
func mergeCandidates(m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) (from, packaging []inventory.Entry, already int) {
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case sourceIs(e.Source, m):
|
||||
if e.Source.BuiltFrom == m.Commit {
|
||||
already++
|
||||
continue
|
||||
}
|
||||
// **A merge older than the last look at the source is history, not a move.** The forge
|
||||
// announces what it finds merged, and an old merge surfacing late would otherwise move
|
||||
// the recorded head backwards and rebuild everything built from that repository, once
|
||||
// per old merge (2026-09-28).
|
||||
if isHistory(m.MergedAt, e.Source.Seen) {
|
||||
continue
|
||||
}
|
||||
from = append(from, e)
|
||||
case readsFrom(read[e.Manifest.Module], m):
|
||||
packaging = append(packaging, e)
|
||||
}
|
||||
}
|
||||
return from, packaging, already
|
||||
}
|
||||
|
||||
// wouldMove is the modules built from the merged repository that acting on this merge would mark as
|
||||
// moved and rebuild — SourceMoved's judgement, made without acting (novox/hq issue 266). Empty for a
|
||||
// merge already acted on: acting marks each of them as looked at, so the merge then reads as history.
|
||||
//
|
||||
// **Only the modules built from it, never the ones that merely package source from it.** Acting
|
||||
// records nothing about those, so a merge acted on would go on reading as unacted for them, and be
|
||||
// acted on again on every look. A merge that moves both is caught by the first kind, and acting on it
|
||||
// rebuilds the second as well.
|
||||
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||
from, _, _ := mergeCandidates(m, entries, read)
|
||||
return whatTheMergeTouched(from, entries, m)
|
||||
}
|
||||
|
||||
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
|
||||
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
|
||||
// An empty recorded ref is the repository's default branch, which is what a merge into the base
|
||||
|
||||
@@ -0,0 +1,591 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The watchdogs (novox/hq to-be 45 §3): every row of the signals table, run over what the serving
|
||||
// controller knows, every half minute.
|
||||
//
|
||||
// **One loop, one gathering, every row.** The facts are gathered once a tick — the store's machines,
|
||||
// plans and durations, the bus's queue and consumers, what this process heard — and each row's watch
|
||||
// is a pure reading of them, which is what lets the test generated from the table suppress a signal by
|
||||
// changing a fact. A part that cannot be gathered makes the rows that read it blind: each says so as
|
||||
// a condition of its own (`probe.<row>.failed`) and keeps what it raised before, because a watchdog
|
||||
// that cannot see must not read as one that sees nothing wrong (ADR 0227 rule 4).
|
||||
//
|
||||
// **The watchdogs are themselves watched.** Each tick is recorded; the self-check (doctor.go) fails
|
||||
// its probe DW when the ticks stop, and the watchdogs raise S10 when the self-check stops — two loops
|
||||
// watching each other, and mesh-watcher on a second machine watching the self-check's heartbeat for
|
||||
// the case both stop with the process.
|
||||
|
||||
// watchEvery is how often the watchdogs run.
|
||||
var watchEvery = 30 * time.Second
|
||||
|
||||
// signalFacts is what one tick of the watchdogs reads.
|
||||
type signalFacts struct {
|
||||
now time.Time
|
||||
started time.Time
|
||||
// host is the machine this controller runs on, as the mesh names it, for a condition about itself.
|
||||
host string
|
||||
|
||||
machines []machineFacts
|
||||
machinesErr error
|
||||
// toolsHeardFrom is when this process began hearing node tools: one not heard since is silent
|
||||
// since then at the most.
|
||||
toolsHeardFrom time.Time
|
||||
|
||||
plans []planFacts
|
||||
plansErr error
|
||||
|
||||
loop loopFacts
|
||||
loopErr error
|
||||
|
||||
merges []missedMerge
|
||||
mergesPassed time.Time
|
||||
mergesErr error
|
||||
|
||||
asks []askFacts
|
||||
asksErr error
|
||||
|
||||
calls []link.Call
|
||||
|
||||
standings []conditions.Condition
|
||||
standingsErr error
|
||||
|
||||
advisories []link.Advisory
|
||||
lostConsumers map[string]bool
|
||||
advisoriesErr error
|
||||
|
||||
selfCheck selfCheckFacts
|
||||
|
||||
// staleRefusals are the writers refused as older lately, and epochs the mesh's record of each epoch
|
||||
// they name (novox/hq to-be 45 §6, S13).
|
||||
staleRefusals []link.WriterRefusals
|
||||
epochs map[int64]inventory.Epoch
|
||||
|
||||
// lease is this controller's standing to the lease, and the epochs that ended lately (S12).
|
||||
lease leaseFacts
|
||||
leaseErr error
|
||||
}
|
||||
|
||||
type leaseFacts struct {
|
||||
held bool
|
||||
epoch uint64
|
||||
renewed time.Time
|
||||
unleased string
|
||||
ended []inventory.Epoch
|
||||
// reset is when the lease bucket was found raised again from nothing; resetSaid what of it.
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
}
|
||||
|
||||
type machineFacts struct {
|
||||
name string
|
||||
control bool
|
||||
// lastHeard is the store's last word from it, any word; zero when never.
|
||||
lastHeard time.Time
|
||||
// every is the heartbeat interval it said; zero when it said none.
|
||||
every time.Duration
|
||||
power link.PowerState
|
||||
// sentAt is when it was last sent a declaration; reportedCurrent whether it has reported that one.
|
||||
sentAt time.Time
|
||||
reportedCurrent bool
|
||||
reportedAt time.Time
|
||||
// lastApply is its newest measured apply.
|
||||
lastApply time.Duration
|
||||
// tools is whether node-tools is assigned there; toolsHeard and toolsEvery its heartbeat.
|
||||
tools bool
|
||||
toolsHeard time.Time
|
||||
toolsEvery time.Duration
|
||||
}
|
||||
|
||||
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||
func (m machineFacts) asleep() bool { return m.power.Away() }
|
||||
|
||||
type planFacts struct {
|
||||
id, repository, commit string
|
||||
tier, tiers int
|
||||
entered time.Time
|
||||
bound time.Duration
|
||||
waiting string
|
||||
paused bool
|
||||
}
|
||||
|
||||
type loopFacts struct {
|
||||
took time.Time
|
||||
pending uint64
|
||||
where string
|
||||
}
|
||||
|
||||
type askFacts struct {
|
||||
id, seat, what, state, on string
|
||||
since time.Time
|
||||
bound time.Duration
|
||||
}
|
||||
|
||||
type selfCheckFacts struct {
|
||||
last time.Time
|
||||
every time.Duration
|
||||
}
|
||||
|
||||
// watchdogs is the loop and what it needs.
|
||||
type watchdogs struct {
|
||||
open *stores
|
||||
server *link.Server
|
||||
js *broker.JetStream
|
||||
keeper *conditions.Keeper
|
||||
doctor *doctor
|
||||
|
||||
started time.Time
|
||||
// acting says this controller is the one acting, not one standing by (link.Holding): a controller
|
||||
// standing by hears no heartbeat and would call every machine silent.
|
||||
acting func() bool
|
||||
|
||||
mu sync.Mutex
|
||||
ticked time.Time
|
||||
last *signalFacts
|
||||
failed string
|
||||
}
|
||||
|
||||
// lastTick is when the watchdogs last finished a tick.
|
||||
func (w *watchdogs) lastTick() time.Time {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.ticked
|
||||
}
|
||||
|
||||
// lastFacts is the facts of the newest tick, for `doctor signals`; nil before the first.
|
||||
func (w *watchdogs) lastFacts() *signalFacts {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.last
|
||||
}
|
||||
|
||||
// keep runs the watchdogs until ctx ends.
|
||||
func (w *watchdogs) keep(ctx context.Context) {
|
||||
tick := time.NewTicker(watchEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
w.tick(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick is one run of every row.
|
||||
func (w *watchdogs) tick(ctx context.Context) {
|
||||
if w.acting != nil && !w.acting() {
|
||||
// Standing by: nothing seen, nothing said, and the tick counted — this process's self-check
|
||||
// is not the one that matters while another acts.
|
||||
w.mu.Lock()
|
||||
w.ticked = time.Now()
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
running, cancel := context.WithTimeout(ctx, watchEvery)
|
||||
defer cancel()
|
||||
w.see(running, w.gather(running))
|
||||
}
|
||||
|
||||
// see runs every watched row over one gathering, keeps what each found, and records the tick.
|
||||
func (w *watchdogs) see(running context.Context, f *signalFacts) {
|
||||
var problems []string
|
||||
var blind []conditions.Observation
|
||||
for _, row := range watchedRows() {
|
||||
if err := row.needs(f); err != nil {
|
||||
blind = append(blind, blindRow(row, err))
|
||||
continue
|
||||
}
|
||||
if err := w.keeper.Reconcile(running, row.Row, row.watch(f)); err != nil {
|
||||
problems = append(problems, row.Row+": "+err.Error())
|
||||
}
|
||||
}
|
||||
// The rows that could not see, said; the ones that see again, cleared.
|
||||
if err := w.keeper.Reconcile(running, sourceWatchdogs, blind); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
// A provider no longer assigned where it ran: its standing is resolved by the assignment.
|
||||
if f.standingsErr == nil && w.open != nil {
|
||||
if err := unassignedProviders(running, w.open.inventory, w.keeper, f.standings); err != nil {
|
||||
problems = append(problems, "S8: "+err.Error())
|
||||
}
|
||||
}
|
||||
if err := w.keeper.EndSilences(running); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
failed := ""
|
||||
if len(problems) > 0 {
|
||||
failed = fmt.Sprintf("%v", problems)
|
||||
}
|
||||
w.mu.Lock()
|
||||
said := w.failed
|
||||
w.ticked, w.last, w.failed = time.Now(), f, failed
|
||||
w.mu.Unlock()
|
||||
if failed != said {
|
||||
if failed != "" {
|
||||
fmt.Printf("the watchdogs could not keep what they saw: %s\n", failed)
|
||||
} else if said != "" {
|
||||
fmt.Println("the watchdogs keep what they see again")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sourceWatchdogs is what raises a blind row's condition.
|
||||
const sourceWatchdogs = "watchdogs"
|
||||
|
||||
// blindRow is a row whose facts could not be gathered, as a condition of its own.
|
||||
func blindRow(row signalRow, err error) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeProbe, ID: row.Row, Kind: "probe-failed", Token: "failed",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the watchdog of %s (%s) cannot see: what it reads could not be read, so nothing "+
|
||||
"it would raise can be — and nothing it raised before is cleared", row.Row, row.Signal),
|
||||
Said: firstLine(err.Error())}
|
||||
}
|
||||
|
||||
// gather reads every fact a tick needs. Each part's failure is kept beside it, never an empty part.
|
||||
func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
now := time.Now()
|
||||
f := &signalFacts{now: now, started: w.started, toolsHeardFrom: link.ToolsBeats.Started(), calls: link.Calls.Running(),
|
||||
staleRefusals: link.StaleRefusals.Within(now.Add(-staleRefusalsWithin)), lostConsumers: map[string]bool{},
|
||||
epochs: map[int64]inventory.Epoch{}}
|
||||
if w.doctor != nil {
|
||||
f.selfCheck = selfCheckFacts{last: w.doctor.lastRunEnded(), every: doctorEvery}
|
||||
}
|
||||
inv := w.open.inventory
|
||||
f.host = controlHost(ctx, inv)
|
||||
f.lease, f.leaseErr = gatherLease(ctx, inv, now)
|
||||
for _, r := range f.staleRefusals {
|
||||
if r.Epoch <= 0 {
|
||||
continue
|
||||
}
|
||||
// Named where the record has it; a writer the record cannot name is still said by its epoch.
|
||||
if e, found, err := inv.EpochOf(ctx, uint64(r.Epoch)); err == nil && found {
|
||||
f.epochs[r.Epoch] = e
|
||||
}
|
||||
}
|
||||
f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now)
|
||||
f.plans, f.plansErr = gatherPlans(ctx, inv, now)
|
||||
f.loop, f.loopErr = w.gatherLoop()
|
||||
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
||||
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
||||
f.mergesErr = fmt.Errorf("the catch-up of merges has not passed since %s", f.mergesPassed.UTC().Format(time.RFC3339))
|
||||
}
|
||||
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||
if open, err := w.keeper.Open(ctx); err != nil {
|
||||
f.standingsErr = err
|
||||
} else {
|
||||
f.standings = providerStandings(open)
|
||||
}
|
||||
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||
return f
|
||||
}
|
||||
|
||||
// gatherLease is this controller's standing to the lease and the epochs that ended within the hour.
|
||||
func gatherLease(ctx context.Context, inv *inventory.Inventory, now time.Time) (leaseFacts, error) {
|
||||
st := theLease.standing()
|
||||
f := leaseFacts{held: st.Held, epoch: st.Epoch, renewed: st.Renewed, unleased: st.Unleased, reset: st.Reset,
|
||||
resetSaid: st.ResetSaid}
|
||||
ended, err := inv.EpochsSince(ctx, now.Add(-advisoryQuiet))
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the epochs the mesh issued cannot be read: %w", err)
|
||||
}
|
||||
f.ended = ended
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// controlHost is the machine running the controller, as the mesh names it: the one the controller
|
||||
// module is assigned to, or this process's host name where that is not one machine.
|
||||
func controlHost(ctx context.Context, inv *inventory.Inventory) string {
|
||||
if on, err := inv.Running(ctx, "mesh-controller"); err == nil && len(on) == 1 {
|
||||
return on[0]
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
return host
|
||||
}
|
||||
|
||||
func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]machineFacts, error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the machines cannot be read: %w", err)
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what each machine last reported cannot be read: %w", err)
|
||||
}
|
||||
reported := map[string]inventory.Reported{}
|
||||
for _, r := range reports {
|
||||
reported[r.Node] = r
|
||||
}
|
||||
control, err := inv.Running(ctx, "mesh-controller")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("where the controller runs cannot be read: %w", err)
|
||||
}
|
||||
tooled, err := inv.Running(ctx, broker.RuntimeModule)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("where the node tools run cannot be read: %w", err)
|
||||
}
|
||||
applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the measured applies cannot be read: %w", err)
|
||||
}
|
||||
lastApply := map[string]time.Duration{}
|
||||
for _, d := range applies { // oldest first: the last one read is the newest
|
||||
lastApply[d.Node] = d.Took
|
||||
}
|
||||
var out []machineFacts
|
||||
anyPast := false
|
||||
for _, n := range nodes {
|
||||
m := machineFacts{name: n.Name, control: slices.Contains(control, n.Name), lastHeard: n.LastSeen,
|
||||
lastApply: lastApply[n.Name], tools: slices.Contains(tooled, n.Name)}
|
||||
if beat, ok := link.HostBeats.Of(n.Name); ok {
|
||||
m.every = beat.Every
|
||||
}
|
||||
if beat, ok := link.ToolsBeats.Of(n.Name); ok {
|
||||
m.toolsHeard, m.toolsEvery = beat.At, beat.Every
|
||||
}
|
||||
if r, ok := reported[n.Name]; ok {
|
||||
if r.Sent != nil {
|
||||
m.sentAt = *r.Sent
|
||||
}
|
||||
if r.At != nil {
|
||||
m.reportedAt = *r.At
|
||||
}
|
||||
m.reportedCurrent = r.Current
|
||||
}
|
||||
if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
anyPast = true
|
||||
}
|
||||
if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
|
||||
anyPast = true
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
// What a machine past its bound last said of its power, read only then: a machine that said it
|
||||
// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
|
||||
// which is the louder mistake and the right one to make.
|
||||
if anyPast && w.server != nil {
|
||||
states, err := w.server.PowerStates(ctx, now.Add(-7*24*time.Hour))
|
||||
if err != nil {
|
||||
fmt.Printf("what machines said of their power cannot be read, so a sleeping one is called silent: %v\n", err)
|
||||
}
|
||||
for i := range out {
|
||||
out[i].power = states[out[i].name]
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// gatherPlans is every open plan, its tier's bound from what was measured, and what it waits on.
|
||||
func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]planFacts, error) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the open plans cannot be read: %w", err)
|
||||
}
|
||||
if len(plans) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
|
||||
}
|
||||
measured := map[string][]time.Duration{}
|
||||
for _, d := range tiers {
|
||||
measured[d.Subject] = append(measured[d.Subject], d.Took)
|
||||
}
|
||||
pause := buildSeatPause(ctx, inv, plans)
|
||||
var out []planFacts
|
||||
for _, p := range plans {
|
||||
_, paused := pausedWaiting(p, pause, now)
|
||||
out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
|
||||
tiers: len(p.Tiers), entered: p.TierEntered, bound: max(tierAtLeast, 3*p90(measured[p.Repository])),
|
||||
waiting: planLineWith(p, now, pause), paused: paused})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// p90 is the ninetieth percentile of measurements; zero for none.
|
||||
func p90(took []time.Duration) time.Duration {
|
||||
if len(took) == 0 {
|
||||
return 0
|
||||
}
|
||||
sorted := append([]time.Duration(nil), took...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
|
||||
return sorted[int(0.9*float64(len(sorted)-1))]
|
||||
}
|
||||
|
||||
// gatherLoop is when the event loop last took a message and what its consumers hold.
|
||||
func (w *watchdogs) gatherLoop() (loopFacts, error) {
|
||||
f := loopFacts{took: link.Loop.Last()}
|
||||
if w.js == nil {
|
||||
return f, errors.New("this controller is not on the bus")
|
||||
}
|
||||
var where []string
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
info, err := w.js.Context().ConsumerInfo(c.Stream, c.Name)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the controller's consumer on %s cannot be read: %w", c.Stream, err)
|
||||
}
|
||||
if held := info.NumPending + uint64(info.NumAckPending); held > 0 {
|
||||
f.pending += held
|
||||
where = append(where, fmt.Sprintf("%d on %s", held, c.Stream))
|
||||
}
|
||||
}
|
||||
f.where = fmt.Sprint(where)
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// gatherAsks is every ask in the build seat's queue that is in flight or dead, with its bound.
|
||||
func (w *watchdogs) gatherAsks(ctx context.Context, inv *inventory.Inventory) ([]askFacts, error) {
|
||||
if w.js == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||
}
|
||||
seat := buildSeatAmong(entries)
|
||||
q, err := link.ReadQueue(ctx, w.js, seat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds, err := inv.Durations(ctx, inventory.DurationBuild, time.Now().Add(-14*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the measured builds cannot be read: %w", err)
|
||||
}
|
||||
var took []time.Duration
|
||||
for _, d := range builds {
|
||||
took = append(took, d.Took)
|
||||
}
|
||||
bound := askDefault
|
||||
if len(took) > 0 {
|
||||
bound = max(askAtLeast, 3*p90(took))
|
||||
}
|
||||
var out []askFacts
|
||||
for _, a := range q.Asks {
|
||||
if a.State != link.AskInFlight && a.State != link.AskDead {
|
||||
continue
|
||||
}
|
||||
since := a.Started
|
||||
if since.IsZero() {
|
||||
since = a.AskedAt
|
||||
}
|
||||
what := a.Repository
|
||||
if a.Path != "" {
|
||||
what += " " + a.Path
|
||||
}
|
||||
out = append(out, askFacts{id: a.ID, seat: seat, what: what, state: a.State, on: a.On, since: since, bound: bound})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// lostConsumers is, of the consumers the bus said were deleted, each that is still missing and that
|
||||
// the mesh expects: a consumer removed because its module was unassigned is not lost.
|
||||
func (w *watchdogs) lostConsumers(ctx context.Context, heard []link.Advisory) (map[string]bool, error) {
|
||||
out := map[string]bool{}
|
||||
var deleted []link.Advisory
|
||||
for _, a := range heard {
|
||||
if a.Kind == link.AdvisoryConsumerLost {
|
||||
deleted = append(deleted, a)
|
||||
}
|
||||
}
|
||||
if len(deleted) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
if w.js == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
_, expected, err := expectedBusObjects(ctx, w.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, c := range expected {
|
||||
want[c.Stream+"."+c.Name] = true
|
||||
}
|
||||
for _, a := range deleted {
|
||||
_, err := w.js.Context().ConsumerInfo(a.Stream, a.Consumer)
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||
out[a.Stream+"."+a.Consumer] = want[a.Stream+"."+a.Consumer]
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("whether %s exists cannot be read: %w", link.ConsumerInWords(a.Stream, a.Consumer), err)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// later is the later of two moments.
|
||||
func later(a, b time.Time) time.Time {
|
||||
if a.After(b) {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// watchTheMesh opens the condition store and starts the watchdogs, the bus's advisories and the
|
||||
// self-check, for the serving controller; the returned function stops them. Nil when the store could
|
||||
// not be opened, which is said.
|
||||
func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
|
||||
keeper, err := keeperOn(ctx, bus.Conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
|
||||
"status says the conditions cannot be read: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
conditionsFrom = keeper
|
||||
logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
stopHearing, err := server.HearAdvisories(logf)
|
||||
if err != nil {
|
||||
fmt.Printf("what the bus says about itself cannot be heard (S9 is blind): %v\n", err)
|
||||
stopHearing = func() {}
|
||||
}
|
||||
host := controlHost(ctx, open.inventory)
|
||||
w := &watchdogs{open: open, server: server, js: server.JetStream(), keeper: keeper, started: time.Now(),
|
||||
acting: link.Holding}
|
||||
d := &doctor{open: open, js: server.JetStream(), keeper: keeper, teller: bus, watchdogs: w, host: host}
|
||||
w.doctor = d
|
||||
doctorFrom = d
|
||||
watching, stop := context.WithCancel(ctx)
|
||||
go w.keep(watching)
|
||||
go d.keep(watching)
|
||||
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||
broker.ConditionsBucket, conditions.Seat)
|
||||
return func() {
|
||||
stop()
|
||||
stopHearing()
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
keeper.Close(flushing)
|
||||
}
|
||||
}
|
||||
|
||||
// runnableProbes are the probes the registry runs.
|
||||
func runnableProbes() []probe {
|
||||
var out []probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.run != nil {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -5,7 +5,9 @@ go 1.26.0
|
||||
require (
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/novox/mesh-host v0.0.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/net v0.58.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -15,8 +17,15 @@ require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
)
|
||||
|
||||
// The node-engine's own validator (mesh-host/validate, novox/hq to-be 45 D1): one validator, the host's.
|
||||
// The host's module path names no forge a build can fetch from, so the module is read from the one
|
||||
// that holds it, at the host's commit — **once, by whoever moves the pin, into vendor/**, which is
|
||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||
// `go mod vendor` fails loudly, at once, everywhere.
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e h1:g9h4QRaAMg5yaJLwqtb0FoOs23DVGUYpW6qvnQ3oY5A=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The controller's own key-value buckets (novox/hq to-be 45 §1, §6, §7).
|
||||
//
|
||||
// **What the controller must remember across its own restart, it keeps on the bus.** A call's
|
||||
// outcome lived in the memory of the process that served it (novox/hq issue 265), so a controller
|
||||
// replaced while a push ran answered "no such call" for the one thing its caller had been told to
|
||||
// ask about. The bus already outlives the controller and is the shape ADR 0201 gives a module's
|
||||
// current state: one value per key, written by one owner, read by anybody granted it. These are the
|
||||
// controller's, written by it alone — the writers table of to-be 45 §1 — and asserted on every start
|
||||
// like the streams, so a bus raised from nothing has them before the first call is served.
|
||||
|
||||
// CallsBucket keeps every call of the mesh's own verbs and what came of it; HandActsBucket every act
|
||||
// a person did by hand, with why; ConditionsBucket every condition open now (to-be 45 §2), one key
|
||||
// each, and ConditionHistoryBucket every transition of one — raised, changed, silenced, cleared —
|
||||
// for ninety days.
|
||||
//
|
||||
// **The history is a bucket of its own** because its keys expire and an open condition's must not:
|
||||
// a bucket has one age for every key, and a condition open longer than the history is kept would
|
||||
// otherwise vanish from the store while still true.
|
||||
var (
|
||||
CallsBucket = BucketName(ControllerSeat, "calls")
|
||||
HandActsBucket = BucketName(ControllerSeat, "hand-acts")
|
||||
ConditionsBucket = BucketName(ControllerSeat, "conditions")
|
||||
ConditionHistoryBucket = BucketName(ControllerSeat, "condition-history")
|
||||
// LeaseBucket holds the controller's lease (to-be 45 §6): one key, `holder`, which the instance
|
||||
// allowed to act writes by compare-and-set and renews; its revision when taken is the epoch.
|
||||
LeaseBucket = BucketName(ControllerSeat, "lease")
|
||||
)
|
||||
|
||||
// LeaseTTL is how long the lease's key lives unrenewed (to-be 45 §6): fifteen seconds, renewed
|
||||
// every five. The bucket's age, so the bus forgets a holder that stopped renewing.
|
||||
const LeaseTTL = 15 * time.Second
|
||||
|
||||
// The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
|
||||
// A call is two keys — its record, and its answer apart so a listing does not read every answer —
|
||||
// so the stream holds twice as many messages as it keeps calls.
|
||||
const (
|
||||
KeptCallsDurably = 1000
|
||||
CallsKeptFor = 14 * 24 * time.Hour
|
||||
// CallAnswerBytes is the most of one answer kept: a whole declaration is far smaller, and an
|
||||
// answer larger is cut and says so.
|
||||
CallAnswerBytes = 64 << 10
|
||||
// HandActsKeptFor is as long as a condition's history (to-be 45 §2): an act by hand is read
|
||||
// back beside what it addressed.
|
||||
HandActsKeptFor = 90 * 24 * time.Hour
|
||||
// ConditionHistoryKeptFor is how long a condition's transitions are kept (to-be 45 §2).
|
||||
ConditionHistoryKeptFor = 90 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
|
||||
func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
type ControllerBucketsAsserter interface {
|
||||
EnsureControllerBuckets() error
|
||||
}
|
||||
|
||||
// EnsureControllerBuckets creates the controller's buckets if absent and brings their options to
|
||||
// match. An update, never a delete: what they hold is the record of what the mesh was asked.
|
||||
func (j *JetStream) EnsureControllerBuckets() error {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := EnsureLeaseBucket(ctx, js); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: CallsBucket,
|
||||
Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +
|
||||
"265): written by the controller alone, read through `calls`; the last thousand, or fourteen days",
|
||||
History: 1,
|
||||
TTL: CallsKeptFor,
|
||||
MaxValueSize: CallAnswerBytes + 4<<10,
|
||||
MaxBytes: 2 * KeptCallsDurably * (CallAnswerBytes + 4<<10),
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CallsBucket, err)
|
||||
}
|
||||
// **The count, on the stream under the bucket.** A bucket has an age and a size and no count;
|
||||
// the stream it is made of does, and with one value per key the oldest message is the oldest
|
||||
// call. Asserted after the bucket, every time, because asserting the bucket writes the stream's
|
||||
// configuration whole and puts the count back to none.
|
||||
stream, err := js.Stream(ctx, "KV_"+CallsBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the stream under %s: %w", CallsBucket, err)
|
||||
}
|
||||
cfg := stream.CachedInfo().Config
|
||||
if cfg.MaxMsgs != 2*KeptCallsDurably {
|
||||
cfg.MaxMsgs = 2 * KeptCallsDurably
|
||||
cfg.Discard = jetstream.DiscardOld
|
||||
if _, err := js.UpdateStream(ctx, cfg); err != nil {
|
||||
return fmt.Errorf("bounding %s to the last %d calls: %w", CallsBucket, KeptCallsDurably, err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: HandActsBucket,
|
||||
Description: "every act a person did by hand, with why (novox/hq to-be 45 §7): written by the " +
|
||||
"controller's repairing verbs and `hand-act record`, read through `hand-acts`",
|
||||
History: 1,
|
||||
TTL: HandActsKeptFor,
|
||||
MaxValueSize: 16 << 10,
|
||||
MaxBytes: 64 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", HandActsBucket, err)
|
||||
}
|
||||
// **No age on the open conditions.** A condition is removed when observation clears it and at no
|
||||
// other moment: one that expired would be a fault the store forgot while it was still true.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: ConditionsBucket,
|
||||
Description: "every condition open now, one key each (novox/hq to-be 45 §2): written by the " +
|
||||
"controller alone, raised and cleared by observation, read through `conditions`",
|
||||
History: 1,
|
||||
MaxValueSize: 64 << 10,
|
||||
MaxBytes: 64 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", ConditionsBucket, err)
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: ConditionHistoryBucket,
|
||||
Description: "every transition of a condition — raised, changed, silenced, cleared — kept ninety " +
|
||||
"days (novox/hq to-be 45 §2): written by the controller alone, read through `conditions history`",
|
||||
History: 1,
|
||||
TTL: ConditionHistoryKeptFor,
|
||||
MaxValueSize: 64 << 10,
|
||||
MaxBytes: 256 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", ConditionHistoryBucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureLeaseBucket creates the lease's bucket if absent and brings its options to match (to-be 45
|
||||
// §6). **Before the lease is taken, by any candidate**: it is the lease's own precondition, and asserting
|
||||
// a bucket that exists changes nothing. File storage, so its revisions — the epochs — outlive a restart of
|
||||
// the bus; one raised again from nothing is moved past the highest epoch issued when the lease is taken.
|
||||
func EnsureLeaseBucket(ctx context.Context, js jetstream.JetStream) error {
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: LeaseBucket,
|
||||
Description: "the controller's lease (novox/hq to-be 45 §6): the key `holder`, written by compare-and-set " +
|
||||
"by the one controller instance that may act and renewed every five seconds; its revision when taken " +
|
||||
"is the epoch every declaration and plan write carries",
|
||||
History: 1,
|
||||
TTL: LeaseTTL,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 1 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", LeaseBucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a
|
||||
// key is a publish to the bucket's own subject, which the management interface's grant does not
|
||||
// cover: the cancelled sets' writes timed out for want of this, and the controller's own buckets
|
||||
// would have.
|
||||
func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
|
||||
p, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>"}
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
if hasCancelledSet(seat) {
|
||||
want = append(want, "$KV."+CancelledSetName(seat)+".>")
|
||||
}
|
||||
}
|
||||
for _, subject := range want {
|
||||
if !slices.Contains(p.Publish, subject) {
|
||||
t.Errorf("the controller may not publish %s, so it cannot write that bucket", subject)
|
||||
}
|
||||
}
|
||||
if slices.Contains(p.Publish, "$KV.>") {
|
||||
t.Error("the controller may write any bucket, a module's state included")
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine's node tools may say they are there, as that machine and no other** (novox/hq to-be 45
|
||||
// S11), and the controller may hear the bus's advisories and ask who answers — read-only, named.
|
||||
func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) {
|
||||
tools, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(tools.Publish, "mesh.control.anchor.tools-alive") {
|
||||
t.Error("the node tools may not say they are there")
|
||||
}
|
||||
for _, s := range tools.Publish {
|
||||
if strings.Contains(s, "tools-alive") && s != "mesh.control.anchor.tools-alive" {
|
||||
t.Errorf("the node tools may say %s", s)
|
||||
}
|
||||
}
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range BusAdvisories {
|
||||
if !slices.Contains(controller.Subscribe, s) {
|
||||
t.Errorf("the controller may not hear %s", s)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(controller.Publish, "$SRV.INFO") || slices.Contains(controller.Subscribe, "$JS.EVENT.>") {
|
||||
t.Error("the controller may not ask who answers, or hears every API call")
|
||||
}
|
||||
}
|
||||
+61
-4
@@ -18,6 +18,7 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A Kind is what a principal is, which decides the shape of its authority rather than its
|
||||
@@ -124,6 +125,16 @@ type Principal struct {
|
||||
// goes with the retired seat row.
|
||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||
|
||||
// SeatVerb is one verb of one seat, on every machine holding it.
|
||||
type SeatVerb struct{ Seat, Verb string }
|
||||
|
||||
// VerbsTheSelfCheckAsks are the seat verbs the controller's self-check and watchdogs call (novox/hq
|
||||
// to-be 45 §4): D8 reads every machine's ban list. **Named one by one, and the test that holds them
|
||||
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
|
||||
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
|
||||
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
|
||||
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}
|
||||
|
||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
||||
var perMachineEvents = map[string]bool{"paused.*": true}
|
||||
@@ -189,6 +200,13 @@ type Permissions struct {
|
||||
AllowResponses bool
|
||||
}
|
||||
|
||||
// ResponseTTL is how long the bus lets a principal answer a request it received. Its one answer has
|
||||
// to come inside this, and a seat's holder answers within link.AnswerWithin — inside it by design.
|
||||
// **A broker reloading its user list forgets every answer it was about to permit**, whatever this
|
||||
// says (novox/hq issue 265): a call that is still running when the list reloads has its answer
|
||||
// refused, which is why a holder answers before it does what can reload it.
|
||||
const ResponseTTL = time.Minute
|
||||
|
||||
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
|
||||
// a permission that cannot be derived from a declaration is a permission nobody can explain.
|
||||
func PermissionsFor(p Principal) (Permissions, error) {
|
||||
@@ -211,7 +229,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// stream definitions (design 25 §3), so it alone reaches the JetStream API — and it alone
|
||||
// issues memberships (novox/hq ADR 0160), which it publishes into the assignments stream
|
||||
// after each push; refused by the server on 2026-10-01 until this line named them.
|
||||
pub = []string{"mesh.control.>", "mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
|
||||
//
|
||||
// **Not what the machines say** (novox/hq to-be 45 §1): `mesh.control.>` is subscribed, never
|
||||
// published — a machine's report has one writer, its node-engine, and the controller granted
|
||||
// that subject would be a second (the writers table refuses it). It was granted from the first
|
||||
// composition and nothing ever published under it.
|
||||
pub = []string{"mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
|
||||
// **And where its consumers deliver.** A push consumer delivers on `_DELIVER.<its name>`,
|
||||
// and a client bound to it subscribes exactly that; the server refused it for every
|
||||
// principal the first time one bound a consumer (2026-09-28). Each kind below is granted
|
||||
@@ -229,6 +252,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
||||
// **And its cancelled set** (novox/hq ADR 0219, issue 269): a cancel writes the ask's id
|
||||
// there before it deletes the ask, and a write is a publish to the bucket's subject, which
|
||||
// `$JS.API.>` does not cover — so every cancel timed out, refused by this list.
|
||||
if hasCancelledSet(seat) {
|
||||
pub = append(pub, "$KV."+CancelledSetName(seat)+".>")
|
||||
}
|
||||
}
|
||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||
// facts under the seat it holds, because a role's events belong to the role and keep their
|
||||
@@ -252,11 +281,19 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||
sub = append(sub, announcing(ControllerSeat)...)
|
||||
// And the verbs the self-check reads with, of any machine's holder (novox/hq to-be 45 §4).
|
||||
for _, v := range VerbsTheSelfCheckAsks {
|
||||
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*")
|
||||
}
|
||||
// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
|
||||
// the same discovery the console reads. The question only; the answers come to its own inbox.
|
||||
pub = append(pub, "$SRV.INFO")
|
||||
|
||||
// The two events it reacts to, and its ack subject on the stream they arrive from
|
||||
// The events it reacts to, and its ack subject on the stream they arrive from
|
||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||
// controller a subscriber to every event in the mesh, and its permission list would stop
|
||||
// saying what it is for. The ack grant below is scoped per stream because the controller's
|
||||
// saying what it is for. The one wildcard is the emitter of a provider's standing (ADR
|
||||
// 0224) — still two named events, from whichever module provides. The ack grant below is scoped per stream because the controller's
|
||||
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
|
||||
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
||||
// ever, refused by the list it already has.
|
||||
@@ -279,6 +316,18 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// enrolments and can reach nothing else.
|
||||
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
||||
|
||||
// **And its own buckets** (novox/hq to-be 45 §1): the calls it served and the acts done by
|
||||
// hand, which it alone writes. A put is a publish to the bucket's subject, which `$JS.API.>`
|
||||
// does not cover; each bucket named, not `$KV.>`, which would let it write any module's state.
|
||||
for _, bucket := range ControllerBuckets() {
|
||||
pub = append(pub, "$KV."+bucket+".>")
|
||||
}
|
||||
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
|
||||
// consumer that gave up on a message, or one that was deleted. The server already publishes
|
||||
// both in the mesh's own account; the controller says each as a condition in the mesh's words.
|
||||
// Named, not `$JS.EVENT.>`: the other advisories are every API call the mesh makes.
|
||||
sub = append(sub, BusAdvisories...)
|
||||
|
||||
case KindPerson:
|
||||
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||
// who could publish an event would be able to claim a module said something.
|
||||
@@ -490,6 +539,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// that varies is the module, so the pattern is the machine's own assignments.
|
||||
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
||||
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
||||
// And that it is there (novox/hq to-be 45 §3, S11): its own heartbeat, under its machine's
|
||||
// name and no other's, on core NATS like the host's.
|
||||
pub = append(pub, "mesh.control."+p.Node+".tools-alive")
|
||||
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
||||
// node, as the console already could — the runtime is the console's serving mode.
|
||||
invoked, err := invokedSubjects([]string{"*"})
|
||||
@@ -557,6 +609,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
|
||||
sort.Strings(pub)
|
||||
sort.Strings(sub)
|
||||
// One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is
|
||||
// refused here, at composition, naming the state and its writer.
|
||||
if err := CheckWriters(p, pub); err != nil {
|
||||
return Permissions{}, err
|
||||
}
|
||||
return Permissions{
|
||||
Publish: pub,
|
||||
Subscribe: sub,
|
||||
@@ -777,7 +834,7 @@ func ComposeAccounts(principals []Principal) (string, error) {
|
||||
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
|
||||
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
|
||||
if perms.AllowResponses {
|
||||
b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n")
|
||||
fmt.Fprintf(&b, " allow_responses: { max: 1, ttl: \"%dm\" }\n", int(ResponseTTL/time.Minute))
|
||||
}
|
||||
b.WriteString(" } }\n")
|
||||
}
|
||||
|
||||
@@ -5,16 +5,16 @@ import "testing"
|
||||
// A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
|
||||
// node-scoped seat derive two addresses, and a user of the seat may publish any node's.
|
||||
func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
|
||||
seat := Seat{Name: "node-hosts-file", Scope: "node", Serves: []string{"entries"}}
|
||||
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hosts", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hosts", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries.one")
|
||||
has(t, two.Subscribe, "mesh.seat.node-hosts-file.tool.entries.two")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries.two")
|
||||
seat := Seat{Name: "node-hostname", Scope: "node", Serves: []string{"entries"}}
|
||||
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.one")
|
||||
has(t, two.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
|
||||
|
||||
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, user.Publish, "mesh.seat.node-hosts-file.tool.entries.*")
|
||||
has(t, user.Publish, "mesh.seat.node-hostname.tool.entries.*")
|
||||
}
|
||||
|
||||
// A mesh-scoped seat's tool stays flat: nothing about it changes.
|
||||
|
||||
@@ -160,8 +160,9 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err
|
||||
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
||||
}
|
||||
for _, n := range names {
|
||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
|
||||
if !declared[n] && !IsCancelledSet(n) {
|
||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state; so are
|
||||
// the controller's own buckets (novox/hq to-be 45 §1).
|
||||
if !declared[n] && !IsCancelledSet(n) && !IsControllerBucket(n) {
|
||||
undeclared = append(undeclared, n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -20,12 +21,17 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
||||
t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q",
|
||||
broker.ControllerSeat, link.MeshControllerSeat)
|
||||
}
|
||||
for _, event := range []string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore} {
|
||||
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||
states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...)
|
||||
states = append(states, conditions.HeartbeatEvent)
|
||||
// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
states = append(states, link.KeySecretReplaced)
|
||||
for _, event := range states {
|
||||
if !slices.Contains(broker.ControllerStates, event) {
|
||||
t.Errorf("the mesh states %q and its account may not publish it", event)
|
||||
}
|
||||
}
|
||||
if len(broker.ControllerStates) != 3 {
|
||||
if len(broker.ControllerStates) != len(states) {
|
||||
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
||||
}
|
||||
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
||||
|
||||
@@ -201,7 +201,25 @@ const ControllerName = "controller"
|
||||
// something to say.
|
||||
const ControllerSeat = "mesh-controller"
|
||||
|
||||
var ControllerStates = []string{"applied", "refused", "built-before"}
|
||||
var ControllerStates = []string{"applied", "refused", "built-before",
|
||||
// What is wrong, said as it changes (novox/hq to-be 45 §2): a condition raised, changed in
|
||||
// severity, resolver or silence, and cleared. The operator-channel's holder and any other surface
|
||||
// consume them; the controller tells nobody itself.
|
||||
"condition-raised", "condition-changed", "condition-cleared",
|
||||
// And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a
|
||||
// machine that is not the control node, so the controller going quiet is itself said.
|
||||
"doctor-heartbeat",
|
||||
// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
"secret-replaced"}
|
||||
|
||||
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
||||
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
||||
// may and gave up on it, and one that was deleted. Read-only: an advisory is the server's to
|
||||
// publish, and the controller's subscription changes nothing on the bus.
|
||||
var BusAdvisories = []string{
|
||||
"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>",
|
||||
"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>",
|
||||
}
|
||||
|
||||
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
||||
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
||||
@@ -226,8 +244,23 @@ var ControllerFollows = []string{
|
||||
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
|
||||
// with the retired seat row.
|
||||
seatEventSubject("mesh-build-machine", "built"),
|
||||
// **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and
|
||||
// that consumer recovered. The one pattern on this list, and a narrow one — two named events,
|
||||
// from whichever module provides — because the rule is about every provider, and a list of
|
||||
// providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider
|
||||
// failed every consumer for a day and only its journal said so (issue 179). Appended, because
|
||||
// the index is a name.
|
||||
moduleEventSubject("*", ProvisionerFailing),
|
||||
moduleEventSubject("*", ProvisionerRecovered),
|
||||
}
|
||||
|
||||
// The provider standing events, by their local names. Written here as well as in the catalogue
|
||||
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
|
||||
func moduleEventSubject(module, event string) string {
|
||||
@@ -271,7 +304,7 @@ func MeshConsumers() []Consumer {
|
||||
// client and come back to be acted on again.
|
||||
MaxAckPending: 1,
|
||||
FromNow: true,
|
||||
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
|
||||
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
||||
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
||||
"become actionable",
|
||||
},
|
||||
|
||||
+2
-2
@@ -24,8 +24,8 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The writers table (novox/hq to-be 45 §1, ADR 0227 rule 1), compiled in.
|
||||
//
|
||||
// **Every kind of state the core keeps has one writer; anyone else asks it.** The table is the design's,
|
||||
// row for row, with what each row writes on the bus where it writes there. Two things read it: the
|
||||
// composition of every principal's grants (PermissionsFor), which **refuses a grant that lets a
|
||||
// principal publish on a subject another writes** — so a second writer cannot be granted by accident,
|
||||
// and the composition says which state and whose — and the test beside it, which walks the rows
|
||||
// against the design's list and the composition of a whole mesh. Changing a writer is a change to
|
||||
// this table, through a decision.
|
||||
|
||||
// WriterRow is one row of the writers table.
|
||||
type WriterRow struct {
|
||||
State string
|
||||
Writer string
|
||||
KeptIn string
|
||||
Others string
|
||||
// Subjects are the bus subjects a write of this state is a publish to; none for state kept off the
|
||||
// bus (the controller's store, a module's own) or not built yet.
|
||||
Subjects []string
|
||||
// Writes says a principal granted a publish pattern overlapping Subjects is this state's writer —
|
||||
// given the pattern, because a machine writes its own report and no other's.
|
||||
Writes func(p Principal, pattern string) bool
|
||||
// Shared says why more than one principal may publish here; empty for one writer.
|
||||
Shared string
|
||||
}
|
||||
|
||||
// isController, and the other writers' tests, are who a row's writer is as a principal.
|
||||
func isController(p Principal, _ string) bool { return p.Kind == KindController }
|
||||
|
||||
// ownMachine is a node writing a subject whose third token is its own name, and no wildcard there.
|
||||
func ownMachine(p Principal, pattern string) bool {
|
||||
tokens := strings.Split(pattern, ".")
|
||||
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
|
||||
}
|
||||
|
||||
// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
|
||||
// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
|
||||
func holdsSeatOf(p Principal, pattern string) bool {
|
||||
tokens := strings.Split(pattern, ".")
|
||||
if len(tokens) < 3 {
|
||||
return false
|
||||
}
|
||||
seat := tokens[2]
|
||||
holds := func(seats []Seat) bool {
|
||||
return slices.ContainsFunc(seats, func(s Seat) bool { return s.Name == seat })
|
||||
}
|
||||
switch p.Kind {
|
||||
case KindModule:
|
||||
return holds(p.Holds)
|
||||
case KindNodeTools:
|
||||
return slices.ContainsFunc(p.Carries, func(d Declared) bool { return holds(d.Holds) })
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ownModule is a module publishing under its own name, or the node tools carrying it.
|
||||
func ownModule(p Principal, pattern string) bool {
|
||||
tokens := strings.Split(pattern, ".")
|
||||
if len(tokens) < 3 {
|
||||
return false
|
||||
}
|
||||
module := tokens[2]
|
||||
switch p.Kind {
|
||||
case KindModule:
|
||||
return p.Module == module
|
||||
case KindNodeTools:
|
||||
return slices.ContainsFunc(p.Carries, func(d Declared) bool { return d.Module == module })
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// kvOf is a bucket's write subjects.
|
||||
func kvOf(bucket string) []string { return []string{"$KV." + bucket + ".>"} }
|
||||
|
||||
// WritersTable is to-be 45 §1, in its order.
|
||||
var WritersTable = []WriterRow{
|
||||
{State: "a machine's declaration", Writer: "controller (lease holder)", KeptIn: "the bus, last per subject",
|
||||
Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
|
||||
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
|
||||
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
|
||||
Subjects: []string{"mesh.control.*.report"}, Writes: ownMachine},
|
||||
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
||||
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
||||
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
||||
KeptIn: "the controller's store", Others: "read through plans"},
|
||||
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
|
||||
ConditionHistoryBucket + ")", Others: "raise or clear only through observations the controller reads",
|
||||
Subjects: append(kvOf(ConditionsBucket), kvOf(ConditionHistoryBucket)...), Writes: isController},
|
||||
{State: "calls and their outcomes", Writer: "controller", KeptIn: "key-value " + CallsBucket,
|
||||
Others: "read by id", Subjects: kvOf(CallsBucket), Writes: isController},
|
||||
{State: "the hand-act log", Writer: "controller, through the verbs that act", KeptIn: "key-value " + HandActsBucket,
|
||||
Others: "—", Subjects: kvOf(HandActsBucket), Writes: isController},
|
||||
{State: "stream definitions and bus permissions", Writer: "controller", KeptIn: "the bus", Others: "—",
|
||||
// A stream's definition, and a durable consumer's by the API that names it so. Not every
|
||||
// consumer create: a module watching its own bucket makes and deletes an ordered consumer on the
|
||||
// bucket's stream (ADR 0201), which defines nothing the mesh keeps.
|
||||
Subjects: []string{"$JS.API.STREAM.CREATE.>", "$JS.API.STREAM.UPDATE.>", "$JS.API.STREAM.DELETE.>",
|
||||
"$JS.API.CONSUMER.DURABLE.CREATE.>"},
|
||||
Writes: isController},
|
||||
{State: "builds and their outcomes", Writer: "the build seat's holder", KeptIn: "its own state",
|
||||
Others: "the controller asks",
|
||||
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
|
||||
Writes: holdsSeatOf,
|
||||
Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"},
|
||||
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
|
||||
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
|
||||
{State: "a provider's standing", Writer: "the provider", KeptIn: "the provider's events",
|
||||
Others: "the controller keeps the newest word as a condition",
|
||||
Subjects: []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"},
|
||||
Writes: ownModule},
|
||||
{State: "the operator-channel's open messages", Writer: "the seat's holder", KeptIn: "its own key-value state",
|
||||
Others: "—"},
|
||||
{State: "the facts snapshot", Writer: "controller", KeptIn: "the artifact store, facts/latest",
|
||||
Others: "the build seat reads"},
|
||||
}
|
||||
|
||||
// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
|
||||
// another writer (to-be 45 §1): which state, whose, and the pattern that would make a second writer.
|
||||
func CheckWriters(p Principal, publish []string) error {
|
||||
var problems []string
|
||||
for _, pattern := range publish {
|
||||
for _, row := range WritersTable {
|
||||
if row.Writes == nil {
|
||||
continue
|
||||
}
|
||||
for _, subject := range row.Subjects {
|
||||
if !SubjectsOverlap(pattern, subject) || row.Writes(p, pattern) {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, fmt.Sprintf("%s may publish %s, which writes %s (%s), whose writer is %s",
|
||||
p.Username(), pattern, row.State, subject, row.Writer))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("a second writer would be granted (novox/hq to-be 45 §1, one writer per piece of state):\n %s",
|
||||
strings.Join(problems, "\n "))
|
||||
}
|
||||
|
||||
// SubjectsOverlap says some subject matches both patterns: `*` is one token, `>` one or more to the end.
|
||||
func SubjectsOverlap(a, b string) bool {
|
||||
x, y := strings.Split(a, "."), strings.Split(b, ".")
|
||||
for i := 0; ; i++ {
|
||||
switch {
|
||||
case i == len(x) && i == len(y):
|
||||
return true
|
||||
case i == len(x) || i == len(y):
|
||||
return false
|
||||
case x[i] == ">" || y[i] == ">":
|
||||
return true
|
||||
case x[i] == "*" || y[i] == "*" || x[i] == y[i]:
|
||||
continue
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The writers table, checked (novox/hq to-be 45 §1, ADR 0227 rule 1, "how it is checked"): it is the
|
||||
// design's table row for row; every row that writes on the bus names its writer; a whole mesh composes
|
||||
// with one writer per piece of state; and a grant that would make a second writer is refused, naming
|
||||
// the state and whose it is.
|
||||
|
||||
// designRows are the states of to-be 45 §1, in its order. A row added to the design is added here and
|
||||
// to the table; one dropped from either fails.
|
||||
var designRows = []string{
|
||||
"a machine's declaration",
|
||||
"a machine's applied state and its report",
|
||||
"the controller lease",
|
||||
"plans and their tiers",
|
||||
"conditions",
|
||||
"calls and their outcomes",
|
||||
"the hand-act log",
|
||||
"stream definitions and bus permissions",
|
||||
"builds and their outcomes",
|
||||
"a merge announced",
|
||||
"a provider's standing",
|
||||
"the operator-channel's open messages",
|
||||
"the facts snapshot",
|
||||
}
|
||||
|
||||
func TestTheWritersTableIsTheDesigns(t *testing.T) {
|
||||
var states []string
|
||||
for _, row := range WritersTable {
|
||||
states = append(states, row.State)
|
||||
if row.Writer == "" || row.KeptIn == "" || row.Others == "" {
|
||||
t.Errorf("%q does not say who writes it, where it is kept and what others do", row.State)
|
||||
}
|
||||
if len(row.Subjects) > 0 && row.Writes == nil {
|
||||
t.Errorf("%q is written on the bus and names no writer among the principals", row.State)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(states, designRows) {
|
||||
t.Fatalf("the writers table is not to-be 45 §1's:\n have %q\n want %q", states, designRows)
|
||||
}
|
||||
}
|
||||
|
||||
// A mesh of every kind of principal composes: nobody is granted a second writer's subject.
|
||||
func TestAWholeMeshComposesWithOneWriterPerState(t *testing.T) {
|
||||
builder := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||
principals := []Principal{
|
||||
{Kind: KindController, PasswordHash: "x"},
|
||||
{Kind: KindNode, Node: "one", PasswordHash: "x"},
|
||||
{Kind: KindEnrolment, Node: "two", PasswordHash: "x"},
|
||||
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "postgres", Emits: []string{"provisioner.failing", "provisioner.recovered"},
|
||||
PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"},
|
||||
{Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{
|
||||
{Module: "gitea", Emits: []string{"pull.merged"}},
|
||||
{Module: "build-agent", Holds: []Seat{builder}}}},
|
||||
}
|
||||
for _, p := range principals {
|
||||
if _, err := PermissionsFor(p); err != nil {
|
||||
t.Errorf("%s does not compose: %v", p.Username(), err)
|
||||
}
|
||||
}
|
||||
if _, err := ComposeAccounts(principals); err != nil {
|
||||
t.Fatalf("the mesh does not compose: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondWriterIsRefusedAtComposition(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
p Principal
|
||||
publish []string
|
||||
state string
|
||||
}{
|
||||
{"the controller publishing what machines say", Principal{Kind: KindController},
|
||||
[]string{"mesh.control.>"}, "a machine's applied state and its report"},
|
||||
{"a machine publishing another's report", Principal{Kind: KindNode, Node: "one"},
|
||||
[]string{"mesh.control.two.report"}, "a machine's applied state and its report"},
|
||||
{"a machine publishing every machine's", Principal{Kind: KindNode, Node: "one"},
|
||||
[]string{"mesh.control.*.>"}, "a machine's applied state and its report"},
|
||||
{"a module writing the lease", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"$KV.mesh-controller_lease.>"}, "the controller lease"},
|
||||
{"a module sending a declaration", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"mesh.node.one.declare"}, "a machine's declaration"},
|
||||
{"a module defining a stream", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"$JS.API.>"}, "stream definitions and bus permissions"},
|
||||
{"a module announcing another forge's merge", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"mesh.mod.gitea.event.pull.merged"}, "a merge announced"},
|
||||
{"a module saying a build it did not do", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"mesh.seat.node-build-agent.event.built"}, "builds and their outcomes"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
err := CheckWriters(c.p, c.publish)
|
||||
if err == nil {
|
||||
t.Fatalf("%v granted to %s was not refused", c.publish, c.p.Username())
|
||||
}
|
||||
if !strings.Contains(err.Error(), c.state) {
|
||||
t.Fatalf("the refusal does not name %q: %v", c.state, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// And the writers themselves are not refused.
|
||||
for _, ok := range []struct {
|
||||
p Principal
|
||||
publish []string
|
||||
}{
|
||||
{Principal{Kind: KindNode, Node: "one"}, []string{"mesh.control.one.>"}},
|
||||
{Principal{Kind: KindController}, []string{"mesh.node.>", "$JS.API.>", "$KV.mesh-controller_lease.>"}},
|
||||
{Principal{Kind: KindModule, Module: "gitea"}, []string{"mesh.mod.gitea.event.pull.merged"}},
|
||||
{Principal{Kind: KindModule, Module: "shop"}, []string{"$JS.API.CONSUMER.CREATE.KV_shop_carts.>"}},
|
||||
} {
|
||||
if err := CheckWriters(ok.p, ok.publish); err != nil {
|
||||
t.Errorf("a writer was refused its own: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubjectsOverlap(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
a, b string
|
||||
want bool
|
||||
}{
|
||||
{"mesh.control.>", "mesh.control.*.report", true},
|
||||
{"mesh.control.one.>", "mesh.control.*.report", true},
|
||||
{"mesh.control.one.alive", "mesh.control.*.report", false},
|
||||
{"mesh.control.*", "mesh.control.*.report", false},
|
||||
{"$JS.API.>", "$JS.API.STREAM.CREATE.>", true},
|
||||
{"$JS.API.CONSUMER.CREATE.KV_x.>", "$JS.API.STREAM.CREATE.>", false},
|
||||
{"a.b", "a.b", true},
|
||||
{"a.b", "a.b.c", false},
|
||||
{"a.>", "a", false},
|
||||
} {
|
||||
if got := SubjectsOverlap(c.a, c.b); got != c.want || SubjectsOverlap(c.b, c.a) != c.want {
|
||||
t.Errorf("%s ~ %s: %v, want %v", c.a, c.b, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -119,3 +119,41 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
|
||||
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEveryCatalogueIdentityFitsWhatItRequires is ADR 0225's check over the real catalogue: every
|
||||
// module's identity, on the longest machine name, fits the bound of every provision it wants. An
|
||||
// overflow fails here, in the pull request that introduces it, rather than on the provider's machine
|
||||
// the first time a real machine's name meets the module's (issue 263).
|
||||
func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
|
||||
root := catalogueRoot(t)
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil || len(found) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
shelf := Shelf{}
|
||||
for _, p := range found {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for _, p := range IdentityProblems(shelf, DefaultLongestMachine) {
|
||||
t.Error(p)
|
||||
}
|
||||
// The night it was found: the resolver provision bounds nothing, and the object store still 20.
|
||||
if dns, ok := shelf["dnsmasq"]; ok {
|
||||
if b := dns.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||
t.Errorf("the resolver provision bounds its consumers' identities: %+v", b)
|
||||
}
|
||||
}
|
||||
if store, ok := shelf["minio"]; ok {
|
||||
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 {
|
||||
t.Errorf("the object store's access key is not bounded at 20: %+v", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,7 +114,9 @@ func consumerInto(value, as string) (string, error) {
|
||||
// asDNSLabel writes a minted identity as a DNS label.
|
||||
//
|
||||
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
|
||||
// already short enough for the tightest backend they reach (CheckIdentity, twenty characters). So
|
||||
// already inside the bound of the provision serving them: a provider that serves one as a DNS label
|
||||
// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without
|
||||
// saying is held to twenty (IdentityBoundOf). So
|
||||
// this is the separator and nothing else — no lower-casing of what is already lower case, no
|
||||
// truncation to a limit the identity is already inside, no padding of a name that is already long
|
||||
// enough. Each of those would be the mesh guessing at a rule it has not been given.
|
||||
@@ -141,11 +143,31 @@ func CheckServes(m Manifest) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
||||
}
|
||||
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
|
||||
// bound has to keep the identity inside one (ADR 0225).
|
||||
if strings.Contains(text, "${consumer:as:dns}") {
|
||||
if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s's consumers their identity as a DNS label in %q, and bounds "+
|
||||
"that identity at %s: a label keeps %d — state an `identity` of at most %d",
|
||||
m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4).
|
||||
const dnsLabelLimit = 63
|
||||
|
||||
func boundWords(b IdentityBound) string {
|
||||
if !b.Bounded() {
|
||||
return "nothing"
|
||||
}
|
||||
return fmt.Sprintf("%d", b.Max)
|
||||
}
|
||||
|
||||
func sortedServes(serves map[string]map[string]any) []string {
|
||||
out := make([]string, 0, len(serves))
|
||||
for k := range serves {
|
||||
|
||||
@@ -159,12 +159,21 @@ type Rendering struct {
|
||||
// 0199): the mesh's resolver forwards each one there.
|
||||
Zones []ZoneAt
|
||||
|
||||
// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
|
||||
// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
|
||||
// file lists: every holder of the mesh's resolver, this machine first if it is one.
|
||||
Holders map[string]map[string]string
|
||||
|
||||
Settings SettingsBy
|
||||
Generators map[string]Generator
|
||||
// Grants are the credentials this node must create, for the provisions it offers. Passed in
|
||||
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
|
||||
// resolution answers questions about one machine.
|
||||
Grants []Grant
|
||||
// Withheld is every consumer left out of Grants because its identity overflows the provision's
|
||||
// bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say
|
||||
// whom it does not serve, and why, beside what it does.
|
||||
Withheld []Overflow
|
||||
|
||||
// Ports is where this machine puts what each module needs reachable, by module and by the
|
||||
// port the software itself uses (novox/hq ADR 0038).
|
||||
@@ -980,7 +989,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
||||
// this module's name, so they are applied, reported and removed exactly as anything else
|
||||
// it declares.
|
||||
given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
|
||||
given, err := FactsFrom(m, r, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -168,7 +168,7 @@ func placedManifest(m Manifest, with Rendering) (Manifest, error) {
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
own[name] = OwnSecret{Path: filled, Taken: s.Taken}
|
||||
own[name] = OwnSecret{Path: filled, Taken: s.Taken, IssuedBy: s.IssuedBy}
|
||||
}
|
||||
m.OwnSecrets = own
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package catalogue
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -159,3 +160,34 @@ func consumePattern(pattern string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
|
||||
// without one success for minutes, and that consumer succeeding again or being withdrawn. The
|
||||
// controller follows them from every module and `status` names a consumer failing until it recovers.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// ProvisionerEvents are both, in the order they are said.
|
||||
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
|
||||
|
||||
// EmitsAll is every event a module may publish: what it declares and, for a module that receives
|
||||
// contributions — a provider, running a provisioner over them — the provider's standing events.
|
||||
//
|
||||
// **Derived, not declared**, because they are the mesh's rule about every provider rather than
|
||||
// anything one module chose to say: a provider whose manifest forgot them would fail its consumers
|
||||
// as silently as on 2026-10-05, with its announcement refused by the bus (novox/hq issue 179). Every
|
||||
// grant of a module's publishing reads this, never the declared list alone.
|
||||
func (m Manifest) EmitsAll() []string {
|
||||
out := append([]string(nil), m.Emits...)
|
||||
if len(m.Receives) == 0 {
|
||||
return out
|
||||
}
|
||||
for _, e := range ProvisionerEvents {
|
||||
if !slices.Contains(out, e) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0223 part 3: a machine's names are one seat's. The `hosts` module holding
|
||||
// `node-hosts-file` became `hostname` holding `node-hostname`, which writes /etc/hostname beside the
|
||||
// machine's own lines in /etc/hosts. The seat was renamed (ADR 0122), so what claims the old name — a
|
||||
// manifest registered before the rename — still holds the one seat.
|
||||
|
||||
func withHostnameAlias(t *testing.T) {
|
||||
t.Helper()
|
||||
was := aliases
|
||||
t.Cleanup(func() { aliases = was })
|
||||
UseAliases(map[string]string{"node-hosts-file": "node-hostname"})
|
||||
}
|
||||
|
||||
func TestTheHostsFilesFormerNameResolvesToTheHostnameSeat(t *testing.T) {
|
||||
if _, known := SeatNamed("node-hostname"); !known {
|
||||
t.Fatal("node-hostname is not in the mesh's set")
|
||||
}
|
||||
withHostnameAlias(t)
|
||||
seat, known := SeatNamed("node-hosts-file")
|
||||
if !known || seat.Name != "node-hostname" || seat.Scope != ScopeNode {
|
||||
t.Fatalf("the former name did not resolve: %+v %v", seat, known)
|
||||
}
|
||||
var verbs []string
|
||||
for _, v := range seat.Serves {
|
||||
verbs = append(verbs, v.Name)
|
||||
}
|
||||
if strings.Join(verbs, " ") != "entries add remove" {
|
||||
t.Errorf("the renamed seat serves %v; its verbs are unchanged", verbs)
|
||||
}
|
||||
}
|
||||
|
||||
// One seat under either name: the module registered before the rename and the one after cannot both
|
||||
// hold it on one machine.
|
||||
func TestTheOldAndTheNewClaimantAreOneSeatOnAMachine(t *testing.T) {
|
||||
withHostnameAlias(t)
|
||||
cat := shelf(
|
||||
mod("hosts", nil, nil, nil, Claim{Name: "node-hosts-file"}),
|
||||
mod("hostname", nil, nil, nil, Claim{Name: "node-hostname"}),
|
||||
)
|
||||
_, err := Resolve(cat, []string{"hosts", "hostname"}, workstation(), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "node-hostname") {
|
||||
t.Errorf("hosts and hostname both held the machine's names on one machine: %v", err)
|
||||
}
|
||||
if _, err := Resolve(cat, []string{"hosts"}, workstation(), World{}); err != nil {
|
||||
t.Errorf("a machine still assigned hosts under the old name does not resolve: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's module: /etc/hostname is the operator's `hostname` setting. Without it the module is
|
||||
// left out, naming the key — the mesh never renames a machine on its own — and a mesh-wide setting
|
||||
// naming ${machine:name} gives every machine its mesh name.
|
||||
func TestTheMachinesNameIsItsSetting(t *testing.T) {
|
||||
cat := map[string]Manifest{"hostname": catalogueManifest(t, "hostname")}
|
||||
got, err := Resolve(cat, []string{"hostname"}, Node{Name: "ace", At: "ace.internal"}, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
machines := map[string]string{"ace.internal": "10.42.0.2"}
|
||||
nameOf := func(settings SettingsBy) (string, string) {
|
||||
t.Helper()
|
||||
composed, err := got.Compose(Rendering{Names: machines, Machines: machines, Suffix: "internal",
|
||||
Settings: settings})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := composed.LeftOut["hostname"]; why != "" {
|
||||
return "", why
|
||||
}
|
||||
for _, r := range composed.Resources {
|
||||
if r["path"] == "/etc/hostname" {
|
||||
return r["content"].(string), ""
|
||||
}
|
||||
}
|
||||
t.Fatal("no /etc/hostname composed")
|
||||
return "", ""
|
||||
}
|
||||
|
||||
if _, why := nameOf(nil); !strings.Contains(why, "hostname") {
|
||||
t.Errorf("with no setting the machine's name was written, or left out for another reason: %q", why)
|
||||
}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {{From: "ace", Values: map[string]any{"hostname": "Ace"}}}}); name != "Ace\n" {
|
||||
t.Errorf("the operator's name for the machine gave %q (%s)", name, why)
|
||||
}
|
||||
mesh := Layer{From: "the mesh", Values: map[string]any{"hostname": "${machine:name}"}}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {mesh}}); name != "ace\n" {
|
||||
t.Errorf("a mesh-wide ${machine:name} gave %q (%s)", name, why)
|
||||
}
|
||||
node := Layer{From: "ace", Values: map[string]any{"hostname": "Ace"}}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {mesh, node}}); name != "Ace\n" {
|
||||
t.Errorf("a machine's own name over the mesh-wide one gave %q (%s)", name, why)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package catalogue
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -56,13 +57,36 @@ func ConsumerIdentity(node, module string) string {
|
||||
return IdentityPrefix + clean(node) + "_" + clean(module)
|
||||
}
|
||||
|
||||
// identityLimit is the shortest identifier limit among the systems these names reach: an S3 access
|
||||
// key's 20 (novox/hq 04-ISSUES/010). PostgreSQL keeps 63 and MinIO 20, so 20 is the one that binds —
|
||||
// the comment used to name PostgreSQL and was wrong. A name over it is refused, with the remedy a
|
||||
// short slug (ADR 0049), not silently cut to fit.
|
||||
const identityLimit = 20
|
||||
// IdentityBound is the longest consumer identity one provision's backend keeps, and what keeps it
|
||||
// (novox/hq ADR 0049, refined by ADR 0225). Max zero means no bound: the provision keeps no name
|
||||
// derived from its consumer, or keeps one in something with no limit the mesh need respect.
|
||||
type IdentityBound struct {
|
||||
// Max is the longest identity that backend keeps, in characters; zero for none.
|
||||
Max int `json:"max,omitempty"`
|
||||
// In is what keeps it, in words a refusal can quote: "an S3 access key", "a PostgreSQL role".
|
||||
In string `json:"in,omitempty"`
|
||||
}
|
||||
|
||||
// CheckIdentity refuses an identity that would not fit the tightest backend a consumer reaches.
|
||||
// Bounded is whether this bound refuses anything.
|
||||
func (b IdentityBound) Bounded() bool { return b.Max > 0 }
|
||||
|
||||
// DefaultIdentityLimit is the bound on a provision whose provider receives its consumers and does
|
||||
// not say how long a name it keeps: an S3 access key's 20 (novox/hq 04-ISSUES/010, 034), the
|
||||
// tightest backend the mesh has met. It was the bound on every provision until ADR 0225; it stays
|
||||
// the bound on any that has not said otherwise, because a provider that is told each consumer's
|
||||
// identity may create a name from it in a backend nobody has measured.
|
||||
const DefaultIdentityLimit = 20
|
||||
|
||||
// DefaultIdentityBound is DefaultIdentityLimit, said as a bound.
|
||||
var DefaultIdentityBound = IdentityBound{Max: DefaultIdentityLimit,
|
||||
In: "a backend that has not said its limit (the tightest known, an S3 access key's)"}
|
||||
|
||||
// identityLimit is the bound CheckIdentity applies, for a caller that does not know which provision
|
||||
// the identity is for.
|
||||
const identityLimit = DefaultIdentityLimit
|
||||
|
||||
// CheckIdentity refuses an identity that would not fit the tightest backend the mesh knows. A caller
|
||||
// that knows the provision uses CheckIdentityWithin and that provision's own bound (ADR 0225).
|
||||
//
|
||||
// **Truncation is not an error in most of these systems** — a name past the limit is cut to fit and
|
||||
// the statement succeeds, so two consumers agreeing for the first N bytes would become one login
|
||||
@@ -70,12 +94,127 @@ const identityLimit = 20
|
||||
// name and is the only thing that can choose another. The remedy is a first-class one: give the
|
||||
// module a short `slug` (ADR 0049), or shorten the machine's name.
|
||||
func CheckIdentity(node, module string) error {
|
||||
return CheckIdentityWithin(node, module, IdentityBound{Max: identityLimit, In: "a backend (an S3 access key)"})
|
||||
}
|
||||
|
||||
// CheckIdentityWithin refuses an identity that would not fit one provision's bound, and accepts any
|
||||
// identity for a provision with none (novox/hq ADR 0225).
|
||||
func CheckIdentityWithin(node, module string, bound IdentityBound) error {
|
||||
if !bound.Bounded() {
|
||||
return nil
|
||||
}
|
||||
got := ConsumerIdentity(node, module)
|
||||
if len(got) <= identityLimit {
|
||||
if len(got) <= bound.Max {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"%s on %s is identified as %q, %d characters where a backend (an S3 access key) keeps %d — "+
|
||||
"%s on %s is identified as %q, %d characters where %s keeps %d — "+
|
||||
"give the module a shorter `slug` or shorten the machine's name",
|
||||
module, node, got, len(got), identityLimit)
|
||||
module, node, got, len(got), bound.In, bound.Max)
|
||||
}
|
||||
|
||||
// Overflow is one consumer whose identity does not fit the provision it requires: left out of its
|
||||
// provider's grants and reported, never a reason to refuse the provider's machine (ADR 0225).
|
||||
type Overflow struct {
|
||||
// Provision is what was required, Provider the machine answering it.
|
||||
Provision string `json:"provision"`
|
||||
Provider string `json:"provider"`
|
||||
// Consumer is the machine, Module the module on it that required it.
|
||||
Consumer string `json:"consumer"`
|
||||
Module string `json:"module"`
|
||||
// Identity is the name the mesh derived, and Bound what it overflows.
|
||||
Identity string `json:"identity"`
|
||||
Bound IdentityBound `json:"bound"`
|
||||
}
|
||||
|
||||
func (o Overflow) String() string {
|
||||
return fmt.Sprintf("%s on %s requires %s from %s and is identified as %q, %d characters where %s "+
|
||||
"keeps %d — left out of %s's grants until the module's `slug` is shorter",
|
||||
o.Module, o.Consumer, o.Provision, o.Provider, o.Identity, len(o.Identity), o.Bound.In,
|
||||
o.Bound.Max, o.Provider)
|
||||
}
|
||||
|
||||
// Overflowing is every requirement of this machine's modules whose identity overflows the bound of
|
||||
// the provision answering it. The same judgement the provider's composition makes before it grants
|
||||
// (grantsFor), made from the consumer's side so `status` can say it about every machine.
|
||||
func (r Resolution) Overflowing() []Overflow {
|
||||
slugs := map[string]string{}
|
||||
for _, m := range r.Modules {
|
||||
slugs[m.Module] = m.Slug
|
||||
}
|
||||
var out []Overflow
|
||||
seen := map[string]bool{}
|
||||
for _, n := range r.Needs {
|
||||
if n.ByRecord || !n.Identity.Bounded() {
|
||||
continue
|
||||
}
|
||||
source := IdentitySource(slugs[n.For], n.For)
|
||||
if CheckIdentityWithin(r.Node, source, n.Identity) == nil {
|
||||
continue
|
||||
}
|
||||
key := n.Name + "\x00" + n.From + "\x00" + n.For
|
||||
if seen[key] {
|
||||
continue // one line per requirement, however many local names it has
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, Overflow{Provision: n.Name, Provider: n.From, Consumer: r.Node, Module: n.For,
|
||||
Identity: ConsumerIdentity(r.Node, source), Bound: n.Identity})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Module != out[j].Module {
|
||||
return out[i].Module < out[j].Module
|
||||
}
|
||||
return out[i].Provision < out[j].Provision
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// DefaultLongestMachine is the machine name the catalogue check judges identities on when it is not
|
||||
// told one: the longest name of the mesh this catalogue is written for, so a catalogue that passes
|
||||
// passes on every machine that mesh has. `module check --longest-machine-name` says another mesh's;
|
||||
// a mesh that names a longer machine raises this in the same change (ADR 0225).
|
||||
const DefaultLongestMachine = 6
|
||||
|
||||
// IdentityProblems is every module whose identity would overflow a provision it wants, on a machine
|
||||
// whose name is `longestMachine` characters — judged before merge, over the catalogue alone, so the
|
||||
// pull request that introduces an overflow is the one refused (novox/hq ADR 0225, issue 263). A
|
||||
// provision no module in the shelf offers is not judged: its bound is not known here.
|
||||
func IdentityProblems(shelf Shelf, longestMachine int) []string {
|
||||
offeredBy := map[string][]string{}
|
||||
for _, name := range shelfOrder(shelf) {
|
||||
for _, o := range shelf[name].Offers() {
|
||||
offeredBy[o] = append(offeredBy[o], name)
|
||||
}
|
||||
}
|
||||
machine := strings.Repeat("n", longestMachine)
|
||||
var problems []string
|
||||
for _, name := range shelfOrder(shelf) {
|
||||
m := shelf[name]
|
||||
source := IdentitySource(m.Slug, m.Module)
|
||||
for _, want := range m.Wants() {
|
||||
// The tightest bound among the modules offering it: whichever one answers on a given
|
||||
// machine, the identity has to fit it.
|
||||
tightest, by := IdentityBound{}, ""
|
||||
for _, provider := range offeredBy[want] {
|
||||
if provider == name {
|
||||
continue // a module answering its own requirement is not its own consumer
|
||||
}
|
||||
b := shelf[provider].IdentityBoundOf(want)
|
||||
if b.Bounded() && (!tightest.Bounded() || b.Max < tightest.Max) {
|
||||
tightest, by = b, provider
|
||||
}
|
||||
}
|
||||
if CheckIdentityWithin(machine, source, tightest) == nil {
|
||||
continue
|
||||
}
|
||||
got := ConsumerIdentity(machine, source)
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s wants %s, and %s keeps its consumers' identities in %s of at most %d characters: "+
|
||||
"on a machine with a %d-character name it is identified as %q, %d — give %s a "+
|
||||
"`slug` of at most %d characters",
|
||||
name, want, by, tightest.In, tightest.Max, longestMachine, got, len(got), name,
|
||||
tightest.Max-len(IdentityPrefix)-longestMachine-1))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Each test names the decision it defends: novox/hq ADR 0225, which refines ADR 0049 after issue 263.
|
||||
|
||||
// The provision a module requires sets the bound on its identity, not the tightest backend anywhere.
|
||||
func TestABoundIsTheProvisionsOwn(t *testing.T) {
|
||||
store := Manifest{Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}
|
||||
database := Manifest{Module: "db", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 63, In: "a PostgreSQL role"}}},
|
||||
Receives: map[string]string{"postgres-database": "/var/lib/mesh/db/mesh.json"}}
|
||||
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 || b.In != "an S3 access key" {
|
||||
t.Errorf("an object store's stated bound was not taken: %+v", b)
|
||||
}
|
||||
if b := database.IdentityBoundOf("postgres-database"); b.Max != 63 {
|
||||
t.Errorf("a database's stated bound was not taken: %+v", b)
|
||||
}
|
||||
// mesh_workstation_keycloak is 25: refused by the object store, accepted by the database.
|
||||
if CheckIdentityWithin("workstation", "keycloak", store.IdentityBoundOf("s3-bucket")) == nil {
|
||||
t.Error("a 25-character identity fit a 20-character access key")
|
||||
}
|
||||
if err := CheckIdentityWithin("workstation", "keycloak", database.IdentityBoundOf("postgres-database")); err != nil {
|
||||
t.Errorf("a database consumer paid the object store's limit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// What an offer leaves unsaid follows from whether its provider can keep a name at all.
|
||||
func TestAnUnstatedBoundFollowsWhatTheProviderIsTold(t *testing.T) {
|
||||
// Told nothing about its consumers — no receives, nothing served from their identity: no bound.
|
||||
// The resolver provision is exactly this, and its consumers paid an object store's limit (263).
|
||||
resolver := Manifest{Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}}
|
||||
if b := resolver.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||
t.Errorf("a provision that is told nothing of its consumers bounds them: %+v", b)
|
||||
}
|
||||
// Told each consumer and silent about its backend: the old global bound, not none.
|
||||
told := Manifest{Module: "told", Provides: []Offer{{Name: "thing", Scope: ScopeMesh}},
|
||||
Receives: map[string]string{"thing": "/var/lib/mesh/told/mesh.json"}}
|
||||
if b := told.IdentityBoundOf("thing"); b.Max != DefaultIdentityLimit {
|
||||
t.Errorf("a provider told its consumers and silent about its backend is not held to %d: %+v",
|
||||
DefaultIdentityLimit, b)
|
||||
}
|
||||
// Serving a value built from the identity is being told it, too.
|
||||
serving := Manifest{Module: "serving", Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"bucket": {"name": "b-${consumer:as:dns}"}}}
|
||||
if b := serving.IdentityBoundOf("bucket"); b.Max != DefaultIdentityLimit {
|
||||
t.Errorf("a provider deriving a name from its consumers is not bounded: %+v", b)
|
||||
}
|
||||
// And `false` says it outright, even for a provider that receives.
|
||||
routes := Manifest{Module: "routes", Provides: []Offer{{Name: "route", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{None: true}}},
|
||||
Receives: map[string]string{"route": "/var/lib/mesh/routes/mesh.json"}}
|
||||
if b := routes.IdentityBoundOf("route"); b.Bounded() {
|
||||
t.Errorf("`identity: false` still bounds: %+v", b)
|
||||
}
|
||||
}
|
||||
|
||||
// The field reads as written and writes back the same, and refuses what says nothing.
|
||||
func TestAnOffersIdentityIsParsedStrictly(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
`{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}`,
|
||||
`{"name":"wildcard-resolution","scope":"mesh","identity":false}`,
|
||||
`{"name":"redis-cache","scope":"mesh","identity":{"in":"a Redis ACL user"}}`,
|
||||
} {
|
||||
var o Offer
|
||||
if err := json.Unmarshal([]byte(raw), &o); err != nil {
|
||||
t.Fatalf("%s: %v", raw, err)
|
||||
}
|
||||
back, err := json.Marshal(o)
|
||||
if err != nil || string(back) != raw {
|
||||
t.Errorf("did not round-trip:\n%s\n%s (%v)", raw, back, err)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{
|
||||
`{"name":"x","identity":true}`,
|
||||
`{"name":"x","identity":{"max":20,"in":"y","most":3}}`,
|
||||
} {
|
||||
var o Offer
|
||||
if err := json.Unmarshal([]byte(raw), &o); err == nil {
|
||||
t.Errorf("accepted %s", raw)
|
||||
}
|
||||
}
|
||||
bad := Manifest{Module: "bad", Version: "1", Provides: []Offer{
|
||||
{Name: "unsaid", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 20}},
|
||||
{Name: "tiny", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 4, In: "nothing usable"}},
|
||||
}}
|
||||
raw, err := json.Marshal(bad)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = ParseManifest(raw)
|
||||
if err == nil || !strings.Contains(err.Error(), "without saying what keeps them") ||
|
||||
!strings.Contains(err.Error(), "the shortest the mesh makes") {
|
||||
t.Fatalf("a bound with no `in`, or too short for any identity, was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Refused before merge: the catalogue check judges each module's identity, on the longest machine
|
||||
// name, against the bound of every provision it wants — and names the module and the slug to set.
|
||||
func TestTheCatalogueCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
shelf := Shelf{
|
||||
"objects": {Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}},
|
||||
"photoalbum": {Module: "photoalbum", Requires: []string{"s3-bucket"}},
|
||||
"files": {Module: "files", Requires: []string{"s3-bucket"}},
|
||||
}
|
||||
problems := IdentityProblems(shelf, 6)
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], "photoalbum wants s3-bucket") ||
|
||||
!strings.Contains(problems[0], `"mesh_nnnnnn_photoalbum", 22`) ||
|
||||
!strings.Contains(problems[0], "`slug` of at most 8 characters") {
|
||||
t.Fatalf("one overflow, named with its remedy, was expected: %q", problems)
|
||||
}
|
||||
// A longer machine name refuses more: the check is about the mesh's machines, not one.
|
||||
if got := IdentityProblems(shelf, 10); len(got) != 2 {
|
||||
t.Fatalf("on a 10-character name both overflow (mesh_nnnnnnnnnn_files is 21): %q", got)
|
||||
}
|
||||
// And a slug is the remedy it names.
|
||||
album := shelf["photoalbum"]
|
||||
album.Slug = "album"
|
||||
shelf["photoalbum"] = album
|
||||
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||
t.Fatalf("a slug that fits is still refused: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Tonight's case (issue 263): networkmanager, no slug, requiring the mesh's resolver provision on a
|
||||
// machine with a six-character name. Under ADR 0049's one bound it was refused, and its provider's
|
||||
// whole machine with it; the resolver keeps no name, so it is not refused at all.
|
||||
func TestARequirementOnAKeylessProvisionComposesWithALongName(t *testing.T) {
|
||||
shelf := Shelf{
|
||||
"resolver": {Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}},
|
||||
"networkmanager": {Module: "networkmanager", Requires: []string{"wildcard-resolution"}},
|
||||
}
|
||||
if CheckIdentity("laptop", "networkmanager") == nil {
|
||||
t.Fatal("the regression is not reproduced: mesh_laptop_networkmanager fits the old global bound")
|
||||
}
|
||||
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||
t.Fatalf("a requirement on a keyless provision was refused for its length: %q", got)
|
||||
}
|
||||
r := Resolution{Node: "laptop", Modules: []Manifest{shelf["networkmanager"]},
|
||||
Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", For: "networkmanager",
|
||||
Identity: shelf["resolver"].IdentityBoundOf("wildcard-resolution")}}}
|
||||
if got := r.Overflowing(); len(got) != 0 {
|
||||
t.Fatalf("a keyless requirement is reported as overflowing: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's side of the same judgement the provider's composition makes: what `status` says.
|
||||
func TestAnOverflowingRequirementIsNamedFromTheConsumersSide(t *testing.T) {
|
||||
bound := IdentityBound{Max: 20, In: "an S3 access key"}
|
||||
r := Resolution{Node: "laptop",
|
||||
Modules: []Manifest{{Module: "photoalbum"}, {Module: "files"}, {Module: "gallery", Slug: "gal"}},
|
||||
Needs: []Needed{
|
||||
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Local: "second", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "files", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "gallery", Identity: bound},
|
||||
{Name: "licence", From: "records", For: "photoalbum", ByRecord: true, Identity: bound},
|
||||
}}
|
||||
got := r.Overflowing()
|
||||
if len(got) != 1 || got[0].Module != "photoalbum" || got[0].Identity != "mesh_laptop_photoalbum" ||
|
||||
got[0].Provider != "anchor" {
|
||||
t.Fatalf("one overflow, once, was expected: %+v", got)
|
||||
}
|
||||
if said := got[0].String(); !strings.Contains(said, "an S3 access key keeps 20") ||
|
||||
!strings.Contains(said, "slug") {
|
||||
t.Fatalf("the overflow does not say what keeps it or the remedy: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// An identity served as a DNS label is never truncated, so the offer's bound must keep it in one.
|
||||
func TestAnIdentityServedAsADNSLabelIsBoundedToOne(t *testing.T) {
|
||||
serves := map[string]map[string]any{"bucket": {"name": "${consumer:as:dns}"}}
|
||||
wide := Manifest{Module: "wide", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 255, In: "a client id"}}}}
|
||||
if got := CheckServes(wide); len(got) != 1 || !strings.Contains(got[0], "a label keeps 63") {
|
||||
t.Fatalf("a 255-character bound on a DNS label passed: %q", got)
|
||||
}
|
||||
unsaid := Manifest{Module: "unsaid", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}}}
|
||||
if got := CheckServes(unsaid); len(got) != 0 {
|
||||
t.Fatalf("the default bound (20) on a DNS label was refused: %q", got)
|
||||
}
|
||||
}
|
||||
+149
-19
@@ -8,6 +8,7 @@ package catalogue
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
@@ -153,6 +154,84 @@ type Offer struct {
|
||||
// a seat's holder gated by the machine's graphical session, and pulling one in for whatever asked
|
||||
// is the misassignment research 026 found. Unmet, the requirement is refused naming who could.
|
||||
Reach string `json:"reach,omitempty"`
|
||||
// Identity is the longest consumer identity this provision's backend keeps (novox/hq ADR 0225):
|
||||
// `{"max": 63, "in": "a PostgreSQL role"}`, or `false` for a provision that keeps no name derived
|
||||
// from its consumer. Unsaid, the mesh assumes the tightest backend it knows when the provider is
|
||||
// told its consumers, and no bound when it is not — see IdentityBoundOf.
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
|
||||
// OfferIdentity is what an offer says about the names its backend keeps for its consumers.
|
||||
type OfferIdentity struct {
|
||||
// None is set by `"identity": false`: the provision keeps no name derived from its consumer.
|
||||
None bool
|
||||
// Max is the longest identity kept, in characters; zero with In set means no limit worth stating.
|
||||
Max int
|
||||
// In is what keeps it, for a refusal to quote.
|
||||
In string
|
||||
}
|
||||
|
||||
// UnmarshalJSON accepts `false` or `{"max": N, "in": "..."}`.
|
||||
func (i *OfferIdentity) UnmarshalJSON(raw []byte) error {
|
||||
var flag bool
|
||||
if err := json.Unmarshal(raw, &flag); err == nil {
|
||||
if flag {
|
||||
return errors.New("an offer's identity is false (it keeps no name) or {max, in}; true says nothing")
|
||||
}
|
||||
*i = OfferIdentity{None: true}
|
||||
return nil
|
||||
}
|
||||
var full struct {
|
||||
Max int `json:"max,omitempty"`
|
||||
In string `json:"in"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("an offer's identity is false or {max, in}: %w", err)
|
||||
}
|
||||
*i = OfferIdentity{Max: full.Max, In: full.In}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON writes it back in the form it was written.
|
||||
func (i OfferIdentity) MarshalJSON() ([]byte, error) {
|
||||
if i.None {
|
||||
return []byte("false"), nil
|
||||
}
|
||||
return json.Marshal(struct {
|
||||
Max int `json:"max,omitempty"`
|
||||
In string `json:"in"`
|
||||
}{i.Max, i.In})
|
||||
}
|
||||
|
||||
// IdentityBoundOf is the bound this module's offer of a provision puts on its consumers' identities
|
||||
// (novox/hq ADR 0225).
|
||||
//
|
||||
// **What an offer says, it gets.** Where it says nothing, the bound follows from whether the
|
||||
// provider can keep a name at all: a provider that receives the provision, or serves its consumers
|
||||
// a value built from their identity, is told who each consumer is and may create a name from it in
|
||||
// a backend nobody measured — so it keeps the old global bound, DefaultIdentityBound. One that does
|
||||
// neither is told nothing about its consumers and keeps nothing of them: no bound. The resolver
|
||||
// provision is that case, and its consumers paid an object store's limit until this (issue 263).
|
||||
func (m Manifest) IdentityBoundOf(provision string) IdentityBound {
|
||||
for _, o := range m.Provides {
|
||||
if o.Name != provision || o.Identity == nil {
|
||||
continue
|
||||
}
|
||||
if o.Identity.None {
|
||||
return IdentityBound{}
|
||||
}
|
||||
return IdentityBound{Max: o.Identity.Max, In: o.Identity.In}
|
||||
}
|
||||
if _, receives := m.Receives[provision]; receives {
|
||||
return DefaultIdentityBound
|
||||
}
|
||||
if served, err := json.Marshal(m.Serves[provision]); err == nil &&
|
||||
bytes.Contains(served, []byte("${consumer:as")) {
|
||||
return DefaultIdentityBound
|
||||
}
|
||||
return IdentityBound{}
|
||||
}
|
||||
|
||||
// MachineReach is whether a provision is usable only on its provider's own machine.
|
||||
@@ -206,20 +285,21 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Credential *OfferCredential `json:"credential,omitempty"`
|
||||
Reach string `json:"reach,omitempty"`
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach}: %w", err)
|
||||
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity}: %w", err)
|
||||
}
|
||||
o.Name, o.Scope, o.Credential, o.Reach = full.Name, full.Scope, full.Credential, full.Reach
|
||||
o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
|
||||
// went through the mesh comes out looking like the one that went in.
|
||||
func (o Offer) MarshalJSON() ([]byte, error) {
|
||||
if o.Scope == "" && o.Credential == nil && o.Reach == "" {
|
||||
if o.Scope == "" && o.Credential == nil && o.Reach == "" && o.Identity == nil {
|
||||
return json.Marshal(o.Name)
|
||||
}
|
||||
return json.Marshal(struct {
|
||||
@@ -227,7 +307,8 @@ func (o Offer) MarshalJSON() ([]byte, error) {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Credential *OfferCredential `json:"credential,omitempty"`
|
||||
Reach string `json:"reach,omitempty"`
|
||||
}{o.Name, o.Scope, o.Credential, o.Reach})
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}{o.Name, o.Scope, o.Credential, o.Reach, o.Identity})
|
||||
}
|
||||
|
||||
// Manifest is everything a module says about itself.
|
||||
@@ -237,9 +318,10 @@ type Manifest struct {
|
||||
|
||||
// Slug is a short identifier the mesh uses in place of the module name when it derives a
|
||||
// consumer's login (novox/hq ADR 0049). Optional: a module with a short name needs none. It
|
||||
// exists because `mesh_<node>_<module>` must fit the tightest backend a consumer reaches — an S3
|
||||
// access key is 20 characters — and a long module name would overflow it. A person choosing
|
||||
// `kc` for keycloak keeps the identity legible where a hash would not.
|
||||
// exists because `mesh_<node>_<module>` must fit the bound of every provision the module
|
||||
// requires — an S3 access key's 20 characters is the tightest — and a long module name would
|
||||
// overflow it (ADR 0225: the bound is the provision's own, and a keyless one has none). A person
|
||||
// choosing `kc` for keycloak keeps the identity legible where a hash would not.
|
||||
Slug string `json:"slug,omitempty"`
|
||||
|
||||
// Provides are the names other modules may require. A module always provides its own name;
|
||||
@@ -476,7 +558,10 @@ type Manifest struct {
|
||||
// that takes it only once, so a rotation must be staged beside the current value — the form the
|
||||
// mesh does not build yet, and refuses by name. A secret that says neither is not rotated by
|
||||
// the mesh: the one fault worse than an unrotated credential is a rotated one the software
|
||||
// never saw.
|
||||
// never saw. `"issued-by": "outside"` says a party outside the mesh issues the value — an API
|
||||
// key, a bot token, a licence — so the mesh never puts one of its own in its place (novox/hq
|
||||
// ADR 0228); any other at-start secret given by hand lives only until the module's first good
|
||||
// start, and is then replaced.
|
||||
OwnSecrets OwnSecrets `json:"own-secrets,omitempty"`
|
||||
|
||||
// SecretsOwner is who the files holding this module's secrets belong to on the machine —
|
||||
@@ -1265,8 +1350,10 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
|
||||
}
|
||||
// A slug is a short identifier the mesh derives a login from (novox/hq ADR 0049). The same
|
||||
// charset as a name; its length is checked against a backend's limit at assignment, where the
|
||||
// node it joins is known — a slug that is fine on one machine's short name can overflow another's.
|
||||
// charset as a name; its length is judged against the bound of each provision it wants on the
|
||||
// longest machine name by the catalogue check (IdentityProblems, ADR 0225), and against the
|
||||
// machine it is on when its provider grants it — a slug that is fine on one machine's short name
|
||||
// can overflow another's.
|
||||
if m.Slug != "" && !name.MatchString(m.Slug) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
|
||||
@@ -1303,6 +1390,20 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
if !name.MatchString(p) {
|
||||
problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
|
||||
}
|
||||
// A bound says what keeps the name, so the refusal it causes can say it (ADR 0225); and it
|
||||
// leaves room for the shortest identity the mesh makes, or it would refuse every consumer.
|
||||
if id := offer.Identity; id != nil && !id.None {
|
||||
if strings.TrimSpace(id.In) == "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s bounds the identities of %s's consumers without saying what keeps them: `in`",
|
||||
m.Module, p))
|
||||
}
|
||||
if shortest := len(IdentityPrefix) + 3; id.Max < 0 || id.Max > 0 && id.Max < shortest {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s bounds %s's consumers' identities at %d characters, and the shortest the mesh "+
|
||||
"makes is %d", m.Module, p, id.Max, shortest))
|
||||
}
|
||||
}
|
||||
if offer.Credential != nil {
|
||||
own, declared := m.OwnSecrets[offer.Credential.Own]
|
||||
switch {
|
||||
@@ -1703,6 +1804,13 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
"or %q (applied by the module's own code to a backend that takes it once)",
|
||||
m.Module, name, own.Taken, TakenAtStart, TakenApplied))
|
||||
}
|
||||
if own.IssuedBy != "" && own.IssuedBy != IssuedOutside {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says its secret %q is issued by %q; a secret says %q when a party outside the mesh "+
|
||||
"issues it — a vendor's key, a bot's token, a licence — and says nothing when the mesh "+
|
||||
"may make it (novox/hq ADR 0228)",
|
||||
m.Module, name, own.IssuedBy, IssuedOutside))
|
||||
}
|
||||
}
|
||||
localOf := map[string]string{}
|
||||
for _, to := range m.SecretRequirements() {
|
||||
@@ -2165,10 +2273,24 @@ const (
|
||||
TakenApplied = "applied"
|
||||
)
|
||||
|
||||
// OwnSecret is where one of a module's own secrets lands, and how the module takes it.
|
||||
// IssuedOutside says a value was issued by a party outside the mesh — a vendor's API key, a bot's
|
||||
// token, a licence — so no value the mesh makes would work in its place (novox/hq ADR 0228).
|
||||
const IssuedOutside = "outside"
|
||||
|
||||
// OwnSecret is where one of a module's own secrets lands, how the module takes it, and — for a
|
||||
// value only an outside party can issue — that it is one.
|
||||
type OwnSecret struct {
|
||||
Path string
|
||||
Taken string
|
||||
Path string
|
||||
Taken string
|
||||
IssuedBy string
|
||||
}
|
||||
|
||||
// MeshMayMake says the mesh may put a value it makes in place of the one the secret holds: the
|
||||
// module reads it as it starts, and nobody outside the mesh issued it (novox/hq ADR 0228). A value
|
||||
// given to the mesh for such a secret lives only until the module's first good start under the
|
||||
// mesh; anything else given stays as given.
|
||||
func (s OwnSecret) MeshMayMake() bool {
|
||||
return s.Taken == TakenAtStart && s.IssuedBy != IssuedOutside
|
||||
}
|
||||
|
||||
// OwnSecrets is a module's own secrets by name. On the wire each is a path, or an object naming
|
||||
@@ -2189,15 +2311,16 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
|
||||
continue
|
||||
}
|
||||
var long struct {
|
||||
Path string `json:"path"`
|
||||
Taken string `json:"taken,omitempty"`
|
||||
Path string `json:"path"`
|
||||
Taken string `json:"taken,omitempty"`
|
||||
IssuedBy string `json:"issued-by,omitempty"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&long); err != nil {
|
||||
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\"}: %w", name, err)
|
||||
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\"}: %w", name, err)
|
||||
}
|
||||
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken}
|
||||
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken, IssuedBy: long.IssuedBy}
|
||||
}
|
||||
*o = out
|
||||
return nil
|
||||
@@ -2206,11 +2329,18 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
|
||||
func (o OwnSecrets) MarshalJSON() ([]byte, error) {
|
||||
entries := make(map[string]any, len(o))
|
||||
for name, s := range o {
|
||||
if s.Taken == "" {
|
||||
if s.Taken == "" && s.IssuedBy == "" {
|
||||
entries[name] = s.Path
|
||||
continue
|
||||
}
|
||||
entries[name] = map[string]string{"path": s.Path, "taken": s.Taken}
|
||||
long := map[string]string{"path": s.Path}
|
||||
if s.Taken != "" {
|
||||
long["taken"] = s.Taken
|
||||
}
|
||||
if s.IssuedBy != "" {
|
||||
long["issued-by"] = s.IssuedBy
|
||||
}
|
||||
entries[name] = long
|
||||
}
|
||||
return json.Marshal(entries)
|
||||
}
|
||||
|
||||
@@ -52,3 +52,42 @@ func TestAnOwnSecretTakenSomeOtherWayIsRefused(t *testing.T) {
|
||||
t.Fatal("an unknown field on an own secret was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A secret an outside party issues says so (novox/hq ADR 0228), is written back as it was read, and
|
||||
// is the one at-start secret the mesh may not make; any other word for who issued it is refused.
|
||||
func TestAnOwnSecretSaysWhenAnOutsidePartyIssuesIt(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"letta","version":"1","own-secrets":{
|
||||
"server-password":{"path":"/var/lib/letta/server-password","taken":"at-start"},
|
||||
"openai-api-key":{"path":"/var/lib/letta/openai-api-key","taken":"at-start","issued-by":"outside"},
|
||||
"telegram-token":{"path":"/var/lib/letta/telegram-token","issued-by":"outside"},
|
||||
"logflare":{"path":"/var/lib/letta/logflare","taken":"applied"},
|
||||
"broker":"/var/lib/mesh/letta/broker"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, may := range map[string]bool{"server-password": true, "openai-api-key": false,
|
||||
"telegram-token": false, "logflare": false, "broker": false} {
|
||||
if got := m.OwnSecrets[name].MeshMayMake(); got != may {
|
||||
t.Errorf("%s: the mesh may make it = %v, want %v", name, got, may)
|
||||
}
|
||||
}
|
||||
out, err := json.Marshal(m.OwnSecrets)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var again OwnSecrets
|
||||
if err := json.Unmarshal(out, &again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again["openai-api-key"] != m.OwnSecrets["openai-api-key"] || again["telegram-token"] != m.OwnSecrets["telegram-token"] {
|
||||
t.Fatalf("the round trip lost who issued it: %s", out)
|
||||
}
|
||||
if strings.Contains(string(out), `"taken":""`) {
|
||||
t.Fatalf("a secret that says only who issued it gained an empty taken: %s", out)
|
||||
}
|
||||
_, err = ParseManifest([]byte(`{"module":"letta","version":"1","own-secrets":{
|
||||
"openai-api-key":{"path":"/var/lib/letta/openai-api-key","issued-by":"openai"}}}`))
|
||||
if err == nil || !strings.Contains(err.Error(), `issued by "openai"`) {
|
||||
t.Fatalf("an unknown word for who issued a secret was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,6 +194,11 @@ type Needed struct {
|
||||
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
||||
// licence's manager; empty for every consumer.
|
||||
Manager bool
|
||||
// Identity is the longest consumer identity the answering provision keeps (novox/hq ADR 0225),
|
||||
// from the provider's own offer: what the mesh judges this consumer's identity against, on the
|
||||
// consumer's side for `status` and on the provider's before it grants. No bound for a provision
|
||||
// answered by a record, which keeps no name of anybody's.
|
||||
Identity IdentityBound
|
||||
}
|
||||
|
||||
// Refusal is why a set of assignments cannot become a declaration.
|
||||
@@ -403,7 +408,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
needs = append(needs, Needed{
|
||||
Name: want, From: node.Name, At: at,
|
||||
Serves: servedByOne(by, want), For: because[want],
|
||||
SharedOwn: sharedByOne(by, want)})
|
||||
SharedOwn: sharedByOne(by, want), Identity: by.IdentityBoundOf(want)})
|
||||
} else if served := servedByOne(by, want); len(served) > 0 {
|
||||
// Answered here with no credential to mint, but the provider serves facts the
|
||||
// consumer cannot guess — a port, a model name — and so still needs a binding.
|
||||
@@ -447,11 +452,15 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
return
|
||||
}
|
||||
shared := ""
|
||||
// A provider whose definition is not in hand is held to the tightest bound the
|
||||
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
|
||||
bound := DefaultIdentityBound
|
||||
if pm, known := catalogue[p.Module]; known {
|
||||
shared, _ = pm.SharedCredentialOf(want)
|
||||
bound = pm.IdentityBoundOf(want)
|
||||
}
|
||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared})
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
|
||||
}
|
||||
switch {
|
||||
case world.Unchecked:
|
||||
@@ -767,16 +776,27 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
var problems []string
|
||||
var held []Held
|
||||
|
||||
// onRecord is the recorded holder of a seat, if a handover ever named one.
|
||||
onRecord := func(claim, scope string) (Held, bool) {
|
||||
// onRecord is every recorded holder of a seat, if a handover ever named one: one for most seats,
|
||||
// and as many as were added for a replicated one (novox/hq ADR 0223).
|
||||
onRecord := func(claim, scope string) []Held {
|
||||
var out []Held
|
||||
for _, h := range holdings {
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
cs, cok := SeatNamed(claim)
|
||||
if ok && cok && hs.Name == cs.Name && h.Scope == scope {
|
||||
return h, true
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return Held{}, false
|
||||
return out
|
||||
}
|
||||
// recordedHere says this node's module is one of a seat's holders on record.
|
||||
recordedHere := func(claim, scope, module string) bool {
|
||||
for _, rec := range onRecord(claim, scope) {
|
||||
if rec.Node == node.Name && rec.Module == module {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
byScope := map[string]map[string]string{} // scope → claim → module
|
||||
@@ -787,21 +807,35 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
// the seat but is not the one on record is eligible, and that is all: it is not a second
|
||||
// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
|
||||
// lets the next holder stand beside the current one until the seat is handed over.
|
||||
if rec, recorded := onRecord(c.Name, scope); recorded {
|
||||
if rec.Node != node.Name || rec.Module != m.Module {
|
||||
if recs := onRecord(c.Name, scope); len(recs) > 0 {
|
||||
// **A seat held once is on record once** (novox/hq ADR 0223). Only a replicated seat
|
||||
// may have several holders on record; several for any other seat is a store that
|
||||
// disagrees with the mesh's definition of the role, and it is refused, named, rather
|
||||
// than letting two machines answer for what the mesh has one of.
|
||||
if s, known := SeatNamed(c.Name); known && !s.Replicated && len(recs) > 1 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%q is on record as held by %d assignments, and it is held once per %s — "+
|
||||
"`seat %s --to <node>/<module>` records one", c.Name, len(recs), scope, c.Name))
|
||||
continue
|
||||
}
|
||||
if !recordedHere(c.Name, scope, m.Module) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if byScope[scope] == nil {
|
||||
byScope[scope] = map[string]string{}
|
||||
}
|
||||
if other, taken := byScope[scope][c.Name]; taken {
|
||||
// **One seat under either of its names** (novox/hq ADR 0122): a manifest registered before
|
||||
// a rename claims the former name, and one written after it the current — two claimants of
|
||||
// one seat, compared by the seat they resolve to and not by how each spelled it.
|
||||
seat := canonicalSeat(c.Name)
|
||||
if other, taken := byScope[scope][seat]; taken {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s and %s both claim %q, and only one thing may hold it per %s",
|
||||
other, m.Module, c.Name, scope))
|
||||
other, m.Module, seat, scope))
|
||||
continue
|
||||
}
|
||||
byScope[scope][c.Name] = m.Module
|
||||
byScope[scope][seat] = m.Module
|
||||
held = append(held, Held{Claim: c.Name, Scope: scope, Node: node.Name,
|
||||
Module: m.Module, Site: node.Site})
|
||||
}
|
||||
@@ -810,11 +844,17 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
// And against the rest of the mesh, for the scopes that reach past this machine.
|
||||
for _, h := range held {
|
||||
for _, e := range elsewhere {
|
||||
if e.Node == node.Name || e.Claim != h.Claim || e.Scope != h.Scope {
|
||||
if e.Node == node.Name || canonicalSeat(e.Claim) != canonicalSeat(h.Claim) || e.Scope != h.Scope {
|
||||
continue
|
||||
}
|
||||
switch h.Scope {
|
||||
case ScopeMesh:
|
||||
// **A holder on record is never a second claimant** (novox/hq ADR 0223). Records are
|
||||
// the mesh's settled answer: one for most seats, several only for a replicated seat,
|
||||
// each added by an act. Two holders here are two records, and both hold.
|
||||
if recordedHere(h.Claim, h.Scope, h.Module) {
|
||||
continue
|
||||
}
|
||||
// Both claim and nobody is on record, or this refusal could not have happened.
|
||||
// The remedy is the handover that records the holder (novox/hq ADR 0131,
|
||||
// 04-ISSUES/170), so it is named here rather than left to be found.
|
||||
@@ -835,6 +875,15 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
return held, problems
|
||||
}
|
||||
|
||||
// canonicalSeat is the seat a claimed name refers to, by its current name: itself for a name the mesh
|
||||
// does not know (a module's own seat), its seat's name for a former one.
|
||||
func canonicalSeat(name string) string {
|
||||
if s, known := SeatNamed(name); known {
|
||||
return s.Name
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// checkResources refuses two modules writing the same thing.
|
||||
//
|
||||
// This costs no manifest field: the mesh already holds every resource of every module, so two
|
||||
@@ -912,6 +961,23 @@ func checkResources(modules []Manifest) []string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// **A file the mesh renders for a module is that module's path too** (novox/hq ADR 0223). The
|
||||
// machine's resolver file is a fact the uplink's holder asks for, and a second module asking
|
||||
// for it, or declaring it, would have the host write one path twice in every apply, the last
|
||||
// one winning. A fact under the operator's home is placed per account and compared nowhere.
|
||||
for _, name := range sortedFacts(m.Facts) {
|
||||
fact := m.Facts[name]
|
||||
if fact.Home || !strings.HasPrefix(fact.Path, "/") {
|
||||
continue
|
||||
}
|
||||
key := "path " + fact.Path
|
||||
if other, taken := owner[key]; taken && other != m.Module {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s and %s both declare the path %q", other, m.Module, fact.Path))
|
||||
}
|
||||
owner[key] = m.Module
|
||||
ownedPath[fact.Path] = m.Module
|
||||
}
|
||||
}
|
||||
|
||||
// An accessed path is the operator's, so no module may declare it as one of its own
|
||||
@@ -1156,6 +1222,18 @@ func answeredElsewhere(want string, node Node, world World, brokered map[string]
|
||||
if c, pinned := world.Pinned[want]; pinned {
|
||||
return c.Node != node.Name
|
||||
}
|
||||
// **A machine holding the seat answers itself** (novox/hq ADR 0223). With a replicated seat
|
||||
// another machine holds it too, and the first holder in the providers' order may be that one; a
|
||||
// holder is still where this machine's own requirement is answered, so its resolver file lists
|
||||
// itself first.
|
||||
if seat, delivered := SeatDelivering(want); delivered {
|
||||
for _, h := range append(append([]Held(nil), world.Holdings...), world.Held...) {
|
||||
if hs, ok := SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope &&
|
||||
h.Node == node.Name {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
holder, held := HolderAmong(want, world.Offered[want], world.Held)
|
||||
return held && holder.Node != node.Name
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0223 part 2: /etc/resolv.conf belongs to the module holding node-uplink. The seat that
|
||||
// wrote it, node-resolver-config, and ADR 0220's dependency of it on the uplink retire: one owner for
|
||||
// the file, and it is the program that would otherwise rewrite it.
|
||||
|
||||
func TestTheResolverConfigSeatIsGone(t *testing.T) {
|
||||
if _, known := SeatNamed("node-resolver-config"); known {
|
||||
t.Error("node-resolver-config is still in the mesh's set; the uplink's holder writes the resolver file")
|
||||
}
|
||||
// An uplink's holder needs no seat beside it for the file: it is its own.
|
||||
if got := DependsOn(mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"})); len(got) != 0 {
|
||||
t.Errorf("an uplink holder with nothing declared depends on %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it is: nothing claims the retired seat, and every manager the mesh knows holds the
|
||||
// uplink and writes the resolver file.
|
||||
func TestTheCataloguesUplinksWriteTheResolverFileAndNothingElseDoes(t *testing.T) {
|
||||
cat := map[string]Manifest{}
|
||||
for _, m := range theCatalogue(t) {
|
||||
cat[m.Module] = m
|
||||
}
|
||||
if got := PossibleHolders(cat, "node-uplink"); !reflect.DeepEqual(got, uplinks) {
|
||||
t.Errorf("node-uplink can be held by %v, not %v", got, uplinks)
|
||||
}
|
||||
for name, m := range cat {
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == "node-resolver-config" {
|
||||
t.Errorf("%s still claims node-resolver-config", name)
|
||||
}
|
||||
}
|
||||
_, writes := m.Facts["resolvers"]
|
||||
isUplink := false
|
||||
for _, u := range uplinks {
|
||||
isUplink = isUplink || u == name
|
||||
}
|
||||
for _, f := range m.Facts {
|
||||
if f.Path == "/etc/resolv.conf" && !isUplink {
|
||||
t.Errorf("%s writes /etc/resolv.conf and does not hold the uplink", name)
|
||||
}
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["path"] == "/etc/resolv.conf" {
|
||||
t.Errorf("%s declares /etc/resolv.conf as a file of its own", name)
|
||||
}
|
||||
}
|
||||
if isUplink && !writes {
|
||||
t.Errorf("%s holds the uplink and does not write the resolver file", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
// same arrangement, and to the two things a resolver here must never do — read resolv.conf for
|
||||
// its upstreams, or take an address systemd-resolved holds.
|
||||
|
||||
// resolverShelf is the two resolver modules and the container runtime beside something that
|
||||
// answers `mesh-addressing`.
|
||||
// resolverShelf is the resolver, an uplink module that writes what the machine asks (novox/hq ADR
|
||||
// 0223), and the container runtime beside something that answers `mesh-addressing`.
|
||||
// The networking module that really does is composed in the controller and cannot be imported
|
||||
// here, so a stand-in offers the same word; what is under test is the manifests, not the network.
|
||||
func resolverShelf(t *testing.T) map[string]Manifest {
|
||||
@@ -24,7 +24,7 @@ func resolverShelf(t *testing.T) map[string]Manifest {
|
||||
shelf := map[string]Manifest{
|
||||
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
|
||||
}
|
||||
for _, name := range []string{"dnsmasq", "resolv-conf", "docker"} {
|
||||
for _, name := range []string{"dnsmasq", "systemd-networkd", "docker"} {
|
||||
shelf[name] = catalogueManifest(t, name)
|
||||
}
|
||||
return shelf
|
||||
@@ -84,25 +84,26 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
|
||||
t.Errorf("the mesh's resolver still reads or listens on %q", never)
|
||||
}
|
||||
}
|
||||
// And the file that decides what the machine asks names the mesh's resolver first, by address,
|
||||
// and a public one second, asked only when the first is silent (ADR 0196).
|
||||
var resolv string
|
||||
for _, r := range catalogueManifest(t, "resolv-conf").Resources {
|
||||
if r["path"] == "/etc/resolv.conf" {
|
||||
resolv, _ = r["content"].(string)
|
||||
// And the file that decides what the machine asks names every one of the mesh's resolvers, by
|
||||
// address, from the seat's holders, and no public one (ADR 0223): a resolver library that asks
|
||||
// every listed server at once takes the first reply, and a public "no such name" for a mesh name
|
||||
// won it. Written by every uplink module, since the uplink's holder owns the file.
|
||||
for _, uplink := range uplinks {
|
||||
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
|
||||
if !ok || fact.Path != "/etc/resolv.conf" {
|
||||
t.Fatalf("%s's resolver file is not rendered from the roster: %+v", uplink, fact)
|
||||
}
|
||||
}
|
||||
var nameservers []string
|
||||
for _, line := range strings.Split(resolv, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") {
|
||||
nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
|
||||
if !strings.Contains(fact.Template, `{{range index .Holders "mesh-dns-resolver"}}nameserver {{.Address}}`) {
|
||||
t.Errorf("%s's resolv.conf does not list every holder of the mesh's resolver:\n%s", uplink, fact.Template)
|
||||
}
|
||||
for _, line := range strings.Split(fact.Template, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") && !strings.Contains(line, "{{") {
|
||||
t.Errorf("%s's resolv.conf names a resolver of its own beside the mesh's: %s", uplink, line)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(fact.Template, "options timeout:1 attempts:2 edns0\n") {
|
||||
t.Errorf("%s: a silent resolver is not passed over after one short wait:\n%s", uplink, fact.Template)
|
||||
}
|
||||
}
|
||||
if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
|
||||
t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
|
||||
}
|
||||
if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
|
||||
t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,9 +112,10 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
|
||||
// that file, the machine pointed at the resolver by address, and the runtime given no resolver of
|
||||
// its own but kept running across a restart (ADR 0196).
|
||||
func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf", "docker"},
|
||||
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "systemd-networkd", "docker"},
|
||||
Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
|
||||
"package-manager": true, "service-manager": true, "privileged": true}}, World{})
|
||||
"package-manager": true, "service-manager": true, "privileged": true,
|
||||
"uplink-systemd-networkd": true}}, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -126,6 +128,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
// happen to be the same map, since nothing routed is part of it.
|
||||
Names: twoMachines, Machines: twoMachines, Suffix: "internal",
|
||||
Zones: []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
|
||||
Holders: map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||
Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
|
||||
})
|
||||
@@ -170,7 +173,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
t.Errorf("the resolver still writes the runtime's dns: %v", ids["dnsmasq.runtime-dns"])
|
||||
}
|
||||
for id := range ids {
|
||||
if strings.HasPrefix(id, "resolv-conf.runtime") {
|
||||
if strings.HasPrefix(id, "systemd-networkd.runtime") {
|
||||
t.Errorf("what the machine asks still writes the runtime's file: %s", id)
|
||||
}
|
||||
}
|
||||
@@ -205,29 +208,40 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
|
||||
}
|
||||
|
||||
resolv := ids["resolv-conf.resolv"]
|
||||
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
|
||||
t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
|
||||
resolv := ids["systemd-networkd.fact-resolvers"]
|
||||
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\noptions ") {
|
||||
t.Fatalf("the machine is not pointed at the resolver by address, and only at it: %v", resolv)
|
||||
}
|
||||
}
|
||||
|
||||
// Two modules deciding what a machine asks are refused on one machine, as before — the claim
|
||||
// exists so they never take turns overwriting each other.
|
||||
//
|
||||
// **The second is made up.** The catalogue's only other claimant, a systemd-resolved split-DNS
|
||||
// module, was retired with the choice against a stub on every node (novox/hq ADR 0196, ADR 0220);
|
||||
// what is under test is the claim, so any second module claiming it will do.
|
||||
// Two modules deciding what a machine asks are refused on one machine, as before — now that the file
|
||||
// is the uplink's (novox/hq ADR 0223), by two things: a machine runs one network manager, and no other
|
||||
// module may write the resolver file beside the uplink's, as a file or as a rendered fact.
|
||||
func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) {
|
||||
shelf := resolverShelf(t)
|
||||
shelf["other-resolver-config"] = Manifest{Module: "other-resolver-config", Version: "1",
|
||||
Claims: []Claim{{Name: "node-resolver-config", Scope: ScopeNode}}}
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "resolv-conf", "other-resolver-config"},
|
||||
Node{Name: "anchor", At: "anchor.internal"}, World{})
|
||||
if err == nil {
|
||||
t.Fatal("resolv-conf and a second module deciding what the machine asks were both assigned to one machine")
|
||||
shelf["networkmanager"] = catalogueManifest(t, "networkmanager")
|
||||
node := Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
|
||||
"package-manager": true, "service-manager": true,
|
||||
"uplink-systemd-networkd": true, "uplink-networkmanager": true}}
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "networkmanager"}, node, World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "node-uplink") {
|
||||
t.Fatalf("two network managers were both assigned to one machine: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "node-resolver-config") {
|
||||
t.Fatalf("the refusal does not say what was claimed: %v", err)
|
||||
|
||||
// The second is made up: what is under test is that the resolver file has one owner, so any
|
||||
// module asking the mesh to render it — or declaring it — will do.
|
||||
for name, m := range map[string]Manifest{
|
||||
"a-rendered-one": {Module: "a-rendered-one", Version: "1",
|
||||
Facts: map[string]RosterFile{"mine": {Path: "/etc/resolv.conf", Template: "nameserver 10.42.0.1\n"}}},
|
||||
"a-declared-one": {Module: "a-declared-one", Version: "1", Resources: []map[string]any{
|
||||
{"id": "mine", "type": "file", "path": "/etc/resolv.conf", "content": "nameserver 10.42.0.1\n"}}},
|
||||
} {
|
||||
shelf := resolverShelf(t)
|
||||
shelf[name] = m
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", name}, node, World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "/etc/resolv.conf") {
|
||||
t.Errorf("%s wrote the resolver file beside the uplink's: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0220: what a machine asks for names needs the uplink held beside it.
|
||||
//
|
||||
// resolv.conf is the mesh's only while the program managing the machine's network is told to leave
|
||||
// it alone, and the uplink's holder is what tells it (ADR 0117). Without one, the first connectivity
|
||||
// change rewrites the file — so the dependency is checked at assignment, by the same mechanism as a
|
||||
// service's on the service manager (ADR 0207), derived from the claim and never stated in a manifest.
|
||||
|
||||
// resolverAndUplinks is a resolver-config holder and two uplink holders, with no resources of their
|
||||
// own so that nothing but the seats is judged.
|
||||
func resolverAndUplinks() map[string]Manifest {
|
||||
return shelf(
|
||||
mod("resolv-conf", nil, nil, nil, Claim{Name: "node-resolver-config"}),
|
||||
mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"}),
|
||||
mod("systemd-networkd", nil, nil, nil, Claim{Name: "node-uplink"}),
|
||||
)
|
||||
}
|
||||
|
||||
func TestTheResolverConfigSeatNeedsTheUplink(t *testing.T) {
|
||||
s, known := SeatNamed("node-resolver-config")
|
||||
if !known {
|
||||
t.Fatal("node-resolver-config is not in the mesh's set")
|
||||
}
|
||||
if !reflect.DeepEqual(s.Needs, []string{"node-uplink"}) {
|
||||
t.Errorf("node-resolver-config needs %v, want [node-uplink] (ADR 0220)", s.Needs)
|
||||
}
|
||||
// Derived from the claim: a module claiming the seat depends on the uplink with nothing written.
|
||||
got := DependsOn(mod("anything", nil, nil, nil, Claim{Name: "node-resolver-config"}))
|
||||
if !reflect.DeepEqual(got, []string{"node-uplink"}) {
|
||||
t.Errorf("a module claiming node-resolver-config depends on %v, want [node-uplink]", got)
|
||||
}
|
||||
// And the uplink's holders need nothing of the kind: the dependency runs one way.
|
||||
if got := DependsOn(mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"})); len(got) != 0 {
|
||||
t.Errorf("an uplink holder depends on %v; it needs no seat beside it", got)
|
||||
}
|
||||
}
|
||||
|
||||
// What the store loads has no column for it, so the compiled value survives a load.
|
||||
func TestTheNeedSurvivesTheStoresRows(t *testing.T) {
|
||||
defer UseSeats(DefaultSeats())
|
||||
var rows []Seat
|
||||
for _, s := range DefaultSeats() {
|
||||
rows = append(rows, Seat{Name: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision})
|
||||
}
|
||||
UseSeats(rows)
|
||||
if s, _ := SeatNamed("node-resolver-config"); !reflect.DeepEqual(s.Needs, []string{"node-uplink"}) {
|
||||
t.Errorf("after loading the store's rows node-resolver-config needs %v", s.Needs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssigningTheResolverConfigWithoutAnUplinkIsRefused(t *testing.T) {
|
||||
cat := resolverAndUplinks()
|
||||
_, err := AssignRefusal(cat, "laptop", nil, []string{"resolv-conf"})
|
||||
var refusal *Refusal
|
||||
if !errors.As(err, &refusal) {
|
||||
t.Fatalf("resolv-conf was assigned to a machine nothing manages the network of: %v", err)
|
||||
}
|
||||
for _, want := range []string{"resolv-conf on laptop depends on node-uplink", "networkmanager", "systemd-networkd"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q:\n%s", want, err)
|
||||
}
|
||||
}
|
||||
// Beside a holder, or together with one in one act, it is let through.
|
||||
if _, err := AssignRefusal(cat, "laptop", []string{"networkmanager"}, []string{"resolv-conf"}); err != nil {
|
||||
t.Errorf("resolv-conf beside networkmanager was refused: %v", err)
|
||||
}
|
||||
if _, err := AssignRefusal(cat, "anchor", nil, []string{"systemd-networkd", "resolv-conf"}); err != nil {
|
||||
t.Errorf("resolv-conf assigned with systemd-networkd was refused: %v", err)
|
||||
}
|
||||
// And a composition without one is refused once the switch is on.
|
||||
if _, err := Resolve(cat, []string{"resolv-conf"}, workstation(), World{}); err == nil ||
|
||||
!strings.Contains(err.Error(), "node-uplink") {
|
||||
t.Errorf("a node with resolv-conf and no uplink composed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnassigningTheUplinkUnderTheResolverConfigIsRefused(t *testing.T) {
|
||||
cat := resolverAndUplinks()
|
||||
err := UnassignRefusal(cat, "laptop", []string{"networkmanager", "resolv-conf"}, []string{"networkmanager"})
|
||||
if err == nil || !strings.Contains(err.Error(), "resolv-conf") || !strings.Contains(err.Error(), "node-uplink") {
|
||||
t.Errorf("taking the uplink from under resolv-conf gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it is: the module that writes resolv.conf depends on the uplink, and every manager
|
||||
// the mesh knows can meet it — so the refusal always has a remedy to name.
|
||||
func TestTheCataloguesResolverConfigHasUplinkHoldersToName(t *testing.T) {
|
||||
cat := map[string]Manifest{}
|
||||
for _, m := range theCatalogue(t) {
|
||||
cat[m.Module] = m
|
||||
}
|
||||
deps := DependsOn(cat["resolv-conf"])
|
||||
found := false
|
||||
for _, d := range deps {
|
||||
found = found || d == "node-uplink"
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("the catalogue's resolv-conf depends on %v, not on node-uplink", deps)
|
||||
}
|
||||
holders := PossibleHolders(cat, "node-uplink")
|
||||
for _, want := range []string{"dhcpcd", "networkmanager", "systemd-networkd"} {
|
||||
in := false
|
||||
for _, h := range holders {
|
||||
in = in || h == want
|
||||
}
|
||||
if !in {
|
||||
t.Errorf("%s does not hold node-uplink in the catalogue; holders: %v", want, holders)
|
||||
}
|
||||
}
|
||||
if got := PossibleHolders(cat, "node-resolver-config"); !reflect.DeepEqual(got, []string{"resolv-conf"}) {
|
||||
t.Errorf("node-resolver-config can be held by %v; resolv-conf alone since ADR 0220", got)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,12 @@ type rosterView struct {
|
||||
// Zones is every zone a module in the mesh answers itself, with where its answerer is (novox/hq
|
||||
// ADR 0199) — what the mesh's resolver forwards. Ordered by zone.
|
||||
Zones []rosterZone
|
||||
// Holders is the machines holding each replicated mesh seat, by seat (novox/hq ADR 0223) — what
|
||||
// a machine's resolver file lists for `mesh-dns-resolver`. **This machine first when it is one
|
||||
// of them**, then the rest by name: the nearest holder is asked first, and two renderings of
|
||||
// one mesh on one machine are one file. A template reads one seat's with
|
||||
// `index .Holders "<seat>"`; a seat nobody holds ranges over nothing.
|
||||
Holders map[string][]rosterEntry
|
||||
}
|
||||
|
||||
// rosterZone is one zone as a template sees it: the zone, and the address and port answering it.
|
||||
@@ -96,21 +102,25 @@ func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]st
|
||||
// FactsWithZonesInto is FactsInto with the mesh's zones in the view, for a template that ranges them.
|
||||
func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string,
|
||||
zones []ZoneAt) ([]map[string]any, error) {
|
||||
return FactsFrom(m, r, Rendering{Names: every, Machines: machines, Accounts: accounts, Suffix: suffix,
|
||||
Zones: zones})
|
||||
}
|
||||
|
||||
// FactsFrom renders the roster files a module asked for from everything the mesh composed for this
|
||||
// machine: its machines, zones and the holders of each replicated seat.
|
||||
func FactsFrom(m Manifest, r Resolution, with Rendering) ([]map[string]any, error) {
|
||||
if len(m.Facts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
names := make([]string, 0, len(m.Facts))
|
||||
for name := range m.Facts {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
names := sortedFacts(m.Facts)
|
||||
|
||||
view := rosterView{
|
||||
Node: r.Node,
|
||||
Suffix: strings.TrimPrefix(suffixOr(suffix), "."),
|
||||
Names: entriesFrom(every, accounts, suffix),
|
||||
Machines: entriesFrom(machines, accounts, suffix),
|
||||
Zones: zonesFrom(zones),
|
||||
Suffix: strings.TrimPrefix(suffixOr(with.Suffix), "."),
|
||||
Names: entriesFrom(with.Names, with.Accounts, with.Suffix),
|
||||
Machines: entriesFrom(with.Machines, with.Accounts, with.Suffix),
|
||||
Zones: zonesFrom(with.Zones),
|
||||
Holders: holdersFrom(with.Holders, with.Accounts, with.Suffix, r.Node),
|
||||
}
|
||||
|
||||
out := make([]map[string]any, 0, len(names))
|
||||
@@ -250,3 +260,28 @@ func zonesFrom(zones []ZoneAt) []rosterZone {
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Zone < out[j].Zone })
|
||||
return out
|
||||
}
|
||||
|
||||
// holdersFrom is each replicated seat's holders as a template ranges them: this machine first when
|
||||
// it holds the seat, then the others by name (novox/hq ADR 0223). A machine with no address is left
|
||||
// out, as everywhere in the roster — a resolver named at nothing is a lookup that hangs.
|
||||
func holdersFrom(holders map[string]map[string]string, accounts map[string]string, suffix, node string) map[string][]rosterEntry {
|
||||
out := make(map[string][]rosterEntry, len(holders))
|
||||
for seat, at := range holders {
|
||||
entries := entriesFrom(at, accounts, suffix)
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
return entries[i].Name == node && entries[j].Name != node
|
||||
})
|
||||
out[seat] = entries
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedFacts is a module's fact names in order, so what is said about them is said the same way twice.
|
||||
func sortedFacts(facts map[string]RosterFile) []string {
|
||||
out := make([]string, 0, len(facts))
|
||||
for name := range facts {
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -6,9 +6,10 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A module depends on the node seats that apply its resources (novox/hq ADR 0207), on the
|
||||
// seats it contributes to (novox/hq ADR 0210), and on the seats a seat it holds needs beside it
|
||||
// (novox/hq ADR 0220).
|
||||
// A module depends on the node seats that apply its resources (novox/hq ADR 0207) and on the
|
||||
// seats it contributes to (novox/hq ADR 0210). A third source — what a seat it holds needs beside it
|
||||
// (novox/hq ADR 0220) — had one user, node-resolver-config needing node-uplink, and went with that
|
||||
// seat when the resolver file became the uplink's own (novox/hq ADR 0223).
|
||||
//
|
||||
// Some of what a module declares is applied through software on the machine that is itself a
|
||||
// module: a service through the service manager, a package through the package manager, a container
|
||||
@@ -94,9 +95,6 @@ func DependsOn(m Manifest) []string {
|
||||
for _, seat := range contributedTo(m) {
|
||||
seen[seat] = true
|
||||
}
|
||||
for _, seat := range neededBesideClaims(m) {
|
||||
seen[seat] = true
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for s := range seen {
|
||||
out = append(out, s)
|
||||
@@ -128,20 +126,6 @@ func contributedTo(m Manifest) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// neededBesideClaims is every seat a seat the module claims at node scope needs held on the same
|
||||
// node (novox/hq ADR 0220): the holder of node-resolver-config is only right while node-uplink's
|
||||
// holder keeps the network manager off resolv.conf. Derived from the claim, as a resource's seat is
|
||||
// derived from its type, so a module that claims the seat cannot leave the dependency out.
|
||||
func neededBesideClaims(m Manifest) []string {
|
||||
var out []string
|
||||
for _, name := range nodeSeatsClaimed(m) {
|
||||
if s, known := SeatNamed(name); known {
|
||||
out = append(out, s.Needs...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// claimsSeat is whether a module claims a node seat, by its current name or one it used to have
|
||||
// (ADR 0122), so a rename leaves the dependency met.
|
||||
func claimsSeat(m Manifest, seat string) bool {
|
||||
|
||||
+50
-40
@@ -21,8 +21,16 @@ import (
|
||||
type Seat struct {
|
||||
// Name is what a manifest claims.
|
||||
Name string
|
||||
// Scope is where there may be only one holder.
|
||||
// Scope is where there may be only one holder — unless the seat is Replicated.
|
||||
Scope string
|
||||
// Replicated says a mesh seat may be held on several machines at once, each holder answering the
|
||||
// same thing (novox/hq ADR 0223): the mesh's resolver, held on the anchor and the home server so a
|
||||
// machine's resolver file lists two that give one answer. Each holder is on record, added by an
|
||||
// act (`seat <name> --add <node>/<module>`), never by being assigned: two claimants with nothing on
|
||||
// record are refused exactly as for any mesh seat. One per machine still — two modules on one
|
||||
// node claiming it are refused. Compiled, never stored, like Receives: it is the mesh's definition of
|
||||
// the role, and the store's rows carry no column for it.
|
||||
Replicated bool
|
||||
// Delivers is the provision the seat's holder answers for, or empty. A seat that delivers a
|
||||
// provision may only be held by a module providing it at the seat's scope, and its holder is
|
||||
// what a requirement for that provision resolves to when several modules provide it.
|
||||
@@ -45,13 +53,6 @@ type Seat struct {
|
||||
// ${contribution:<seat>:<kind>}. Compiled, never stored: like the protocol, it is the mesh's
|
||||
// definition of the role, and the store's rows carry no column for it.
|
||||
Receives []Receivable
|
||||
// Needs is every node seat this seat's holder needs held on its own node (novox/hq ADR 0220): a
|
||||
// role whose holder is only right while another role is filled beside it. A module claiming this
|
||||
// seat depends on each, exactly as a module declaring a service depends on the service manager
|
||||
// (ADR 0207) — derived from the claim, never written in a manifest, and judged over the node's
|
||||
// whole set of assignments. Compiled, never stored, like Receives: it is the mesh's definition of
|
||||
// the role, and the store's rows carry no column for it.
|
||||
Needs []string
|
||||
// Decision is the record that made it a seat.
|
||||
Decision string
|
||||
}
|
||||
@@ -82,7 +83,11 @@ var defaultSeats = append([]Seat{
|
||||
// `assign` and the rest are a role's interface, not a container's, and stay addressable while
|
||||
// the control plane is replaced.
|
||||
{Name: ControllerSeatName, Scope: ScopeMesh, Decision: "novox/hq ADR 0079",
|
||||
Emits: []string{"applied", "refused", "built-before"},
|
||||
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||
Emits: []string{"applied", "refused", "built-before",
|
||||
"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat",
|
||||
// A value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
"secret-replaced"},
|
||||
Serves: ControllerVerbs},
|
||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||
// served by whichever module holds the seat with tools of these names.
|
||||
@@ -140,14 +145,20 @@ var defaultSeats = append([]Seat{
|
||||
// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
|
||||
{Name: "mesh-build-machine", Scope: ScopeMesh,
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
|
||||
// **The mesh's one resolver** (novox/hq ADR 0194, 0196): every node's internal domain, held in one
|
||||
// place, and every node and container asks it first. Delivers what a machine's resolver
|
||||
// configuration requires, so that requirement resolves to the holder wherever it is placed.
|
||||
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Decision: "novox/hq ADR 0194"},
|
||||
// **A machine's /etc/hosts is one module's** (novox/hq ADR 0199): its holder writes the machine's
|
||||
// own lines and keeps every other line as the operator's, changed through these three verbs on that
|
||||
// machine alone. The controller holds none of it.
|
||||
{Name: "node-hosts-file", Scope: ScopeNode, Decision: "novox/hq ADR 0199",
|
||||
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
|
||||
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
|
||||
// than one machine, each answering the same names from the same roster, and every machine's
|
||||
// resolver file lists every holder — its own first — and no public resolver, so whichever answers
|
||||
// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
|
||||
// requirement resolves to a holder wherever they are placed.
|
||||
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
|
||||
Decision: "novox/hq ADR 0194, ADR 0223"},
|
||||
// **A machine's names are one module's** (novox/hq ADR 0199, ADR 0223): its holder writes
|
||||
// /etc/hostname and the machine's own lines in /etc/hosts, and keeps every other line of the hosts
|
||||
// file as the operator's, changed through these three verbs on that machine alone. The controller
|
||||
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
|
||||
// alias on a mesh that knew it.
|
||||
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
|
||||
Serves: []Verb{
|
||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||
"the operator's, or the block of the module or tool that writes it.",
|
||||
@@ -248,18 +259,12 @@ var defaultSeats = append([]Seat{
|
||||
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
|
||||
// model change, not a rename, so it stays until that is built.
|
||||
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
||||
// What a machine asks for names (novox/hq ADR 0121, ADR 0196): its holder writes resolv.conf.
|
||||
// **And it needs the uplink held beside it** (novox/hq ADR 0220): resolv.conf stays the mesh's
|
||||
// only while the program managing the machine's network is told to keep its hands off it, and
|
||||
// that is what the uplink's holder says (ADR 0117). Without one, the first connectivity change
|
||||
// rewrites the file and every surface of the mesh still reads green — so it is refused at
|
||||
// assignment instead.
|
||||
{Name: "node-resolver-config", Scope: ScopeNode, Decision: "novox/hq ADR 0121, ADR 0220",
|
||||
Needs: []string{"node-uplink"}},
|
||||
// The program that manages the machine's own network. It delivers nothing: its holder only
|
||||
// keeps the manager and the mesh from contradicting each other — the resolver file left to the
|
||||
// mesh, the private network's interface left alone — and never declares a link, an address or
|
||||
// a wireless network, because the link is the only channel a fix could arrive on. A seat
|
||||
// The program that manages the machine's own network. It delivers nothing: its holder keeps the
|
||||
// manager and the mesh from contradicting each other — the private network's interface left
|
||||
// alone — and writes the machine's resolver file itself, because the manager is what would
|
||||
// otherwise rewrite it (novox/hq ADR 0223, which retired node-resolver-config into this seat).
|
||||
// It never declares a link, an address or a wireless network, because the link is the only
|
||||
// channel a fix could arrive on. A seat
|
||||
// rather than a condition in the resolver's module, so a machine running two managers is
|
||||
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117"},
|
||||
@@ -314,11 +319,11 @@ func UseSeats(s []Seat) {
|
||||
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
|
||||
}
|
||||
}
|
||||
// What a seat receives and what its holder needs are never stored (novox/hq ADR 0212, ADR
|
||||
// 0220), so they are always the compiled ones.
|
||||
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
|
||||
// 0223), so they are always the compiled ones.
|
||||
if d, known := byName[row.Name]; known {
|
||||
row.Receives = d.Receives
|
||||
row.Needs = d.Needs
|
||||
row.Replicated = d.Replicated
|
||||
}
|
||||
merged = append(merged, row)
|
||||
}
|
||||
@@ -498,19 +503,24 @@ func seatNames() string {
|
||||
// two modules on one node could both provide a provision, and only the one holding the seat
|
||||
// answers for it. Nothing when no seat delivers the provision, when nobody holds
|
||||
// it, or when the holder is not among the providers offered.
|
||||
//
|
||||
// **The first holder in the providers' own order** (novox/hq ADR 0223). A replicated seat has
|
||||
// several, and the answer must not depend on the order the mesh happened to resolve its machines
|
||||
// in: the providers come sorted by machine, so every consumer is bound to the same one. A seat with
|
||||
// one holder gets the same answer as before.
|
||||
func HolderAmong(provision string, providers []Provider, held []Held) (Provider, bool) {
|
||||
seat, delivered := SeatDelivering(provision)
|
||||
if !delivered {
|
||||
return Provider{}, false
|
||||
}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
|
||||
// former name still matches it after a rename (novox/hq ADR 0122).
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
|
||||
continue
|
||||
}
|
||||
for _, p := range providers {
|
||||
for _, p := range providers {
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
// seat's former name still matches it after a rename (novox/hq ADR 0122).
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
|
||||
continue
|
||||
}
|
||||
if p.Node == h.Node && p.Module == h.Module {
|
||||
return p, true
|
||||
}
|
||||
|
||||
@@ -46,16 +46,18 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
delivered[s.Delivers] = s.Name
|
||||
}
|
||||
}
|
||||
// Thirty-seven since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||
// Thirty-six since node-resolver-config retired into node-uplink (novox/hq ADR 0223); thirty-seven
|
||||
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
|
||||
// thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four
|
||||
// thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199, now
|
||||
// node-hostname); thirty-four
|
||||
// with node-hotkeys (ADR 0212); thirty-three with node-power (ADR 0211); thirty-two since the
|
||||
// graphical session's eleven (ADR 0208); twenty-one with node-package-manager and
|
||||
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
||||
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||
if len(Seats()) != 37 {
|
||||
t.Errorf("the mesh defines %d seats rather than 37; the set is closed, so a change here is "+
|
||||
if len(Seats()) != 36 {
|
||||
t.Errorf("the mesh defines %d seats rather than 36; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The mesh has two resolvers (novox/hq ADR 0223): `mesh-dns-resolver` is replicated, held on the
|
||||
// anchor and on the home server, each answering the same names; every machine's resolver file lists
|
||||
// every holder — its own first when it is one — and no public resolver. ADR 0196 listed the mesh's
|
||||
// resolver then a public one, and musl asks both at once and takes the first reply: from the home
|
||||
// server the public "no such name" for the anchor's mesh name won, every time, in every Alpine build.
|
||||
// The file is written by the module holding the machine's uplink (ADR 0223 part 2).
|
||||
|
||||
// resolverMachines is the anchor and the home server holding the resolver, and a laptop holding nothing.
|
||||
var resolverMachines = map[string]string{
|
||||
"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "home.internal": "10.42.0.3"}
|
||||
|
||||
// bothResolvers is the two holders on record, as `seat mesh-dns-resolver --add` leaves them.
|
||||
var bothResolvers = []Held{
|
||||
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "dnsmasq"},
|
||||
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "home", Module: "dnsmasq"},
|
||||
}
|
||||
|
||||
// uplinks is every module in the catalogue holding node-uplink, and so writing the machine's resolver
|
||||
// file (novox/hq ADR 0223 part 2): the program that would otherwise rewrite it is the one that writes it.
|
||||
var uplinks = []string{"dhcpcd", "networkmanager", "systemd-networkd"}
|
||||
|
||||
// managing is a machine as each uplink module needs it: able to install, run a service and run the
|
||||
// manager that module is for.
|
||||
func managing(node string) Node {
|
||||
caps := map[string]bool{"package-manager": true, "service-manager": true}
|
||||
for _, u := range uplinks {
|
||||
caps["uplink-"+u] = true
|
||||
}
|
||||
return Node{Name: node, At: node + ".internal", Capabilities: caps}
|
||||
}
|
||||
|
||||
// twoResolverShelf is the resolver, the uplink modules that write what a machine asks, and a stand-in
|
||||
// answering `mesh-addressing`.
|
||||
func twoResolverShelf(t *testing.T) map[string]Manifest {
|
||||
t.Helper()
|
||||
out := map[string]Manifest{
|
||||
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
|
||||
"dnsmasq": catalogueManifest(t, "dnsmasq"),
|
||||
}
|
||||
for _, u := range uplinks {
|
||||
out[u] = catalogueManifest(t, u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// worldWithout is the rest of the mesh as a plan for one machine sees it: every other holder's claim
|
||||
// and offer, and both holders on record.
|
||||
func worldWithout(node string) World {
|
||||
w := World{Holdings: bothResolvers, Offered: map[string][]Provider{}}
|
||||
for _, h := range bothResolvers {
|
||||
if h.Node == node {
|
||||
continue
|
||||
}
|
||||
w.Held = append(w.Held, h)
|
||||
w.Offered["wildcard-resolution"] = append(w.Offered["wildcard-resolution"],
|
||||
Provider{Node: h.Node, At: h.Node + ".internal", Module: h.Module})
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// resolvConfOn resolves and composes one machine and answers with the nameservers its resolver file
|
||||
// lists, in order, and the file. The file is the uplink's — `uplink` is one of the assigned modules —
|
||||
// and nothing else on the machine declares that path.
|
||||
func resolvConfOn(t *testing.T, node, uplink string, assigned []string) ([]string, string) {
|
||||
t.Helper()
|
||||
got, err := Resolve(twoResolverShelf(t), assigned, managing(node), worldWithout(node))
|
||||
if err != nil {
|
||||
t.Fatalf("%s with %s does not resolve with two resolvers on record: %v", node, uplink, err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{
|
||||
Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
|
||||
Holders: map[string]map[string]string{"mesh-dns-resolver": {
|
||||
"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("%s with %s does not compose: %v", node, uplink, err)
|
||||
}
|
||||
var file map[string]any
|
||||
for _, r := range out {
|
||||
if r["path"] != "/etc/resolv.conf" {
|
||||
continue
|
||||
}
|
||||
if file != nil {
|
||||
t.Fatalf("%s declares /etc/resolv.conf twice: %v and %v", node, file["id"], r["id"])
|
||||
}
|
||||
file = r
|
||||
}
|
||||
if file == nil || file["id"] != uplink+".fact-resolvers" {
|
||||
t.Fatalf("%s's resolver file is not %s's: %v", node, uplink, file)
|
||||
}
|
||||
content, _ := file["content"].(string)
|
||||
var servers []string
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") {
|
||||
servers = append(servers, strings.TrimPrefix(line, "nameserver "))
|
||||
}
|
||||
}
|
||||
return servers, content
|
||||
}
|
||||
|
||||
// Per uplink module: each writes a resolver file listing both holders, the machine's own first on a
|
||||
// holder, and only the holders on a machine that is none.
|
||||
func TestEveryUplinkListsBothResolversItsOwnFirst(t *testing.T) {
|
||||
for _, uplink := range uplinks {
|
||||
for node, want := range map[string][]string{
|
||||
"anchor": {"10.42.0.1", "10.42.0.3"},
|
||||
"home": {"10.42.0.3", "10.42.0.1"},
|
||||
"laptop": {"10.42.0.1", "10.42.0.3"},
|
||||
} {
|
||||
assigned := []string{uplink}
|
||||
if node != "laptop" {
|
||||
assigned = append(assigned, "dnsmasq")
|
||||
}
|
||||
servers, content := resolvConfOn(t, node, uplink, assigned)
|
||||
if strings.Join(servers, " ") != strings.Join(want, " ") {
|
||||
t.Errorf("%s on %s lists %v; want %v\n%s", uplink, node, servers, want, content)
|
||||
}
|
||||
for _, public := range []string{"1.1.1.1", "8.8.8.8", "9.9.9.9"} {
|
||||
if strings.Contains(content, public) {
|
||||
t.Errorf("%s on %s lists a public resolver beside the mesh's (ADR 0223):\n%s", uplink, node, content)
|
||||
}
|
||||
}
|
||||
if !strings.HasSuffix(content, "\noptions timeout:1 attempts:2 edns0\n") {
|
||||
t.Errorf("%s on %s does not end with two short attempts:\n%s", uplink, node, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One file, whichever manager the machine runs: the three modules carry the same template, so a
|
||||
// machine changing its manager changes nothing in what it asks, and a fix made to one is made to all.
|
||||
func TestEveryUplinkWritesTheSameResolverFile(t *testing.T) {
|
||||
var first, firstOf string
|
||||
for _, uplink := range uplinks {
|
||||
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
|
||||
if !ok || fact.Path != "/etc/resolv.conf" || fact.Shared || fact.Home {
|
||||
t.Fatalf("%s does not write the machine's resolver file whole: %+v", uplink, fact)
|
||||
}
|
||||
if first == "" {
|
||||
first, firstOf = fact.Template, uplink
|
||||
continue
|
||||
}
|
||||
if fact.Template != first {
|
||||
t.Errorf("%s's resolver file differs from %s's; the three are kept identical", uplink, firstOf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The requirement stays with what writes the file (ADR 0223 part 1): a machine is refused when nothing
|
||||
// in the mesh resolves, rather than given a file listing nothing.
|
||||
func TestEveryUplinkRequiresTheMeshsResolver(t *testing.T) {
|
||||
for _, uplink := range uplinks {
|
||||
found := false
|
||||
for _, r := range catalogueManifest(t, uplink).Requires {
|
||||
found = found || r == "wildcard-resolution"
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s writes the resolver file and does not require wildcard-resolution", uplink)
|
||||
}
|
||||
}
|
||||
_, err := Resolve(twoResolverShelf(t), []string{"networkmanager"}, managing("laptop"), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "wildcard-resolution") {
|
||||
t.Errorf("an uplink was composed on a mesh with no resolver: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder answers its own requirement, even though the other holder sorts first (issue 258 kept).
|
||||
func TestAHolderAnswersItsOwnRequirement(t *testing.T) {
|
||||
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq", "networkmanager"},
|
||||
managing("home"), worldWithout("home"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range got.Needs {
|
||||
if n.Name == "wildcard-resolution" && n.From != "home" {
|
||||
t.Errorf("the home server's uplink is bound to %s; it holds the seat itself", n.From)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat held once is still held once: a second claimant on another machine is refused while
|
||||
// nothing is on record, and a store recording two holders of it is refused, naming the seat.
|
||||
func TestASingleHolderMeshSeatStillRefusesASecondHolder(t *testing.T) {
|
||||
store := shelf(mod("postgres", nil, nil, nil, Claim{Name: "mesh-store", Scope: ScopeMesh}))
|
||||
other := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "postgres"}
|
||||
if _, err := Resolve(store, []string{"postgres"}, workstation(), World{Held: []Held{other}}); err == nil ||
|
||||
!strings.Contains(err.Error(), "one per mesh") {
|
||||
t.Errorf("a second claimant of a seat held once was not refused: %v", err)
|
||||
}
|
||||
here := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: workstation().Name, Module: "postgres"}
|
||||
_, err := Resolve(store, []string{"postgres"}, workstation(),
|
||||
World{Held: []Held{other}, Holdings: []Held{other, here}})
|
||||
if err == nil || !strings.Contains(err.Error(), "on record as held by 2") {
|
||||
t.Errorf("two holders on record for a seat held once were not refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Replicated is not "whoever is assigned": two claimants with nothing on record are refused, as for
|
||||
// any mesh seat, and each holder is added by an act.
|
||||
func TestTwoUnrecordedClaimantsOfTheReplicatedSeatAreRefused(t *testing.T) {
|
||||
w := worldWithout("home")
|
||||
w.Holdings = nil
|
||||
_, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "home", At: "home.internal"}, w)
|
||||
if err == nil || !strings.Contains(err.Error(), "seat mesh-dns-resolver --to") {
|
||||
t.Errorf("a second resolver with nothing on record was not refused, naming the handover: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the mesh's resolver is replicated: a seat being replicated is a decision, recorded.
|
||||
func TestOnlyTheResolverIsReplicated(t *testing.T) {
|
||||
for _, s := range Seats() {
|
||||
if s.Replicated != (s.Name == "mesh-dns-resolver") {
|
||||
t.Errorf("%s replicated = %v; only mesh-dns-resolver is (ADR 0223)", s.Name, s.Replicated)
|
||||
}
|
||||
}
|
||||
UseSeats([]Seat{{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution"}})
|
||||
defer UseSeats(DefaultSeats())
|
||||
if s, _ := SeatNamed("mesh-dns-resolver"); !s.Replicated {
|
||||
t.Error("loading the set from the store, which has no column for it, lost the resolver's replication")
|
||||
}
|
||||
}
|
||||
|
||||
// Every consumer is bound to the same holder whatever order the mesh resolved its machines in.
|
||||
func TestTheFirstHolderIsTheFirstProvider(t *testing.T) {
|
||||
providers := []Provider{{Node: "anchor", Module: "dnsmasq"}, {Node: "home", Module: "dnsmasq"}}
|
||||
for _, held := range [][]Held{bothResolvers, {bothResolvers[1], bothResolvers[0]}} {
|
||||
if p, ok := HolderAmong("wildcard-resolution", providers, held); !ok || p.Node != "anchor" {
|
||||
t.Errorf("held in order %v answered %v", held, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One host record per machine, beside its wildcard (novox/hq issue 262): a name with a host record
|
||||
// says it exists and has no IPv6 address, where the wildcard alone said there is no such name, and
|
||||
// musl reads that as final.
|
||||
func TestTheResolverHasOneHostRecordPerMachine(t *testing.T) {
|
||||
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor", At: "anchor.internal"},
|
||||
World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, _ := byID(out)["dnsmasq.fact-node-zones"]["content"].(string)
|
||||
records := map[string]int{}
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
if strings.HasPrefix(line, "host-record=") {
|
||||
records[strings.TrimPrefix(line, "host-record=")]++
|
||||
}
|
||||
}
|
||||
for name, at := range resolverMachines {
|
||||
if records[name+","+at] != 1 {
|
||||
t.Errorf("%s has %d host records at %s, and has one:\n%s", name, records[name+","+at], at, content)
|
||||
}
|
||||
}
|
||||
if len(records) != len(resolverMachines) {
|
||||
t.Errorf("%d host records for %d machines:\n%s", len(records), len(resolverMachines), content)
|
||||
}
|
||||
}
|
||||
+105
-15
@@ -73,6 +73,13 @@ var ControllerVerbs = []Verb{
|
||||
{Name: "tools", Description: "Every seat's tools, from the mesh's own records: what each role " +
|
||||
"answers, whether or not its holder is up. The mesh's own verbs are the mesh-controller seat's.",
|
||||
Input: schema(nil, nil)},
|
||||
{Name: "calls", Description: "The calls this controller answered lately and what came of each — " +
|
||||
"one still running, one that finished after its caller was told it was running, one whose answer " +
|
||||
"the bus refused — and, given a call's id, its whole answer (novox/hq issue 265). A call that has " +
|
||||
"not finished within ten seconds answers that it is running, with its id; this is where it ends.",
|
||||
Input: schema(map[string]string{
|
||||
"call": "a call's id, as a running answer or `calls` gives it: that call and its whole answer",
|
||||
}, nil)},
|
||||
{Name: "status", Description: "What is wrong, what is quiet, what is out of date, and which " +
|
||||
"machines are behind what the mesh would send them.",
|
||||
Input: schema(nil, nil)},
|
||||
@@ -90,6 +97,7 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{
|
||||
"module": "one module's name; every module when absent",
|
||||
"log": "a build's id (as `builds` lists it): print what the build machine said, line by line",
|
||||
"limit": "how many builds to list (default 20); not with log",
|
||||
}, nil)},
|
||||
{Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " +
|
||||
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id.",
|
||||
@@ -101,9 +109,16 @@ var ControllerVerbs = []Verb{
|
||||
"repository": "owner/repository: the plan a merge there would produce, saving nothing (what-if); with paths or modules",
|
||||
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
||||
"modules": "with repository: or the modules it would change, comma-separated",
|
||||
"limit": "how many plans to list (default 10); only when listing",
|
||||
"why": "with stop or close: why it is ended by hand — required, and recorded in the hand-act log (novox/hq to-be 45 §7)",
|
||||
"cause": "with stop or close: the cause in a word, or a condition's kind (optional)",
|
||||
}, nil)},
|
||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it.",
|
||||
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})},
|
||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
||||
"or, with files, the files it would be given.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name",
|
||||
"files": "\"true\": the files this machine would be given, instead of the declaration as JSON",
|
||||
}, []string{"node"}, "files")},
|
||||
{Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " +
|
||||
"or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207).",
|
||||
Input: schema(map[string]string{"node": "the machine's name",
|
||||
@@ -121,18 +136,30 @@ var ControllerVerbs = []Verb{
|
||||
}, []string{"node", "provision", "from", "module"})},
|
||||
{Name: "unpin", Description: "Take that choice back, putting the question to the mesh again.",
|
||||
Input: schema(map[string]string{"node": "the machine's name", "provision": "the provision"}, []string{"node", "provision"})},
|
||||
{Name: "push", Description: "Send a machine everything it should be — or every machine that is behind, when no machine is named.",
|
||||
Input: schema(map[string]string{"node": "the machine's name; every machine behind when absent"}, nil)},
|
||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine, " +
|
||||
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
||||
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
||||
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
|
||||
"(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
|
||||
"and says why: recorded in the hand-act log (novox/hq to-be 45 §7).",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name; without it, every machine that is behind",
|
||||
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
||||
"why": "why this is pushed by hand: recorded in the hand-act log",
|
||||
"cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)",
|
||||
}, []string{"why"}, "behind")},
|
||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
|
||||
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
||||
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
|
||||
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
|
||||
"name: made anew and the machine sent, so the module starts again on it — for a secret its definition " +
|
||||
"says it reads at start, whether the mesh made the value or a person gave it (novox/hq ADR 0228); one " +
|
||||
"an outside party issued, or one the module applies to a backend, is refused with the reason.",
|
||||
Input: schema(map[string]string{
|
||||
"provision": "a pair credential: the provision whose credential to replace",
|
||||
"consumer": "with provision: only the holder on this machine (optional)",
|
||||
"node": "an own secret: the machine",
|
||||
"module": "an own secret: the module",
|
||||
"module": "an own secret: the module; with provision: only this consuming module's credential (optional)",
|
||||
"secret": "an own secret: its name in the module's definition",
|
||||
"why": "an own secret: why it is rotated — recorded in the hand-act log (optional)",
|
||||
"cause": "with why: the cause in a word, the word a second rotation for the same reason uses (optional)",
|
||||
}, nil)},
|
||||
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
||||
"machine as the module's own secret named broker, read at the next push of that machine. For a module " +
|
||||
@@ -148,8 +175,8 @@ var ControllerVerbs = []Verb{
|
||||
"module": "the module's name",
|
||||
"values": "the settings as a JSON object, for set",
|
||||
"node": "one machine; the whole mesh when absent",
|
||||
"clear": "\"true\" to remove the layer instead of setting it",
|
||||
}, []string{"module"})},
|
||||
"clear": "\"true\" to remove the layer instead of setting it; not with values",
|
||||
}, []string{"module"}, "clear")},
|
||||
{Name: "command", Description: "Run one command line of the controller's own, as you would type it at its " +
|
||||
"shell — `node account g14 jochen`, `node show ace`, `module list` — and answer what it printed. The " +
|
||||
"generic verb beside the named ones (novox/hq ADR 0154): everything the binary can do, without a verb " +
|
||||
@@ -167,7 +194,7 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{"id": "the ask's build id, as `queue` lists it"}, []string{"id"})},
|
||||
{Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " +
|
||||
"recorded failed, cancelled by hand. Never touches one in flight.",
|
||||
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil)},
|
||||
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")},
|
||||
{Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " +
|
||||
"build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.",
|
||||
Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})},
|
||||
@@ -178,7 +205,7 @@ var ControllerVerbs = []Verb{
|
||||
"id": "the build's id",
|
||||
"register": "\"true\" to register what it builds",
|
||||
"older": "\"true\", with register: even though a newer build of the module is registered",
|
||||
}, []string{"id"})},
|
||||
}, []string{"id"}, "register", "older")},
|
||||
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
|
||||
"announces it failed, killed by hand — settled, never handed to another machine.",
|
||||
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
|
||||
@@ -187,6 +214,53 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
|
||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||
// Acts done by hand, and what the bounds are set from (novox/hq to-be 45 §7, Phase 0).
|
||||
{Name: "hand-act", Description: "Record an act done by hand outside the mesh — a container restarted, a file " +
|
||||
"edited, a service started on a machine — with why and its cause, in the hand-act log beside the pushes and " +
|
||||
"plans ended by hand (novox/hq to-be 45 §7). A cause recorded twice in a fortnight is a healer wanted.",
|
||||
Input: schema(map[string]string{
|
||||
"what": "what was done, in a line",
|
||||
"why": "why it had to be done by hand",
|
||||
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
||||
"condition": "the key of the condition it addressed, if any (optional)",
|
||||
}, []string{"what", "why", "cause"})},
|
||||
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
||||
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
||||
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
||||
{Name: "durations", Description: "How long things take, as the controller measured them: a send to its machine's " +
|
||||
"report (apply), a machine's silence between words (heartbeat-gap), a plan's tier, a build — per machine, " +
|
||||
"repository or module, with median, p90 and max. What the core's bounds are set from (novox/hq to-be 45 Phase 0).",
|
||||
Input: schema(map[string]string{
|
||||
"kind": "one kind: apply, heartbeat-gap, plan-tier or build; every kind when absent",
|
||||
"days": "how many days back (default 14)",
|
||||
}, nil)},
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
{Name: "conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, " +
|
||||
"then oldest — raised by the watchdogs of the signals table, the self-check's probes and the providers' " +
|
||||
"own words, and cleared when observation says it is resolved, never by hand. Given a key, that one " +
|
||||
"whole with its evidence; with history, every transition lately; with silence, stop one's messages " +
|
||||
"for a while — a hand act, which says why (novox/hq to-be 45 §2).",
|
||||
Input: schema(map[string]string{
|
||||
"key": "a condition's key: that one whole; with history, only its transitions",
|
||||
"scope": "only this scope: machine, plan, call, build, merge, provider, seat, bus, core, probe or mesh",
|
||||
"severity": "only urgent, or only warning",
|
||||
"machine": "only those about this machine",
|
||||
"history": "\"true\": every raising, change, silence and clearing lately, oldest first",
|
||||
"days": "with history: how many days back (default 7, at most 90)",
|
||||
"silence": "a condition's key: send no message for it for a while; it stays open and in status",
|
||||
"for": "with silence: how long — 30m, 4h, 2d; at most 7d",
|
||||
"why": "with silence: why — required, and recorded in the hand-act log",
|
||||
"cause": "with silence: the cause in a word (the condition's kind when absent)",
|
||||
}, nil, "history")},
|
||||
{Name: "doctor", Description: "The self-check (novox/hq to-be 45 §4): the last run's verdict at once — " +
|
||||
"each probe of the design's live invariants passed, failed or could not run, and how long ago. With " +
|
||||
"run, a run now; with probes, the registry; with signals, every row of the signals table and the age " +
|
||||
"of its newest signal. Runs every five minutes on its own; each failure is an open condition.",
|
||||
Input: schema(map[string]string{
|
||||
"run": "\"true\": run every probe now and answer the verdict",
|
||||
"probes": "\"true\": the registry — what each probe asserts, and the condition it raises",
|
||||
"signals": "\"true\": the signals table, each row with the age of its newest signal",
|
||||
}, nil, "run", "probes", "signals")},
|
||||
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
||||
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
||||
Input: schema(map[string]string{
|
||||
@@ -197,11 +271,27 @@ var ControllerVerbs = []Verb{
|
||||
}
|
||||
|
||||
// schema is a JSON schema for an object of string properties, which is every argument the verbs
|
||||
// above take. Kept small on purpose: a schema an agent cannot read is a tool it cannot call.
|
||||
func schema(properties map[string]string, required []string) map[string]any {
|
||||
// above take. Kept small on purpose: a schema an agent cannot read is a tool it cannot call. The
|
||||
// switches are the properties that are "true" or "false" and nothing else, said in the schema as an
|
||||
// enum so a caller sees it and the verb can refuse any other word rather than read it as false.
|
||||
//
|
||||
// **The schema is the whole of what a verb takes** (novox/hq issue 244): an argument it does not
|
||||
// name is refused when the verb is called, never passed over.
|
||||
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
|
||||
isSwitch := map[string]bool{}
|
||||
for _, s := range switches {
|
||||
if _, declared := properties[s]; !declared {
|
||||
panic("a switch that is not a property: " + s)
|
||||
}
|
||||
isSwitch[s] = true
|
||||
}
|
||||
props := map[string]any{}
|
||||
for name, description := range properties {
|
||||
props[name] = map[string]any{"type": "string", "description": description}
|
||||
p := map[string]any{"type": "string", "description": description}
|
||||
if isSwitch[name] {
|
||||
p["enum"] = []string{"true", "false"}
|
||||
}
|
||||
props[name] = p
|
||||
}
|
||||
out := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The condition store on the bus (to-be 45 §2, ADR 0201): the controller's two buckets, asserted at
|
||||
// its start like its calls and its hand-act log.
|
||||
|
||||
// OnTheBus opens the store and its history on a connection.
|
||||
func OnTheBus(ctx context.Context, conn *nats.Conn) (Backend, History, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
open, err := api.KeyValue(ctx, broker.ConditionsBucket)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the condition store %s is not on the bus — the controller asserts it at "+
|
||||
"its start, so one older than this has not: %w", broker.ConditionsBucket, err)
|
||||
}
|
||||
history, err := api.KeyValue(ctx, broker.ConditionHistoryBucket)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the condition history %s is not on the bus — the controller asserts it "+
|
||||
"at its start, so one older than this has not: %w", broker.ConditionHistoryBucket, err)
|
||||
}
|
||||
return busStore{open}, &busHistory{api: api, kv: history}, nil
|
||||
}
|
||||
|
||||
type busStore struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (b busStore) Get(ctx context.Context, key string) (Entry, bool, error) {
|
||||
e, err := b.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Entry{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Entry{}, false, err
|
||||
}
|
||||
return Entry{Value: e.Value(), Revision: e.Revision()}, true, nil
|
||||
}
|
||||
|
||||
func (b busStore) Create(ctx context.Context, key string, value []byte) error {
|
||||
_, err := b.kv.Create(ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return ErrMoved
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
_, err := b.kv.Update(ctx, key, value, revision)
|
||||
return moved(err)
|
||||
}
|
||||
|
||||
func (b busStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||
return moved(b.kv.Delete(ctx, key, jetstream.LastRevision(revision)))
|
||||
}
|
||||
|
||||
// moved reads the server's refusal of a compare-and-set as what it is.
|
||||
func moved(err error) error {
|
||||
var apiErr *jetstream.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence {
|
||||
return ErrMoved
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// All is every key, read through a watch that hands over each current value and then says it has.
|
||||
func (b busStore) All(ctx context.Context) (map[string]Entry, error) {
|
||||
w, err := b.kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
out := map[string]Entry{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("reading the condition store: %w", ctx.Err())
|
||||
case e := <-w.Updates():
|
||||
if e == nil {
|
||||
return out, nil
|
||||
}
|
||||
out[e.Key()] = Entry{Value: e.Value(), Revision: e.Revision()}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// busHistory keeps each transition under a key of its time and a sequence, and reads them back
|
||||
// from a moment through the stream under the bucket — by time, so a read of the last ten minutes
|
||||
// does not read ninety days.
|
||||
type busHistory struct {
|
||||
api jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
seq atomic.Uint64
|
||||
}
|
||||
|
||||
func (h *busHistory) Append(ctx context.Context, e Event) error {
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := strconv.FormatInt(e.At.UnixNano(), 10) + "-" + strconv.FormatUint(h.seq.Add(1), 10)
|
||||
_, err = h.kv.Put(ctx, key, body)
|
||||
return err
|
||||
}
|
||||
|
||||
// historyQuiet is how long a read of the history waits for one more transition before it takes the
|
||||
// stream as read to its end; it answers at once while it holds something.
|
||||
const historyQuiet = 2 * time.Second
|
||||
|
||||
func (h *busHistory) Since(ctx context.Context, since time.Time) ([]Event, error) {
|
||||
start := since
|
||||
consumer, err := h.api.OrderedConsumer(ctx, "KV_"+broker.ConditionHistoryBucket, jetstream.OrderedConsumerConfig{
|
||||
DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &start,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||
}
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||
}
|
||||
var out []Event
|
||||
pending := info.NumPending
|
||||
for pending > 0 {
|
||||
msg, err := consumer.Next(jetstream.FetchMaxWait(historyQuiet))
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// Nothing more within the quiet wait: read to its end.
|
||||
break
|
||||
}
|
||||
meta, err := msg.Metadata()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
pending = meta.NumPending
|
||||
var e Event
|
||||
if len(msg.Data()) > 0 && json.Unmarshal(msg.Data(), &e) == nil && e.Key != "" {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The condition store against a real server: compare-and-set, an unreadable store refused, and the
|
||||
// history read back by time are claims about what the bus does.
|
||||
|
||||
func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = api.DeleteKeyValue(t.Context(), broker.ConditionsBucket)
|
||||
_ = api.DeleteKeyValue(t.Context(), broker.ConditionHistoryBucket)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, history, err := OnTheBus(t.Context(), js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js, store, history
|
||||
}
|
||||
|
||||
// **A condition outlives the controller that raised it**, and two writers on the bus cannot lose each
|
||||
// other's word: a stale revision is refused as moved.
|
||||
func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
|
||||
_, store, history := busStoreForTest(t)
|
||||
ctx := t.Context()
|
||||
told := &Told{}
|
||||
k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told})
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k.Close(context.Background())
|
||||
|
||||
again := NewKeeper(ctx, Options{Store: store, History: history})
|
||||
defer again.Close(context.Background())
|
||||
open, err := again.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Observations != 2 {
|
||||
t.Fatalf("a new keeper read %+v", open)
|
||||
}
|
||||
e, _, err := store.Get(ctx, "machine.ace.silent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved {
|
||||
t.Fatalf("a write at a stale revision answered %v", err)
|
||||
}
|
||||
if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved {
|
||||
t.Fatalf("creating an open condition answered %v", err)
|
||||
}
|
||||
if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved {
|
||||
t.Fatalf("a delete at a stale revision answered %v", err)
|
||||
}
|
||||
if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared {
|
||||
t.Fatalf("cleared %v: %v", cleared, err)
|
||||
}
|
||||
// Raised again at once: the store takes a key whose last word was a delete.
|
||||
if _, err := again.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := again.Get(ctx, "machine.ace.silent")
|
||||
if got.Count != 2 {
|
||||
t.Fatalf("raised again after its clearing as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The history is read back from a moment, oldest first**, through the stream under its bucket.
|
||||
func TestNatsTheHistoryIsReadByTime(t *testing.T) {
|
||||
_, store, history := busStoreForTest(t)
|
||||
ctx := t.Context()
|
||||
k := NewKeeper(ctx, Options{Store: store, History: history})
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k.Close(context.Background())
|
||||
all, err := history.Since(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(all) != 2 || all[0].Change != ChangeRaised || all[1].Change != ChangeCleared {
|
||||
t.Fatalf("history %+v", all)
|
||||
}
|
||||
none, err := history.Since(ctx, time.Now().Add(time.Hour))
|
||||
if err != nil || len(none) != 0 {
|
||||
t.Fatalf("history from the future: %+v %v", none, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
// Package conditions is the condition store (novox/hq to-be 45 §2, ADR 0227 rules 5 and 6).
|
||||
//
|
||||
// **A condition is a durable fact about something the mesh owns that is wrong.** Until this, every
|
||||
// one of the forty-eight core failures of research 031 was noticed because a person or an agent
|
||||
// looked: the mesh's own answers carried the fact for whoever asked, and told nobody. A condition is
|
||||
// raised when an observation says something is wrong past its bound, kept with since-when, evidence
|
||||
// and who can resolve it, said on the bus as it changes, and cleared when an observation says it is
|
||||
// resolved — never by hand.
|
||||
//
|
||||
// The controller is the store's only writer (to-be 45 §1). Two of its processes may write at once —
|
||||
// the serving controller's watchdogs, and a command a person runs to silence one — so every write
|
||||
// is a compare-and-set on the key's revision, and a write that lost the race reads again and redoes
|
||||
// itself rather than overwriting what the other said.
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Severity is how soon the operator is needed: two levels, no more (to-be 45 §2).
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
// Urgent needs the operator now.
|
||||
Urgent Severity = "urgent"
|
||||
// Warning needs the operator when they can.
|
||||
Warning Severity = "warning"
|
||||
)
|
||||
|
||||
// The scopes a condition's key starts with: what kind of thing is wrong.
|
||||
const (
|
||||
ScopeMachine = "machine"
|
||||
ScopePlan = "plan"
|
||||
ScopeCall = "call"
|
||||
ScopeBuild = "build"
|
||||
ScopeMerge = "merge"
|
||||
ScopeProvider = "provider"
|
||||
ScopeSeat = "seat"
|
||||
ScopeBus = "bus"
|
||||
ScopeCore = "core"
|
||||
ScopeProbe = "probe"
|
||||
ScopeMesh = "mesh"
|
||||
)
|
||||
|
||||
// Scopes is every scope, in the order a person reads them.
|
||||
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh}
|
||||
|
||||
// Who resolves a condition.
|
||||
const (
|
||||
// ResolverSelf clears on observation: the signal returns, the probe passes.
|
||||
ResolverSelf = "self"
|
||||
// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
|
||||
ResolverOperator = "operator"
|
||||
// ResolverAgent is work handed to an agent (research 017; not raised by anything yet).
|
||||
ResolverAgent = "agent"
|
||||
)
|
||||
|
||||
// ResolverHealer is the resolver of a condition a registered healer works on (Phase 3).
|
||||
func ResolverHealer(name string) string { return "healer:" + name }
|
||||
|
||||
// Subject is what the condition is about: its scope, its id within the scope, and the machine it
|
||||
// concerns when there is one.
|
||||
type Subject struct {
|
||||
Scope string `json:"scope"`
|
||||
ID string `json:"id"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
// Also are the other machines it concerns: a consumer's, for a provider failing it.
|
||||
Also []string `json:"also,omitempty"`
|
||||
}
|
||||
|
||||
// Evidence is one observation, as it was said.
|
||||
type Evidence struct {
|
||||
At time.Time `json:"at"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// Attempt is one healer's try at a condition (to-be 45 §7; written from Phase 3).
|
||||
type Attempt struct {
|
||||
At time.Time `json:"at"`
|
||||
What string `json:"what"`
|
||||
Outcome string `json:"outcome"`
|
||||
}
|
||||
|
||||
// Silence is a person saying they know: no messages until it ends (to-be 45 §2). Recorded as a hand
|
||||
// act; the condition stays open, and `status` still says it.
|
||||
type Silence struct {
|
||||
Until time.Time `json:"until"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// KeptEvidence is how many observations a condition keeps, newest first.
|
||||
const KeptEvidence = 10
|
||||
|
||||
// MaxSilence is the longest a condition may be silenced at once: past it, a person says so again.
|
||||
const MaxSilence = 7 * 24 * time.Hour
|
||||
|
||||
// ReopenWithin is how soon after it cleared a condition raised again is the same one again, with its
|
||||
// count increased, rather than news (to-be 45 §2).
|
||||
const ReopenWithin = 10 * time.Minute
|
||||
|
||||
// Condition is one open condition, as the store keeps it and its events carry it.
|
||||
type Condition struct {
|
||||
Key string `json:"key"`
|
||||
// Kind is the condition kind: from the signals table, the probe registry or an event kind.
|
||||
Kind string `json:"kind"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity Severity `json:"severity"`
|
||||
// Summary is one line in the mesh's words.
|
||||
Summary string `json:"summary"`
|
||||
// Evidence is the newest observations, at most KeptEvidence, newest first.
|
||||
Evidence []Evidence `json:"evidence"`
|
||||
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||
Source string `json:"source"`
|
||||
// Raised is when it was first observed this time; LastObserved the newest observation.
|
||||
Raised time.Time `json:"raised"`
|
||||
LastObserved time.Time `json:"last-observed"`
|
||||
// Observations is how many times it was observed since raised.
|
||||
Observations int `json:"observations"`
|
||||
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
|
||||
Count int `json:"count"`
|
||||
Tried []Attempt `json:"tried,omitempty"`
|
||||
Resolver string `json:"resolver"`
|
||||
// Silenced is null when no silence is in force: said, not left out, so a reader need not guess.
|
||||
Silenced *Silence `json:"silenced"`
|
||||
// Epoch is the controller lease epoch that last wrote it (to-be 45 §6). Zero where it was written
|
||||
// by a controller serving without the lease, or before the lease existed.
|
||||
Epoch uint64 `json:"epoch"`
|
||||
}
|
||||
|
||||
// SilencedAt says whether a person's silence is in force at a moment.
|
||||
func (c Condition) SilencedAt(now time.Time) bool {
|
||||
return c.Silenced != nil && now.Before(c.Silenced.Until)
|
||||
}
|
||||
|
||||
// Show is the verb that shows more about a condition, as a message carries it.
|
||||
func (c Condition) Show() string { return "mesh-controller.conditions key=" + c.Key }
|
||||
|
||||
// Observation is one watchdog, probe or event saying something is wrong now.
|
||||
type Observation struct {
|
||||
Scope string
|
||||
// ID is the thing within the scope; several tokens joined by dots where the thing is named by
|
||||
// several (a provider's module, its machine and the consumer).
|
||||
ID string
|
||||
// Token is the last part of the key, short for the kind: `silent` for a machine, `failing` for a
|
||||
// provider. Kind's own word when empty.
|
||||
Token string
|
||||
Kind string
|
||||
Machine string
|
||||
// Also are the other machines it concerns.
|
||||
Also []string
|
||||
Severity Severity
|
||||
Summary string
|
||||
// Said is this observation's evidence, in the mesh's words; Summary when empty.
|
||||
Said string
|
||||
Source string
|
||||
Resolver string
|
||||
}
|
||||
|
||||
// Key is where the observation's condition is kept: `<scope>.<id>.<kind>`, so the same fault said
|
||||
// again is the same condition.
|
||||
func (o Observation) Key() string {
|
||||
token := o.Token
|
||||
if token == "" {
|
||||
token = o.Kind
|
||||
}
|
||||
return Key(o.Scope, o.ID, token)
|
||||
}
|
||||
|
||||
// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's
|
||||
// tokens, and a `*` or `>` would make one a wildcard.
|
||||
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||||
|
||||
// Key composes a condition's key from its parts, each token made safe for the bus: a character the
|
||||
// bus would refuse becomes `_`, so a key is never refused for the name of the thing it is about.
|
||||
func Key(scope, id, token string) string {
|
||||
var parts []string
|
||||
for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
|
||||
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
|
||||
if p == "" {
|
||||
p = "_"
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
// check refuses an observation that could not be said: a condition with no kind, no scope the mesh
|
||||
// knows, or no severity is one nobody could route.
|
||||
func (o Observation) check() error {
|
||||
known := false
|
||||
for _, s := range Scopes {
|
||||
if s == o.Scope {
|
||||
known = true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !known:
|
||||
return fmt.Errorf("a condition's scope is one of %s, not %q", strings.Join(Scopes, ", "), o.Scope)
|
||||
case strings.TrimSpace(o.ID) == "":
|
||||
return fmt.Errorf("a %s condition names what it is about", o.Scope)
|
||||
case strings.TrimSpace(o.Kind) == "":
|
||||
return fmt.Errorf("the condition %s has no kind", o.Key())
|
||||
case o.Severity != Urgent && o.Severity != Warning:
|
||||
return fmt.Errorf("the condition %s is urgent or a warning, not %q", o.Key(), o.Severity)
|
||||
case strings.TrimSpace(o.Summary) == "":
|
||||
return fmt.Errorf("the condition %s says nothing", o.Key())
|
||||
case strings.TrimSpace(o.Source) == "":
|
||||
return fmt.Errorf("the condition %s does not say what raised it", o.Key())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Order sorts conditions as `status` says them: urgent before warning, then oldest first.
|
||||
func Order(list []Condition) {
|
||||
sort.SliceStable(list, func(i, j int) bool {
|
||||
if list[i].Severity != list[j].Severity {
|
||||
return list[i].Severity == Urgent
|
||||
}
|
||||
if !list[i].Raised.Equal(list[j].Raised) {
|
||||
return list[i].Raised.Before(list[j].Raised)
|
||||
}
|
||||
return list[i].Key < list[j].Key
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package conditions
|
||||
|
||||
import "time"
|
||||
|
||||
// The events a condition's life emits (to-be 45 §2), as the mesh-controller seat's own: published on
|
||||
// `mesh.seat.mesh-controller.event.<name>`, on the events stream, so a consumer that was away catches
|
||||
// up. **This is a contract**: the operator-channel's holder is written against these names and the
|
||||
// shape of Event, and the controller learns nothing about telling.
|
||||
const (
|
||||
// EventRaised: a condition was raised — new, or the same fault again within ReopenWithin of its
|
||||
// clearing (Change says which). A reopened condition is not news: its key is the one the
|
||||
// first message was about.
|
||||
EventRaised = "condition-raised"
|
||||
// EventChanged: its severity, its resolver or its silence changed. Not every observation: a
|
||||
// condition observed again is written, and says nothing.
|
||||
EventChanged = "condition-changed"
|
||||
// EventCleared: an observation says it is resolved. The condition is removed from the store and
|
||||
// the transition kept in its history.
|
||||
EventCleared = "condition-cleared"
|
||||
)
|
||||
|
||||
// Events is every event a condition's life emits.
|
||||
var Events = []string{EventRaised, EventChanged, EventCleared}
|
||||
|
||||
// HeartbeatEvent is the self-check's heartbeat (to-be 45 §4, S10), said under the same seat at the end
|
||||
// of every run: `{run, at, interval-seconds, counts: {passed, failed, failed-to-run, deferred}, probes,
|
||||
// controller, why}`. mesh-watcher, on a machine that is not the control node, listens for it.
|
||||
const HeartbeatEvent = "doctor-heartbeat"
|
||||
|
||||
// What changed, as an event's Change and a history entry's says it.
|
||||
const (
|
||||
ChangeRaised = "raised"
|
||||
ChangeReopened = "reopened"
|
||||
ChangeSeverity = "severity"
|
||||
ChangeResolver = "resolver"
|
||||
ChangeSilenced = "silenced"
|
||||
ChangeUnsilenced = "silence-ended"
|
||||
ChangeCleared = "cleared"
|
||||
)
|
||||
|
||||
// Event is the body of every condition event, and the shape a history entry keeps: **the condition
|
||||
// itself, at the top level** — key, kind, subject, severity, summary, source, raised, last-observed,
|
||||
// observations, resolver, silenced (null when not), epoch, and the evidence — with what happened to
|
||||
// it beside. One object a consumer reads the same way whichever of the three it is.
|
||||
type Event struct {
|
||||
// Condition is the condition after the transition — as it was last held, for a clearing.
|
||||
Condition
|
||||
// Event is the event's own name, so a body read without its subject still says what it is.
|
||||
Event string `json:"event"`
|
||||
// At is when the transition happened.
|
||||
At time.Time `json:"at"`
|
||||
// Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, cleared.
|
||||
Change string `json:"change"`
|
||||
// Was is the value before, for a severity or resolver change.
|
||||
Was string `json:"was,omitempty"`
|
||||
// Why says why it cleared, or why it was silenced.
|
||||
Why string `json:"why,omitempty"`
|
||||
// Cleared is when it cleared, on a clearing.
|
||||
Cleared *time.Time `json:"cleared,omitempty"`
|
||||
// Show is the verb that shows more.
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// eventFor is the event a change is said under.
|
||||
func eventFor(change string) string {
|
||||
switch change {
|
||||
case ChangeRaised, ChangeReopened:
|
||||
return EventRaised
|
||||
case ChangeCleared:
|
||||
return EventCleared
|
||||
}
|
||||
return EventChanged
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// InMemory is a store and a history held in this process: for tests, and for nothing else — a
|
||||
// condition kept here is forgotten by a restart, which is the fault the store exists to remove.
|
||||
type InMemory struct {
|
||||
mu sync.Mutex
|
||||
values map[string]Entry
|
||||
revision uint64
|
||||
events []Event
|
||||
// Fail, when set, is what every read and write answers: a store that is away.
|
||||
Fail error
|
||||
}
|
||||
|
||||
// NewInMemory is an empty store.
|
||||
func NewInMemory() *InMemory { return &InMemory{values: map[string]Entry{}} }
|
||||
|
||||
func (m *InMemory) Get(_ context.Context, key string) (Entry, bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return Entry{}, false, m.Fail
|
||||
}
|
||||
e, ok := m.values[key]
|
||||
return e, ok, nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Create(_ context.Context, key string, value []byte) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if _, ok := m.values[key]; ok {
|
||||
return ErrMoved
|
||||
}
|
||||
m.revision++
|
||||
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Update(_ context.Context, key string, value []byte, revision uint64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||
return ErrMoved
|
||||
}
|
||||
m.revision++
|
||||
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Delete(_ context.Context, key string, revision uint64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||
return ErrMoved
|
||||
}
|
||||
delete(m.values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) All(context.Context) (map[string]Entry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return nil, m.Fail
|
||||
}
|
||||
out := make(map[string]Entry, len(m.values))
|
||||
for k, v := range m.values {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Append(_ context.Context, e Event) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
m.events = append(m.events, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Since(_ context.Context, since time.Time) ([]Event, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return nil, m.Fail
|
||||
}
|
||||
var out []Event
|
||||
for _, e := range m.events {
|
||||
if !e.At.Before(since) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Told is a teller that remembers what it was told, for tests.
|
||||
type Told struct {
|
||||
mu sync.Mutex
|
||||
Events []Event
|
||||
Names []string
|
||||
Fail error
|
||||
}
|
||||
|
||||
func (t *Told) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.Fail != nil {
|
||||
return t.Fail
|
||||
}
|
||||
if seat != Seat {
|
||||
return errors.New("told under the wrong seat: " + seat)
|
||||
}
|
||||
var e Event
|
||||
if err := json.Unmarshal(body, &e); err != nil {
|
||||
return err
|
||||
}
|
||||
t.Events = append(t.Events, e)
|
||||
t.Names = append(t.Names, event)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Said is a copy of what was told so far.
|
||||
func (t *Told) Said() []Event {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return append([]Event(nil), t.Events...)
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Backend is where the open conditions are kept: one value per key, written by compare-and-set.
|
||||
type Backend interface {
|
||||
// Get is one key's value and revision; false when it holds none.
|
||||
Get(ctx context.Context, key string) (Entry, bool, error)
|
||||
// Create writes a key that holds nothing, and fails with ErrMoved when it holds something.
|
||||
Create(ctx context.Context, key string, value []byte) error
|
||||
// Update writes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||
Update(ctx context.Context, key string, value []byte, revision uint64) error
|
||||
// Delete removes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||
Delete(ctx context.Context, key string, revision uint64) error
|
||||
// All is every key's value. An error is an error: never an empty store (ADR 0227 rule 4).
|
||||
All(ctx context.Context) (map[string]Entry, error)
|
||||
}
|
||||
|
||||
// Entry is one key's value, at a revision.
|
||||
type Entry struct {
|
||||
Value []byte
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read.
|
||||
var ErrMoved = errors.New("the condition was written by somebody else since it was read")
|
||||
|
||||
// History keeps every transition (to-be 45 §2): appended, read back from a moment.
|
||||
type History interface {
|
||||
Append(ctx context.Context, e Event) error
|
||||
// Since is every transition from a moment, oldest first.
|
||||
Since(ctx context.Context, since time.Time) ([]Event, error)
|
||||
}
|
||||
|
||||
// Teller says a transition on the bus, as the mesh-controller seat's event. The link's bus is one.
|
||||
type Teller interface {
|
||||
PublishSeatEvent(ctx context.Context, seat, event string, body []byte) error
|
||||
}
|
||||
|
||||
// Seat is the role the events are said under (novox/hq ADR 0134): the control plane's.
|
||||
const Seat = "mesh-controller"
|
||||
|
||||
// Keeper raises, observes, silences and clears conditions, and says each transition.
|
||||
type Keeper struct {
|
||||
store Backend
|
||||
history History
|
||||
teller Teller
|
||||
now func() time.Time
|
||||
say func(format string, args ...any)
|
||||
changed func()
|
||||
epoch func() (uint64, error)
|
||||
|
||||
mu sync.Mutex
|
||||
// cleared is when each recently cleared condition cleared and how often it had been raised, so
|
||||
// one raised again within ReopenWithin is the same one again.
|
||||
cleared map[string]clearing
|
||||
|
||||
// out is the transitions still to be said and kept, in order: said by one goroutine, so a
|
||||
// condition's events arrive in the order they happened, and offered again while the bus is away.
|
||||
out chan Event
|
||||
drained chan struct{}
|
||||
closing sync.Once
|
||||
// Unsaid counts the transitions given up on, for the self-check to say.
|
||||
unsaid int
|
||||
}
|
||||
|
||||
type clearing struct {
|
||||
at time.Time
|
||||
count int
|
||||
silenced *Silence
|
||||
}
|
||||
|
||||
// Options are what a Keeper is made with.
|
||||
type Options struct {
|
||||
Store Backend
|
||||
History History
|
||||
// Teller says the transitions; nil says nothing (a test, or a command run with no bus to say on).
|
||||
Teller Teller
|
||||
Now func() time.Time
|
||||
// Say is where a transition that could not be said or kept is said instead.
|
||||
Say func(format string, args ...any)
|
||||
// Changed is told of every transition, at once — for `status`, which leads with what is open.
|
||||
Changed func()
|
||||
// Epoch is the controller lease this keeper writes under (to-be 45 §6): every write carries its
|
||||
// epoch, and none is made while it is not held. Nil writes with no epoch (a test, the memory store).
|
||||
Epoch func() (uint64, error)
|
||||
}
|
||||
|
||||
// TellFor is how long one transition is offered to the bus before it is said lost.
|
||||
var TellFor = 10 * time.Minute
|
||||
|
||||
// NewKeeper is a keeper over a store. It reads what cleared lately from the history, so a condition
|
||||
// that cleared just before this controller started and is raised again now is a reopening.
|
||||
func NewKeeper(ctx context.Context, o Options) *Keeper {
|
||||
k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
|
||||
epoch: o.Epoch,
|
||||
cleared: map[string]clearing{}, out: make(chan Event, 1024), drained: make(chan struct{})}
|
||||
if k.now == nil {
|
||||
k.now = time.Now
|
||||
}
|
||||
if k.say == nil {
|
||||
k.say = func(string, ...any) {}
|
||||
}
|
||||
if k.history != nil {
|
||||
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
|
||||
for _, e := range recent {
|
||||
if e.Change == ChangeCleared {
|
||||
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, silenced: e.Condition.Silenced}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
k.say("what cleared lately could not be read from the condition history, so a condition "+
|
||||
"raised again now is said as new rather than reopened: %v", err)
|
||||
}
|
||||
}
|
||||
go k.telling()
|
||||
return k
|
||||
}
|
||||
|
||||
// Close says what is still to be said, waiting at most until ctx ends.
|
||||
func (k *Keeper) Close(ctx context.Context) {
|
||||
k.closing.Do(func() { close(k.out) })
|
||||
select {
|
||||
case <-k.drained:
|
||||
case <-ctx.Done():
|
||||
k.say("%d condition transition(s) were not yet said when this process ended", len(k.out))
|
||||
}
|
||||
}
|
||||
|
||||
// Unsaid is how many transitions were given up on since this keeper started.
|
||||
func (k *Keeper) Unsaid() int {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.unsaid
|
||||
}
|
||||
|
||||
// stamp is the gate every write passes: the epoch it carries, set on the condition, or why this
|
||||
// keeper may not write — a controller that lost the lease raises, observes and clears nothing.
|
||||
func (k *Keeper) stamp(c *Condition) error {
|
||||
if k.epoch == nil {
|
||||
return nil
|
||||
}
|
||||
epoch, err := k.epoch()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the condition %s is not written: %w", c.Key, err)
|
||||
}
|
||||
c.Epoch = epoch
|
||||
return nil
|
||||
}
|
||||
|
||||
// tries bounds one compare-and-set: two writers rarely race more than once.
|
||||
const tries = 8
|
||||
|
||||
// Observe records one observation: raises the condition if it is not open, and otherwise adds the
|
||||
// evidence. Says a raising, a reopening, and a change of severity or resolver; an observation that
|
||||
// changes neither is written and said nowhere.
|
||||
func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) {
|
||||
if err := o.check(); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
key := o.Key()
|
||||
for i := 0; i < tries; i++ {
|
||||
now := k.now().UTC()
|
||||
said := o.Said
|
||||
if said == "" {
|
||||
said = o.Summary
|
||||
}
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
|
||||
Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}},
|
||||
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
|
||||
Resolver: orSelf(o.Resolver)}
|
||||
change := ChangeRaised
|
||||
k.mu.Lock()
|
||||
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= ReopenWithin {
|
||||
c.Count, change = before.count+1, ChangeReopened
|
||||
// A silence a person gave the condition before it cleared still holds: they said
|
||||
// they knew, and the same fault again ten minutes later is what they knew about.
|
||||
if before.silenced != nil && now.Before(before.silenced.Until) {
|
||||
c.Silenced = before.silenced
|
||||
}
|
||||
}
|
||||
k.mu.Unlock()
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Create(ctx, key, body); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("raising the condition %s: %w", key, err)
|
||||
}
|
||||
k.mu.Lock()
|
||||
delete(k.cleared, key)
|
||||
k.mu.Unlock()
|
||||
k.tell(Event{Condition: c, At: now, Change: change})
|
||||
return c, nil
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
var changes []Event
|
||||
if o.Severity != c.Severity {
|
||||
changes = append(changes, Event{Change: ChangeSeverity, Was: string(c.Severity)})
|
||||
c.Severity = o.Severity
|
||||
}
|
||||
if r := orSelf(o.Resolver); o.Resolver != "" && r != c.Resolver {
|
||||
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver})
|
||||
c.Resolver = r
|
||||
}
|
||||
c.Summary, c.Source, c.LastObserved = o.Summary, o.Source, now
|
||||
if o.Machine != "" {
|
||||
c.Subject.Machine = o.Machine
|
||||
}
|
||||
if len(o.Also) > 0 {
|
||||
c.Subject.Also = o.Also
|
||||
}
|
||||
c.Observations++
|
||||
c.Evidence = append([]Evidence{{At: now, Said: said}}, c.Evidence...)
|
||||
if len(c.Evidence) > KeptEvidence {
|
||||
c.Evidence = c.Evidence[:KeptEvidence]
|
||||
}
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err)
|
||||
}
|
||||
for _, e := range changes {
|
||||
e.At, e.Condition = now, c
|
||||
k.tell(e)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
return Condition{}, fmt.Errorf("the condition %s kept moving under this write; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// Clear removes a condition an observation says is resolved, and says so. False when none was open.
|
||||
func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return false, nil
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
// Unreadable is not resolved: kept, and said, rather than removed unread.
|
||||
return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err)
|
||||
}
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("clearing the condition %s: %w", key, err)
|
||||
}
|
||||
now := k.now().UTC()
|
||||
k.mu.Lock()
|
||||
k.cleared[key] = clearing{at: now, count: c.Count, silenced: c.Silenced}
|
||||
k.mu.Unlock()
|
||||
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the condition %s kept moving under this clearing; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// Reconcile is one source's whole observation: every condition it observes is observed, and every
|
||||
// condition it raised before and no longer observes is cleared — the observation says it is
|
||||
// resolved. A source that could not observe must not call this: an empty observation clears all it
|
||||
// raised, which is exactly the fault of saying "none" for "I could not tell" (ADR 0227 rule 4).
|
||||
func (k *Keeper) Reconcile(ctx context.Context, source string, observed []Observation) error {
|
||||
all, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var problems []string
|
||||
for _, o := range observed {
|
||||
o.Source = source
|
||||
seen[o.Key()] = true
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
for _, c := range all {
|
||||
if c.Source != source || seen[c.Key] {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Clear(ctx, c.Key, source+" no longer observes it"); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return errors.New(strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Silence stops a condition's messages for a while, with a reason, by somebody (to-be 45 §2). The
|
||||
// condition stays open and `status` still says it; recording the act in the hand-act log is the
|
||||
// caller's, which knows who acted.
|
||||
func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, why string) (Condition, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return Condition{}, errors.New("a silence says why: --why <text>")
|
||||
}
|
||||
if d <= 0 || d > MaxSilence {
|
||||
return Condition{}, fmt.Errorf("a condition is silenced for a while, at most %s — not %s", MaxSilence, d)
|
||||
}
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return Condition{}, fmt.Errorf("no condition %s is open — `conditions` lists them", key)
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
now := k.now().UTC()
|
||||
c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now}
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("silencing the condition %s: %w", key, err)
|
||||
}
|
||||
k.tell(Event{Condition: c, At: now, Change: ChangeSilenced, Why: c.Silenced.Why})
|
||||
return c, nil
|
||||
}
|
||||
return Condition{}, fmt.Errorf("the condition %s kept moving under this silence; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// EndSilences ends every silence that has run out, and says each: the condition is still open, and
|
||||
// its messages start again.
|
||||
func (k *Keeper) EndSilences(ctx context.Context) error {
|
||||
all, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := k.now().UTC()
|
||||
for _, c := range all {
|
||||
if c.Silenced == nil || now.Before(c.Silenced.Until) {
|
||||
continue
|
||||
}
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, c.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
break
|
||||
}
|
||||
var held Condition
|
||||
if err := json.Unmarshal(entry.Value, &held); err != nil {
|
||||
return fmt.Errorf("the condition %s on the bus cannot be read: %w", c.Key, err)
|
||||
}
|
||||
if held.Silenced == nil || now.Before(held.Silenced.Until) {
|
||||
break
|
||||
}
|
||||
was := held.Silenced.Why
|
||||
held.Silenced = nil
|
||||
if err := k.stamp(&held); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(held)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.store.Update(ctx, c.Key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
k.tell(Event{Condition: held, At: now, Change: ChangeUnsilenced, Why: was})
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open is every open condition, urgent first and then oldest first.
|
||||
func (k *Keeper) Open(ctx context.Context) ([]Condition, error) {
|
||||
return Read(ctx, k.store)
|
||||
}
|
||||
|
||||
// Get is one open condition.
|
||||
func (k *Keeper) Get(ctx context.Context, key string) (Condition, bool, error) {
|
||||
return ReadOne(ctx, k.store, key)
|
||||
}
|
||||
|
||||
// HistorySince is every transition from a moment, oldest first.
|
||||
func (k *Keeper) HistorySince(ctx context.Context, since time.Time) ([]Event, error) {
|
||||
if k.history == nil {
|
||||
return nil, errors.New("this keeper has no history to read")
|
||||
}
|
||||
return k.history.Since(ctx, since)
|
||||
}
|
||||
|
||||
// Read is every open condition in a store, in the order status says them. A value that cannot be
|
||||
// read is an error naming its key, never a condition left out (ADR 0227 rule 4).
|
||||
func Read(ctx context.Context, store Backend) ([]Condition, error) {
|
||||
all, err := store.All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the open conditions cannot be read: %w", err)
|
||||
}
|
||||
out := make([]Condition, 0, len(all))
|
||||
for key, e := range all {
|
||||
var c Condition
|
||||
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||
return nil, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
Order(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ReadOne is one open condition from a store.
|
||||
func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, error) {
|
||||
e, found, err := store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return Condition{}, found, err
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||
return Condition{}, false, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// tell queues a transition to be kept and said. Never blocks the caller for long: a queue that is
|
||||
// full is a bus away for a long time, and the transition is said lost rather than holding a watchdog.
|
||||
func (k *Keeper) tell(e Event) {
|
||||
e.Event = eventFor(e.Change)
|
||||
e.Show = e.Condition.Show()
|
||||
if k.changed != nil {
|
||||
k.changed()
|
||||
}
|
||||
defer func() {
|
||||
// A keeper closed while a write was in flight: said, not a panic.
|
||||
if recover() != nil {
|
||||
k.lost(e, errors.New("the keeper was closed"))
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case k.out <- e:
|
||||
default:
|
||||
k.lost(e, errors.New("too many transitions are waiting to be said"))
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Keeper) lost(e Event, err error) {
|
||||
k.mu.Lock()
|
||||
k.unsaid++
|
||||
k.mu.Unlock()
|
||||
k.say("the condition %s was %s and that could NOT be said or kept: %v", e.Key, e.Change, err)
|
||||
}
|
||||
|
||||
// telling keeps and says every transition in order, offering each again while the bus is away.
|
||||
func (k *Keeper) telling() {
|
||||
defer close(k.drained)
|
||||
for e := range k.out {
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
k.lost(e, err)
|
||||
continue
|
||||
}
|
||||
deadline := time.Now().Add(TellFor)
|
||||
wait := 200 * time.Millisecond
|
||||
kept, said := k.history == nil, k.teller == nil
|
||||
for {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if !kept {
|
||||
kept = k.history.Append(ctx, e) == nil
|
||||
}
|
||||
if !said {
|
||||
said = k.teller.PublishSeatEvent(ctx, Seat, e.Event, body) == nil
|
||||
}
|
||||
cancel()
|
||||
if kept && said {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
what := "said"
|
||||
if !kept {
|
||||
what = "kept in the history"
|
||||
}
|
||||
k.lost(e, fmt.Errorf("not %s within %s", what, TellFor))
|
||||
break
|
||||
}
|
||||
time.Sleep(wait)
|
||||
wait = min(2*wait, 10*time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func orSelf(resolver string) string {
|
||||
if resolver == "" {
|
||||
return ResolverSelf
|
||||
}
|
||||
return resolver
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// clock is a time a test moves by hand.
|
||||
type clock struct{ at time.Time }
|
||||
|
||||
func (c *clock) now() time.Time { return c.at }
|
||||
func (c *clock) pass(d time.Duration) { c.at = c.at.Add(d) }
|
||||
func newClock() *clock { return &clock{at: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)} }
|
||||
func keeper(t *testing.T) (*Keeper, *InMemory, *Told, *clock) {
|
||||
t.Helper()
|
||||
store, told, c := NewInMemory(), &Told{}, newClock()
|
||||
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now,
|
||||
Say: func(f string, a ...any) { t.Logf(f, a...) }})
|
||||
t.Cleanup(func() { k.Close(context.Background()) })
|
||||
return k, store, told, c
|
||||
}
|
||||
|
||||
// settled waits until the teller has been told n events.
|
||||
func settled(t *testing.T, told *Told, n int) []Event {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
said := told.Said()
|
||||
if len(said) >= n {
|
||||
return said
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("told %d event(s), want %d: %+v", len(said), n, said)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func silent(node string) Observation {
|
||||
return Observation{Scope: ScopeMachine, ID: node, Kind: "silent", Machine: node, Severity: Warning,
|
||||
Summary: node + " has not been heard from", Source: "S1"}
|
||||
}
|
||||
|
||||
// **A condition is raised once, observed many times, and said on the bus only when it changes**
|
||||
// (to-be 45 §2): an observation that changes nothing is written and said nowhere, or the operator's
|
||||
// channel would hear the same fault every thirty seconds.
|
||||
func TestAConditionIsSaidWhenItChangesNotWhenItIsSeenAgain(t *testing.T) {
|
||||
k, _, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(time.Minute)
|
||||
}
|
||||
urgent := silent("ace")
|
||||
urgent.Severity = Urgent
|
||||
got, err := k.Observe(ctx, urgent)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Key != "machine.ace.silent" || got.Observations != 4 || got.Count != 1 || got.Severity != Urgent {
|
||||
t.Fatalf("held %+v", got)
|
||||
}
|
||||
if len(got.Evidence) != 4 || !got.Evidence[0].At.Equal(c.at) {
|
||||
t.Fatalf("evidence is not newest first: %+v", got.Evidence)
|
||||
}
|
||||
said := settled(t, told, 2)
|
||||
if said[0].Event != EventRaised || said[0].Change != ChangeRaised || said[1].Event != EventChanged ||
|
||||
said[1].Change != ChangeSeverity || said[1].Was != string(Warning) {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if n := len(told.Said()); n != 2 {
|
||||
t.Fatalf("said %d events for one raising and one change", n)
|
||||
}
|
||||
for i, name := range told.Names {
|
||||
if name != told.Events[i].Event {
|
||||
t.Errorf("event %d published as %s and says it is %s", i, name, told.Events[i].Event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Evidence is bounded**: a condition open for a week keeps its newest ten observations, not all.
|
||||
func TestEvidenceKeepsTheNewestTen(t *testing.T) {
|
||||
k, _, _, c := keeper(t)
|
||||
var got Condition
|
||||
for i := 0; i < 25; i++ {
|
||||
var err error
|
||||
if got, err = k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(time.Minute)
|
||||
}
|
||||
if len(got.Evidence) != KeptEvidence || got.Observations != 25 {
|
||||
t.Fatalf("kept %d evidence of %d observations", len(got.Evidence), got.Observations)
|
||||
}
|
||||
}
|
||||
|
||||
// **Cleared and raised again within ten minutes is the same condition again** (to-be 45 §2): its
|
||||
// count goes up and it is said as reopened, not as news; a person's silence of it still holds.
|
||||
func TestRaisedAgainSoonAfterClearingReopens(t *testing.T) {
|
||||
k, store, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "the laptop is on the train"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cleared, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil || !cleared {
|
||||
t.Fatalf("cleared %v: %v", cleared, err)
|
||||
}
|
||||
c.pass(5 * time.Minute)
|
||||
again, err := k.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Count != 2 || again.Silenced == nil {
|
||||
t.Fatalf("reopened as %+v", again)
|
||||
}
|
||||
said := settled(t, told, 4)
|
||||
if said[3].Event != EventRaised || said[3].Change != ChangeReopened {
|
||||
t.Fatalf("the reopening was said as %+v", said[3])
|
||||
}
|
||||
// And from a new keeper — the controller restarted between — reading what cleared from history.
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settled(t, told, 5)
|
||||
k.Close(context.Background())
|
||||
next := NewKeeper(ctx, Options{Store: store, History: store, Now: c.now})
|
||||
defer next.Close(context.Background())
|
||||
c.pass(time.Minute)
|
||||
third, err := next.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if third.Count != 3 {
|
||||
t.Fatalf("a controller restarted between cleared and raised said it as new: %+v", third)
|
||||
}
|
||||
// Past the window it is news.
|
||||
if _, err := next.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(ReopenWithin + time.Minute)
|
||||
fourth, err := next.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fourth.Count != 1 || fourth.Silenced != nil {
|
||||
t.Fatalf("raised past the window as %+v", fourth)
|
||||
}
|
||||
}
|
||||
|
||||
// **A source's whole observation clears what it no longer observes, and only its own.** A watchdog
|
||||
// that stops seeing a fault says it is resolved; it does not clear what another raised.
|
||||
func TestReconcileClearsOnlyTheSourcesOwn(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
other := Observation{Scope: ScopeProbe, ID: "D3", Kind: "probe-failed", Token: "failed", Severity: Warning,
|
||||
Summary: "D3 did not answer", Source: "doctor"}
|
||||
if _, err := k.Observe(ctx, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.Reconcile(ctx, "S1", []Observation{silent("ace"), silent("g14")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.Reconcile(ctx, "S1", []Observation{silent("g14")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
if strings.Join(keys, ",") != "machine.g14.silent,probe.D3.failed" {
|
||||
t.Fatalf("open after the second observation: %v", keys)
|
||||
}
|
||||
said := settled(t, told, 4)
|
||||
last := said[3]
|
||||
if last.Event != EventCleared || last.Key != "machine.ace.silent" || last.Why == "" {
|
||||
t.Fatalf("the clearing was said as %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
// **A store that cannot be read is never an empty one** (ADR 0227 rule 4): reconciling against it
|
||||
// clears nothing and says why.
|
||||
func TestAnUnreadableStoreClearsNothing(t *testing.T) {
|
||||
k, store, _, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Fail = errors.New("the bus is away")
|
||||
if err := k.Reconcile(ctx, "S1", nil); err == nil {
|
||||
t.Fatal("reconciled against a store it could not read")
|
||||
}
|
||||
if _, err := k.Open(ctx); err == nil {
|
||||
t.Fatal("an unreadable store answered as read")
|
||||
}
|
||||
store.Fail = nil
|
||||
store.values["machine.g14.silent"] = Entry{Value: []byte("{not a condition"), Revision: 99}
|
||||
if _, err := k.Open(ctx); err == nil || !strings.Contains(err.Error(), "machine.g14.silent") {
|
||||
t.Fatalf("an unreadable condition was left out rather than said: %v", err)
|
||||
}
|
||||
if cleared, err := k.Clear(ctx, "machine.g14.silent", "x"); err == nil || cleared {
|
||||
t.Fatal("an unreadable condition was cleared unread")
|
||||
}
|
||||
}
|
||||
|
||||
// **A silence is bounded, says why, and ends on its own** (to-be 45 §2): the condition stays open
|
||||
// through it, and its messages start again when it ends.
|
||||
func TestASilenceIsBoundedAndEnds(t *testing.T) {
|
||||
k, _, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", 8*24*time.Hour, "jochen", "away"); err == nil {
|
||||
t.Fatal("silenced for longer than a week")
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", " "); err == nil {
|
||||
t.Fatal("silenced without a reason")
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.nothing.silent", time.Hour, "jochen", "x"); err == nil {
|
||||
t.Fatal("silenced a condition that is not open")
|
||||
}
|
||||
held, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "on the train")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !held.SilencedAt(c.at) || held.Silenced.By != "jochen" {
|
||||
t.Fatalf("silenced as %+v", held.Silenced)
|
||||
}
|
||||
c.pass(30 * time.Minute)
|
||||
if err := k.EndSilences(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(31 * time.Minute)
|
||||
if err := k.EndSilences(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := k.Get(ctx, "machine.ace.silent")
|
||||
if got.Silenced != nil {
|
||||
t.Fatalf("a silence past its end still held: %+v", got.Silenced)
|
||||
}
|
||||
said := settled(t, told, 3)
|
||||
if said[1].Change != ChangeSilenced || said[2].Change != ChangeUnsilenced || said[2].Event != EventChanged {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// **Two writers never lose each other's word.** The serving controller observes while a person's
|
||||
// command silences: the write that lost the compare-and-set reads again and redoes itself.
|
||||
func TestAWriteThatLostTheRaceRedoesItself(t *testing.T) {
|
||||
k, store, _, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
racing := &racingStore{InMemory: store, before: func() {
|
||||
// Another process silences between this keeper's read and its write.
|
||||
other := NewKeeper(ctx, Options{Store: store})
|
||||
defer other.Close(context.Background())
|
||||
if _, err := other.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "known"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}}
|
||||
k.store = racing
|
||||
got, err := k.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Silenced == nil || got.Observations != 2 {
|
||||
t.Fatalf("the observation overwrote the silence: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// racingStore lets another writer in once, between a read and the write after it.
|
||||
type racingStore struct {
|
||||
*InMemory
|
||||
before func()
|
||||
done bool
|
||||
}
|
||||
|
||||
func (r *racingStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
if !r.done {
|
||||
r.done = true
|
||||
r.before()
|
||||
}
|
||||
return r.InMemory.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// **An observation that could not be routed is refused**, naming what it lacks.
|
||||
func TestAnObservationSaysWhatItIs(t *testing.T) {
|
||||
k, _, _, _ := keeper(t)
|
||||
for _, o := range []Observation{
|
||||
{Scope: "elsewhere", ID: "x", Kind: "k", Severity: Warning, Summary: "s", Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: "loud", Summary: "s", Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Summary: "s"},
|
||||
} {
|
||||
if _, err := k.Observe(t.Context(), o); err == nil {
|
||||
t.Errorf("observed %+v", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A key holds nothing the bus would refuse or read as a wildcard**, whatever the thing is called.
|
||||
func TestAKeyIsSafeForTheBus(t *testing.T) {
|
||||
if got := Key(ScopeProvider, "keycloak.novox.my app*", "failing"); got != "provider.keycloak.novox.my_app_.failing" {
|
||||
t.Fatalf("key %q", got)
|
||||
}
|
||||
if got := Key(ScopeBus, "EVENTS.>", "consumer-lost"); got != "bus.EVENTS._.consumer-lost" {
|
||||
t.Fatalf("key %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The event's shape is a contract** (to-be 45 §2): the operator-channel's holder is written against
|
||||
// these field names. A rename here is a channel that reads nothing, so they are held still.
|
||||
func TestTheEventShapeIsTheContract(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := settled(t, told, 1)
|
||||
body, err := json.Marshal(said[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var shape map[string]any
|
||||
if err := json.Unmarshal(body, &shape); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The condition at the top level, kebab-case, beside what happened to it.
|
||||
for _, field := range []string{"event", "at", "change", "show", "key", "kind", "subject", "severity",
|
||||
"summary", "evidence", "source", "raised", "last-observed", "observations", "count", "resolver",
|
||||
"silenced", "epoch"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the event carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
if shape["silenced"] != nil {
|
||||
t.Errorf("an unsilenced condition says silenced %v, not null", shape["silenced"])
|
||||
}
|
||||
subject, _ := shape["subject"].(map[string]any)
|
||||
if subject["scope"] != "machine" || subject["id"] != "ace" || subject["machine"] != "ace" {
|
||||
t.Errorf("subject %v", shape["subject"])
|
||||
}
|
||||
if said[0].Show != "mesh-controller.conditions key=machine.ace.silent" {
|
||||
t.Errorf("show is %q", said[0].Show)
|
||||
}
|
||||
}
|
||||
|
||||
// **A transition the bus will not take is offered again**, and said lost only after TellFor.
|
||||
func TestATransitionIsOfferedAgainWhileTheBusIsAway(t *testing.T) {
|
||||
store, told, c := NewInMemory(), &Told{Fail: errors.New("no responders")}, newClock()
|
||||
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now})
|
||||
defer k.Close(context.Background())
|
||||
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
told.mu.Lock()
|
||||
told.Fail = nil
|
||||
told.mu.Unlock()
|
||||
said := settled(t, told, 1)
|
||||
if said[0].Key != "machine.ace.silent" || k.Unsaid() != 0 {
|
||||
t.Fatalf("said %+v, unsaid %d", said, k.Unsaid())
|
||||
}
|
||||
}
|
||||
@@ -124,7 +124,7 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
||||
|
||||
d := broker.Declared{
|
||||
Module: m.Module,
|
||||
Emits: m.Emits,
|
||||
Emits: m.EmitsAll(),
|
||||
Consumes: fromModules,
|
||||
Watches: watches,
|
||||
// The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user